From 0f887a758f105bd37d4738b9fa1eaf42a1b0599f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 8 Aug 2026 02:01:14 +0000 Subject: [PATCH] sync(codeapi): import geospatial, Lambda MicroVM, and virtio-fs updates (#2969) Source: ClickHouse/ai@dfc602b9a56968454cdda68254d16d989bc167c2 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: danny-avila <110412045+danny-avila@users.noreply.github.com> --- .github/workflows/ci.yml | 3 - docker/sandbox-runner-healthcheck.sh | 135 +----------- .../templates/worker-sandbox-deployment.yaml | 31 +-- helm/codeapi/values.yaml | 9 - tests/block_root_package_delivery.sh | 104 --------- tests/sandbox_runner_healthcheck.sh | 206 ------------------ 6 files changed, 12 insertions(+), 476 deletions(-) delete mode 100755 tests/sandbox_runner_healthcheck.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ec8559..a8318f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,9 +31,6 @@ jobs: - name: Block-root package delivery run: tests/block_root_package_delivery.sh - - name: Sandbox-runner liveness checks - run: tests/sandbox_runner_healthcheck.sh - - name: Validate sandbox Dockerfiles run: | docker buildx build --check -f api/Dockerfile . diff --git a/docker/sandbox-runner-healthcheck.sh b/docker/sandbox-runner-healthcheck.sh index c639848..fdb03de 100755 --- a/docker/sandbox-runner-healthcheck.sh +++ b/docker/sandbox-runner-healthcheck.sh @@ -2,84 +2,16 @@ set -euo pipefail fd_limit="${SANDBOX_RUNNER_FD_LIVENESS_LIMIT:-40000}" -clock_skew_limit_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS:-0}" -clock_skew_jitter_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS:-2}" timeout_seconds="${SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS:-5}" -manifest_clock_tolerance_seconds=30 port="${PORT:-2000}" -url="${SANDBOX_RUNNER_HEALTHCHECK_URL:-http://127.0.0.1:${port}/api/v2/health}" +url="${SANDBOX_RUNNER_HEALTHCHECK_URL:-http://127.0.0.1:${port}/api/v2/runtimes}" -validate_non_negative_integer() { - local name="$1" - local value="$2" - - case "$value" in - 0|[1-9]|[1-9][0-9]*) ;; - *) - echo "invalid ${name}: ${value} (expected a canonical non-negative integer)" >&2 - exit 2 - ;; - esac -} - -validate_non_negative_integer SANDBOX_RUNNER_FD_LIVENESS_LIMIT "$fd_limit" -validate_non_negative_integer \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \ - "$clock_skew_limit_seconds" -validate_non_negative_integer \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS \ - "$clock_skew_jitter_seconds" -validate_non_negative_integer \ - SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS \ - "$timeout_seconds" - -if [ "$timeout_seconds" -eq 0 ]; then - echo "SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be greater than zero" >&2 - exit 2 -fi - -if [ "$clock_skew_limit_seconds" -gt 0 ]; then - if [ "${#clock_skew_limit_seconds}" -gt 2 ] || \ - [ "$clock_skew_limit_seconds" -ge "$manifest_clock_tolerance_seconds" ]; then - echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the ${manifest_clock_tolerance_seconds}-second execution-manifest tolerance" >&2 +case "$fd_limit" in + ''|*[!0-9]*) + echo "invalid SANDBOX_RUNNER_FD_LIVENESS_LIMIT: $fd_limit" >&2 exit 2 - fi - - remaining_tolerance_seconds=$((manifest_clock_tolerance_seconds - clock_skew_limit_seconds)) - if [ "${#timeout_seconds}" -gt 2 ] || \ - [ "$timeout_seconds" -ge "$remaining_tolerance_seconds" ]; then - echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the ${manifest_clock_tolerance_seconds}-second execution-manifest tolerance" >&2 - exit 2 - fi -fi - -if [ "$clock_skew_limit_seconds" -gt 0 ] && ! command -v date >/dev/null 2>&1; then - echo "sandbox-runner clock-skew check requires date" >&2 - exit 2 -fi - -effective_clock_skew_limit_seconds="$clock_skew_limit_seconds" -if [ "$clock_skew_limit_seconds" -gt 0 ] && [ "$clock_skew_jitter_seconds" -gt 0 ]; then - if [ "${#clock_skew_jitter_seconds}" -gt 2 ] || \ - [ "$clock_skew_jitter_seconds" -ge "$clock_skew_limit_seconds" ]; then - echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS must be less than SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS" >&2 - exit 2 - fi - if ! command -v cksum >/dev/null 2>&1; then - echo "sandbox-runner clock-skew jitter requires cksum" >&2 - exit 2 - fi - - # Pods from one rollout have nearly identical ages and drift rates. Give - # each pod a stable threshold within the configured range to reduce the - # chance that same-age replicas restart together. - jitter_key="${HOSTNAME:-sandbox-runner}" - jitter_checksum=$(printf '%s' "$jitter_key" | cksum) - jitter_checksum="${jitter_checksum%% *}" - effective_clock_skew_limit_seconds=$(( - clock_skew_limit_seconds - (jitter_checksum % (clock_skew_jitter_seconds + 1)) - )) -fi + ;; +esac if [ "$fd_limit" -gt 0 ]; then fd_count=$(find /proc/1/fd -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d '[:space:]') @@ -89,57 +21,4 @@ if [ "$fd_limit" -gt 0 ]; then fi fi -if [ "$clock_skew_limit_seconds" -eq 0 ]; then - curl -fsS --max-time "$timeout_seconds" "$url" >/dev/null - exit 0 -fi - -host_before_seconds=$(date -u +%s) -response_headers=$(curl -fsS --max-time "$timeout_seconds" --dump-header - --output /dev/null "$url") -host_after_seconds=$(date -u +%s) - -guest_date=$(printf '%s\n' "$response_headers" | awk ' - tolower($1) == "date:" { - sub(/\r$/, "") - sub(/^[^:]*:[[:space:]]*/, "") - value = $0 - } - END { print value } -') - -if [ -z "$guest_date" ]; then - echo "sandbox-runner unhealthy: guest response is missing the HTTP Date header" >&2 - exit 1 -fi - -http_date_pattern='^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), [0-9]{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} GMT$' -if ! [[ "$guest_date" =~ $http_date_pattern ]]; then - echo "sandbox-runner unhealthy: guest returned an invalid HTTP Date header: ${guest_date}" >&2 - exit 1 -fi - -if ! guest_seconds=$(date -u -d "$guest_date" +%s 2>/dev/null); then - echo "sandbox-runner unhealthy: guest returned an invalid HTTP Date header: ${guest_date}" >&2 - exit 1 -fi - -# The response can take time to arrive, so compare guest time with the host -# interval that enclosed the request instead of with a single sample. This -# prevents probe latency from looking like backward clock drift. -if [ "$guest_seconds" -lt "$host_before_seconds" ]; then - skew_seconds=$((guest_seconds - host_before_seconds)) -elif [ "$guest_seconds" -gt "$host_after_seconds" ]; then - skew_seconds=$((guest_seconds - host_after_seconds)) -else - skew_seconds=0 -fi - -absolute_skew_seconds="$skew_seconds" -if [ "$absolute_skew_seconds" -lt 0 ]; then - absolute_skew_seconds=$((-absolute_skew_seconds)) -fi - -if [ "$absolute_skew_seconds" -ge "$effective_clock_skew_limit_seconds" ]; then - echo "sandbox-runner unhealthy: guest clock skew is ${skew_seconds}s, pod limit is ${effective_clock_skew_limit_seconds}s (configured maximum ${clock_skew_limit_seconds}s)" >&2 - exit 1 -fi +curl -fsS --max-time "$timeout_seconds" "$url" >/dev/null diff --git a/helm/codeapi/templates/worker-sandbox-deployment.yaml b/helm/codeapi/templates/worker-sandbox-deployment.yaml index 3484a61..8a82a61 100644 --- a/helm/codeapi/templates/worker-sandbox-deployment.yaml +++ b/helm/codeapi/templates/worker-sandbox-deployment.yaml @@ -27,22 +27,6 @@ Split runtime deployments: {{- if and (not $packagesFromPvc) (not .Values.workerSandbox.kvmEnabled) }} {{- fail "workerSandbox.packages.source=image requires workerSandbox.kvmEnabled=true because the baked runner boots from a libkrun block root image" }} {{- end }} -{{- $clockSkewLimitValue := toString .Values.workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds }} -{{- $clockSkewJitterValue := toString .Values.workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds }} -{{- if not (regexMatch "^(0|[1-9][0-9]*)$" $clockSkewLimitValue) }} -{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds must be a non-negative integer" }} -{{- end }} -{{- if not (regexMatch "^(0|[1-9][0-9]*)$" $clockSkewJitterValue) }} -{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds must be a non-negative integer" }} -{{- end }} -{{- $clockSkewLimit := int $clockSkewLimitValue }} -{{- $clockSkewJitter := int $clockSkewJitterValue }} -{{- if or (lt $clockSkewLimit 0) (ge $clockSkewLimit 25) }} -{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds must be between 0 and 24 so the 5-second healthcheck timeout stays below the 30-second execution-manifest tolerance" }} -{{- end }} -{{- if or (lt $clockSkewJitter 0) (and (gt $clockSkewLimit 0) (ge $clockSkewJitter $clockSkewLimit)) }} -{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds must be non-negative and less than clockSkewLivenessLimitSeconds when the clock-skew guard is enabled" }} -{{- end }} {{- range .Values.workerSandbox.sandboxExtraEnv }} {{- $name := .name | default "" }} {{- if and $name (or (regexMatch "(?i)(SECRET|TOKEN|PASSWORD|KEY)" $name) (hasPrefix "REDIS_" $name) (hasPrefix "AWS_" $name) (hasPrefix "S3_" $name) (hasPrefix "MINIO_" $name) (hasPrefix "CODEAPI_" $name) (eq $name "FILE_SERVER_URL") (eq $name "TOOL_CALL_SERVER_URL")) }} @@ -274,10 +258,6 @@ spec: value: {{ .Values.workerSandbox.launcher.filterVsockEnotconn | quote }} - name: SANDBOX_RUNNER_FD_LIVENESS_LIMIT value: {{ .Values.workerSandbox.sandboxRunner.fdLivenessLimit | quote }} - - name: SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS - value: {{ $clockSkewLimit | quote }} - - name: SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS - value: {{ $clockSkewJitter | quote }} - name: EGRESS_GATEWAY_URL value: "http://{{ include "codeapi.fullname" . }}-egress-gateway:{{ .Values.egressGateway.service.port }}" - name: SANDBOX_LOG_LEVEL @@ -382,15 +362,14 @@ spec: - /usr/local/bin/sandbox-runner-healthcheck.sh initialDelaySeconds: 60 periodSeconds: 30 - timeoutSeconds: 7 + timeoutSeconds: 5 readinessProbe: - exec: - command: - - /usr/local/bin/sandbox-runner-healthcheck.sh + httpGet: + path: /api/v2/health + port: sandbox initialDelaySeconds: 30 periodSeconds: 10 - timeoutSeconds: 7 - failureThreshold: 1 + timeoutSeconds: 5 resources: {{- $sandboxResources := deepCopy .Values.workerSandbox.resources }} {{- if $useKvmDevicePlugin }} diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 2385814..c926e76 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -200,15 +200,6 @@ workerSandbox: # Restart a sandbox-runner before launcher/VMM host-side fd retention can # exhaust the process file descriptor limit. Set 0 to disable. fdLivenessLimit: 40000 - # Restart a sandbox-runner when its microVM guest wall clock differs from - # the host by this many seconds. This must remain comfortably below the - # 30-second execution-manifest clock tolerance; the chart reserves the - # five-second probe timeout and accepts at most 24. Set 0 to disable. - clockSkewLivenessLimitSeconds: 10 - # Subtract a stable 0..N-second offset from each pod's clock-skew limit so - # replicas created together do not all recycle at once. Must be less than - # clockSkewLivenessLimitSeconds. - clockSkewLivenessJitterSeconds: 2 inheritSharedScheduling: true nodeSelector: {} tolerations: [] diff --git a/tests/block_root_package_delivery.sh b/tests/block_root_package_delivery.sh index 20ec6ea..c549533 100755 --- a/tests/block_root_package_delivery.sh +++ b/tests/block_root_package_delivery.sh @@ -36,23 +36,6 @@ assert_not_contains() { fi } -assert_env_value() { - local file="$1" - local name="$2" - local expected="$3" - local message="$4" - if ! awk -v name="$name" -v expected="$expected" ' - $0 ~ "- name: " name { - getline - if ($0 ~ "value: \\\"" expected "\\\"") found = 1 - } - END { exit found ? 0 : 1 } - ' "$file"; then - echo "$message" >&2 - exit 1 - fi -} - assert_compose_mode() { local compose_file="$1" local service="$2" @@ -213,25 +196,6 @@ assert_not_contains \ "$TMP_DIR/helm-image.yaml" \ 'app.kubernetes.io/component: package-init' \ "default Helm render must not create package-init" -assert_env_value \ - "$TMP_DIR/helm-image.yaml" \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \ - 10 \ - "default Helm render must keep clock skew below the manifest tolerance" -assert_env_value \ - "$TMP_DIR/helm-image.yaml" \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS \ - 2 \ - "default Helm render must stagger clock-skew recycling across replicas" -assert_contains \ - "$TMP_DIR/helm-image.yaml" \ - '^[[:space:]]+failureThreshold: 1$' \ - "sandbox-runner readiness must fail immediately when clock skew is detected" - -if [ "$(grep -Fc -- '- /usr/local/bin/sandbox-runner-healthcheck.sh' "$TMP_DIR/helm-image.yaml")" -lt 2 ]; then - echo "sandbox-runner liveness and readiness must both check guest clock skew" >&2 - exit 1 -fi helm template codeapi "$TMP_DIR/chart" \ --set executionManifest.privateKey=test \ @@ -279,74 +243,6 @@ assert_contains \ 'value: "0"' \ "fdLivenessLimit=0 must reach the healthcheck instead of reverting to 40000" -helm template codeapi "$TMP_DIR/chart" \ - --set executionManifest.privateKey=test \ - --set executionManifest.publicKey=test \ - --set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=0 \ - > "$TMP_DIR/helm-no-clock-skew-liveness.yaml" - -assert_env_value \ - "$TMP_DIR/helm-no-clock-skew-liveness.yaml" \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \ - 0 \ - "clockSkewLivenessLimitSeconds=0 must reach the healthcheck instead of reverting to 10" - -if helm template codeapi "$TMP_DIR/chart" \ - --set executionManifest.privateKey=test \ - --set executionManifest.publicKey=test \ - --set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=25 \ - > "$TMP_DIR/invalid-clock-skew-limit.yaml" 2> "$TMP_DIR/invalid-clock-skew-limit.log"; then - echo "clock-skew liveness limit must stay below manifest tolerance" >&2 - exit 1 -fi - -assert_contains \ - "$TMP_DIR/invalid-clock-skew-limit.log" \ - 'clockSkewLivenessLimitSeconds must be between 0 and 24' \ - "clock-skew limit validation failed for an unexpected reason" - -if helm template codeapi "$TMP_DIR/chart" \ - --set executionManifest.privateKey=test \ - --set executionManifest.publicKey=test \ - --set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=29.5 \ - > "$TMP_DIR/fractional-clock-skew-limit.yaml" 2> "$TMP_DIR/fractional-clock-skew-limit.log"; then - echo "clock-skew liveness limit must reject fractional values" >&2 - exit 1 -fi - -assert_contains \ - "$TMP_DIR/fractional-clock-skew-limit.log" \ - 'clockSkewLivenessLimitSeconds must be a non-negative integer' \ - "fractional clock-skew limit validation failed for an unexpected reason" - -if helm template codeapi "$TMP_DIR/chart" \ - --set executionManifest.privateKey=test \ - --set executionManifest.publicKey=test \ - --set workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds=10 \ - > "$TMP_DIR/invalid-clock-skew-jitter.yaml" 2> "$TMP_DIR/invalid-clock-skew-jitter.log"; then - echo "clock-skew liveness jitter must stay below the configured limit" >&2 - exit 1 -fi - -assert_contains \ - "$TMP_DIR/invalid-clock-skew-jitter.log" \ - 'clockSkewLivenessJitterSeconds must be non-negative and less than clockSkewLivenessLimitSeconds' \ - "clock-skew jitter validation failed for an unexpected reason" - -if helm template codeapi "$TMP_DIR/chart" \ - --set executionManifest.privateKey=test \ - --set executionManifest.publicKey=test \ - --set workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds=1.5 \ - > "$TMP_DIR/fractional-clock-skew-jitter.yaml" 2> "$TMP_DIR/fractional-clock-skew-jitter.log"; then - echo "clock-skew liveness jitter must reject fractional values" >&2 - exit 1 -fi - -assert_contains \ - "$TMP_DIR/fractional-clock-skew-jitter.log" \ - 'clockSkewLivenessJitterSeconds must be a non-negative integer' \ - "fractional clock-skew jitter validation failed for an unexpected reason" - if helm template codeapi "$TMP_DIR/chart" \ --set executionManifest.privateKey=test \ --set executionManifest.publicKey=test \ diff --git a/tests/sandbox_runner_healthcheck.sh b/tests/sandbox_runner_healthcheck.sh deleted file mode 100755 index 583ff7f..0000000 --- a/tests/sandbox_runner_healthcheck.sh +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -TMP_DIR="$(mktemp -d)" -trap 'rm -rf "$TMP_DIR"' EXIT - -mkdir -p "$TMP_DIR/bin" - -cat > "$TMP_DIR/bin/curl" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail - -if [[ "${TEST_CURL_FAIL:-false}" == "true" ]]; then - exit 22 -fi - -if [[ -n "${TEST_EXPECTED_URL:-}" && "${!#}" != "$TEST_EXPECTED_URL" ]]; then - printf 'unexpected healthcheck URL: %s\n' "${!#}" >&2 - exit 2 -fi - -printf 'HTTP/1.1 200 OK\r\n' -if [[ -n "${TEST_GUEST_DATE:-}" ]]; then - printf 'Date: %s\r\n' "$TEST_GUEST_DATE" -fi -printf '\r\n' -EOF - -cat > "$TMP_DIR/bin/date" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail - -if [[ "$#" -eq 2 && "$1" == "-u" && "$2" == "+%s" ]]; then - if [[ -e "${TEST_HOST_SAMPLE_STATE:?}" ]]; then - printf '%s\n' "${TEST_HOST_AFTER_SECONDS:?}" - else - : > "$TEST_HOST_SAMPLE_STATE" - printf '%s\n' "${TEST_HOST_BEFORE_SECONDS:?}" - fi - exit 0 -fi - -if [[ "$#" -eq 4 && "$1" == "-u" && "$2" == "-d" && "$4" == "+%s" ]]; then - if [[ "${TEST_GUEST_DATE_INVALID:-false}" == "true" ]]; then - exit 1 - fi - printf '%s\n' "${TEST_GUEST_SECONDS:?}" - exit 0 -fi - -printf 'unexpected date arguments: %q\n' "$*" >&2 -exit 2 -EOF - -cat > "$TMP_DIR/bin/cksum" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail - -cat >/dev/null -printf '%s 1\n' "${TEST_CKSUM:-0}" -EOF - -chmod +x "$TMP_DIR/bin/curl" "$TMP_DIR/bin/date" "$TMP_DIR/bin/cksum" - -run_check() { - rm -f "$TMP_DIR/host-sample" - env \ - PATH="$TMP_DIR/bin:$PATH" \ - SANDBOX_RUNNER_FD_LIVENESS_LIMIT=0 \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS=0 \ - TEST_HOST_BEFORE_SECONDS=100 \ - TEST_HOST_AFTER_SECONDS=100 \ - TEST_HOST_SAMPLE_STATE="$TMP_DIR/host-sample" \ - TEST_GUEST_DATE='Tue, 04 Aug 2026 08:00:00 GMT' \ - TEST_EXPECTED_URL='http://127.0.0.1:2000/api/v2/health' \ - "$@" \ - bash "$ROOT/docker/sandbox-runner-healthcheck.sh" -} - -run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ - TEST_GUEST_SECONDS=91 - -# A guest timestamp inside the host interval is healthy even when either -# individual host sample differs by more than the configured limit. -run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=1 \ - TEST_HOST_BEFORE_SECONDS=100 \ - TEST_HOST_AFTER_SECONDS=104 \ - TEST_GUEST_SECONDS=102 - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ - TEST_GUEST_SECONDS=90 2> "$TMP_DIR/backward.log"; then - echo "healthcheck accepted guest clock at the backward-skew limit" >&2 - exit 1 -fi -grep -F 'guest clock skew is -10s, pod limit is 10s (configured maximum 10s)' "$TMP_DIR/backward.log" >/dev/null - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ - TEST_GUEST_SECONDS=110 2> "$TMP_DIR/forward.log"; then - echo "healthcheck accepted guest clock at the forward-skew limit" >&2 - exit 1 -fi -grep -F 'guest clock skew is 10s, pod limit is 10s (configured maximum 10s)' "$TMP_DIR/forward.log" >/dev/null - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS=2 \ - TEST_CKSUM=2 \ - TEST_GUEST_SECONDS=92 2> "$TMP_DIR/jitter.log"; then - echo "healthcheck did not apply its deterministic per-pod threshold" >&2 - exit 1 -fi -grep -F 'guest clock skew is -8s, pod limit is 8s (configured maximum 10s)' "$TMP_DIR/jitter.log" >/dev/null - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ - TEST_GUEST_DATE= \ - TEST_GUEST_SECONDS=100 2> "$TMP_DIR/missing.log"; then - echo "healthcheck accepted a response without a Date header" >&2 - exit 1 -fi -grep -F 'guest response is missing the HTTP Date header' "$TMP_DIR/missing.log" >/dev/null - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ - TEST_GUEST_DATE_INVALID=true \ - TEST_GUEST_SECONDS=100 2> "$TMP_DIR/malformed.log"; then - echo "healthcheck accepted a malformed Date header" >&2 - exit 1 -fi -grep -F 'guest returned an invalid HTTP Date header' "$TMP_DIR/malformed.log" >/dev/null - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ - TEST_GUEST_DATE=now \ - TEST_GUEST_SECONDS=100 2> "$TMP_DIR/non-http-date.log"; then - echo "healthcheck accepted a GNU-date-parseable value that is not an HTTP-date" >&2 - exit 1 -fi -grep -F 'guest returned an invalid HTTP Date header: now' "$TMP_DIR/non-http-date.log" >/dev/null - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ - TEST_CURL_FAIL=true \ - TEST_GUEST_SECONDS=100; then - echo "healthcheck accepted a failed guest request" >&2 - exit 1 -fi - -run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=0 \ - TEST_GUEST_DATE= \ - TEST_GUEST_SECONDS=100 - -# Compose and standalone Docker callers only observe health status, so the -# clock-skew guard stays disabled unless an orchestrator explicitly opts in. -run_check \ - TEST_GUEST_DATE= \ - TEST_GUEST_SECONDS=100 - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=invalid \ - TEST_GUEST_SECONDS=100 2> "$TMP_DIR/invalid.log"; then - echo "healthcheck accepted an invalid clock-skew limit" >&2 - exit 1 -fi -grep -F 'invalid SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS: invalid' "$TMP_DIR/invalid.log" >/dev/null - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=08 \ - TEST_GUEST_SECONDS=100 2> "$TMP_DIR/leading-zero.log"; then - echo "healthcheck accepted a non-canonical clock-skew limit" >&2 - exit 1 -fi -grep -F 'expected a canonical non-negative integer' "$TMP_DIR/leading-zero.log" >/dev/null - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=9223372036854775807 \ - TEST_GUEST_SECONDS=100 2> "$TMP_DIR/overflow-limit.log"; then - echo "healthcheck accepted a clock-skew limit that could overflow arithmetic" >&2 - exit 1 -fi -grep -F 'plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the 30-second execution-manifest tolerance' "$TMP_DIR/overflow-limit.log" >/dev/null - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=25 \ - TEST_GUEST_SECONDS=100 2> "$TMP_DIR/unsafe-limit.log"; then - echo "healthcheck accepted a limit without request-latency headroom" >&2 - exit 1 -fi -grep -F 'plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the 30-second execution-manifest tolerance' "$TMP_DIR/unsafe-limit.log" >/dev/null - -if run_check \ - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ - SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS=0 \ - TEST_GUEST_SECONDS=100 2> "$TMP_DIR/zero-timeout.log"; then - echo "healthcheck accepted a zero request timeout" >&2 - exit 1 -fi -grep -F 'SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be greater than zero' "$TMP_DIR/zero-timeout.log" >/dev/null - -echo "sandbox-runner healthcheck checks passed"