Skip to content

sync(codeapi): import geospatial, Lambda MicroVM, and virtio-fs updates (#2969) #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
github-actions wants to merge 1 commit into from
Closed

sync(codeapi): import geospatial, Lambda MicroVM, and virtio-fs updates (#2969) #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,6 @@ jobs:
- name: Block-root package delivery
run: tests/block_root_package_delivery.sh

- name: Sandbox-runner liveness checks
run: tests/sandbox_runner_healthcheck.sh

- name: Validate sandbox Dockerfiles
run: |
docker buildx build --check -f api/Dockerfile .
Expand Down
135 changes: 7 additions & 128 deletions docker/sandbox-runner-healthcheck.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,84 +2,16 @@
set -euo pipefail

fd_limit="${SANDBOX_RUNNER_FD_LIVENESS_LIMIT:-40000}"
clock_skew_limit_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS:-0}"
clock_skew_jitter_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS:-2}"
timeout_seconds="${SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS:-5}"
manifest_clock_tolerance_seconds=30
port="${PORT:-2000}"
url="${SANDBOX_RUNNER_HEALTHCHECK_URL:-http://127.0.0.1:${port}/api/v2/health}"
url="${SANDBOX_RUNNER_HEALTHCHECK_URL:-http://127.0.0.1:${port}/api/v2/runtimes}"

validate_non_negative_integer() {
local name="$1"
local value="$2"

case "$value" in
0|[1-9]|[1-9][0-9]*) ;;
*)
echo "invalid ${name}: ${value} (expected a canonical non-negative integer)" >&2
exit 2
;;
esac
}

validate_non_negative_integer SANDBOX_RUNNER_FD_LIVENESS_LIMIT "$fd_limit"
validate_non_negative_integer \
SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \
"$clock_skew_limit_seconds"
validate_non_negative_integer \
SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS \
"$clock_skew_jitter_seconds"
validate_non_negative_integer \
SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS \
"$timeout_seconds"

if [ "$timeout_seconds" -eq 0 ]; then
echo "SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be greater than zero" >&2
exit 2
fi

if [ "$clock_skew_limit_seconds" -gt 0 ]; then
if [ "${#clock_skew_limit_seconds}" -gt 2 ] || \
[ "$clock_skew_limit_seconds" -ge "$manifest_clock_tolerance_seconds" ]; then
echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the ${manifest_clock_tolerance_seconds}-second execution-manifest tolerance" >&2
case "$fd_limit" in
''|*[!0-9]*)
echo "invalid SANDBOX_RUNNER_FD_LIVENESS_LIMIT: $fd_limit" >&2
exit 2
fi

remaining_tolerance_seconds=$((manifest_clock_tolerance_seconds - clock_skew_limit_seconds))
if [ "${#timeout_seconds}" -gt 2 ] || \
[ "$timeout_seconds" -ge "$remaining_tolerance_seconds" ]; then
echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the ${manifest_clock_tolerance_seconds}-second execution-manifest tolerance" >&2
exit 2
fi
fi

if [ "$clock_skew_limit_seconds" -gt 0 ] && ! command -v date >/dev/null 2>&1; then
echo "sandbox-runner clock-skew check requires date" >&2
exit 2
fi

effective_clock_skew_limit_seconds="$clock_skew_limit_seconds"
if [ "$clock_skew_limit_seconds" -gt 0 ] && [ "$clock_skew_jitter_seconds" -gt 0 ]; then
if [ "${#clock_skew_jitter_seconds}" -gt 2 ] || \
[ "$clock_skew_jitter_seconds" -ge "$clock_skew_limit_seconds" ]; then
echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS must be less than SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS" >&2
exit 2
fi
if ! command -v cksum >/dev/null 2>&1; then
echo "sandbox-runner clock-skew jitter requires cksum" >&2
exit 2
fi

# Pods from one rollout have nearly identical ages and drift rates. Give
# each pod a stable threshold within the configured range to reduce the
# chance that same-age replicas restart together.
jitter_key="${HOSTNAME:-sandbox-runner}"
jitter_checksum=$(printf '%s' "$jitter_key" | cksum)
jitter_checksum="${jitter_checksum%% *}"
effective_clock_skew_limit_seconds=$((
clock_skew_limit_seconds - (jitter_checksum % (clock_skew_jitter_seconds + 1))
))
fi
;;
esac

if [ "$fd_limit" -gt 0 ]; then
fd_count=$(find /proc/1/fd -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d '[:space:]')
Expand All @@ -89,57 +21,4 @@ if [ "$fd_limit" -gt 0 ]; then
fi
fi

if [ "$clock_skew_limit_seconds" -eq 0 ]; then
curl -fsS --max-time "$timeout_seconds" "$url" >/dev/null
exit 0
fi

host_before_seconds=$(date -u +%s)
response_headers=$(curl -fsS --max-time "$timeout_seconds" --dump-header - --output /dev/null "$url")
host_after_seconds=$(date -u +%s)

guest_date=$(printf '%s\n' "$response_headers" | awk '
tolower($1) == "date:" {
sub(/\r$/, "")
sub(/^[^:]*:[[:space:]]*/, "")
value = $0
}
END { print value }
')

if [ -z "$guest_date" ]; then
echo "sandbox-runner unhealthy: guest response is missing the HTTP Date header" >&2
exit 1
fi

http_date_pattern='^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), [0-9]{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} GMT$'
if ! [[ "$guest_date" =~ $http_date_pattern ]]; then
echo "sandbox-runner unhealthy: guest returned an invalid HTTP Date header: ${guest_date}" >&2
exit 1
fi

if ! guest_seconds=$(date -u -d "$guest_date" +%s 2>/dev/null); then
echo "sandbox-runner unhealthy: guest returned an invalid HTTP Date header: ${guest_date}" >&2
exit 1
fi

# The response can take time to arrive, so compare guest time with the host
# interval that enclosed the request instead of with a single sample. This
# prevents probe latency from looking like backward clock drift.
if [ "$guest_seconds" -lt "$host_before_seconds" ]; then
skew_seconds=$((guest_seconds - host_before_seconds))
elif [ "$guest_seconds" -gt "$host_after_seconds" ]; then
skew_seconds=$((guest_seconds - host_after_seconds))
else
skew_seconds=0
fi

absolute_skew_seconds="$skew_seconds"
if [ "$absolute_skew_seconds" -lt 0 ]; then
absolute_skew_seconds=$((-absolute_skew_seconds))
fi

if [ "$absolute_skew_seconds" -ge "$effective_clock_skew_limit_seconds" ]; then
echo "sandbox-runner unhealthy: guest clock skew is ${skew_seconds}s, pod limit is ${effective_clock_skew_limit_seconds}s (configured maximum ${clock_skew_limit_seconds}s)" >&2
exit 1
fi
curl -fsS --max-time "$timeout_seconds" "$url" >/dev/null

@macroscopeapp macroscopeapp Bot Aug 8, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 High docker/sandbox-runner-healthcheck.sh:24

The liveness probe now exits after a plain HTTP request to $url and no longer checks the guest's clock against the host. When a long-lived microVM drifts past the execution-manifest clock tolerance, /api/v2/runtimes still returns 200, so the probe passes and Kubernetes leaves the pod running while manifest-validated executions continue to fail instead of triggering a restart.

The diff deletes the entire clock-skew check (the clock_skew_limit_seconds branch that compares the guest Date header against host_before_seconds/host_after_seconds). If removing this check is intentional, consider documenting why the clock-skew guard is no longer needed. Otherwise, restore the skew detection so drifted pods are correctly marked unhealthy.

🚀 Reply "fix it for me" or copy this AI Prompt for your agent: 
In file @docker/sandbox-runner-healthcheck.sh around line 24:

The liveness probe now exits after a plain HTTP request to `$url` and no longer checks the guest's clock against the host. When a long-lived microVM drifts past the execution-manifest clock tolerance, `/api/v2/runtimes` still returns 200, so the probe passes and Kubernetes leaves the pod running while manifest-validated executions continue to fail instead of triggering a restart.

The diff deletes the entire clock-skew check (the `clock_skew_limit_seconds` branch that compares the guest `Date` header against `host_before_seconds`/`host_after_seconds`). If removing this check is intentional, consider documenting why the clock-skew guard is no longer needed. Otherwise, restore the skew detection so drifted pods are correctly marked unhealthy.

31 changes: 5 additions & 26 deletions helm/codeapi/templates/worker-sandbox-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,22 +27,6 @@ Split runtime deployments:
{{- if and (not $packagesFromPvc) (not .Values.workerSandbox.kvmEnabled) }}
{{- fail "workerSandbox.packages.source=image requires workerSandbox.kvmEnabled=true because the baked runner boots from a libkrun block root image" }}
{{- end }}
{{- $clockSkewLimitValue := toString .Values.workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds }}
{{- $clockSkewJitterValue := toString .Values.workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds }}
{{- if not (regexMatch "^(0|[1-9][0-9]*)$" $clockSkewLimitValue) }}
{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds must be a non-negative integer" }}
{{- end }}
{{- if not (regexMatch "^(0|[1-9][0-9]*)$" $clockSkewJitterValue) }}
{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds must be a non-negative integer" }}
{{- end }}
{{- $clockSkewLimit := int $clockSkewLimitValue }}
{{- $clockSkewJitter := int $clockSkewJitterValue }}
{{- if or (lt $clockSkewLimit 0) (ge $clockSkewLimit 25) }}
{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds must be between 0 and 24 so the 5-second healthcheck timeout stays below the 30-second execution-manifest tolerance" }}
{{- end }}
{{- if or (lt $clockSkewJitter 0) (and (gt $clockSkewLimit 0) (ge $clockSkewJitter $clockSkewLimit)) }}
{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds must be non-negative and less than clockSkewLivenessLimitSeconds when the clock-skew guard is enabled" }}
{{- end }}
{{- range .Values.workerSandbox.sandboxExtraEnv }}
{{- $name := .name | default "" }}
{{- if and $name (or (regexMatch "(?i)(SECRET|TOKEN|PASSWORD|KEY)" $name) (hasPrefix "REDIS_" $name) (hasPrefix "AWS_" $name) (hasPrefix "S3_" $name) (hasPrefix "MINIO_" $name) (hasPrefix "CODEAPI_" $name) (eq $name "FILE_SERVER_URL") (eq $name "TOOL_CALL_SERVER_URL")) }}
Expand Down Expand Up @@ -274,10 +258,6 @@ spec:
value: {{ .Values.workerSandbox.launcher.filterVsockEnotconn | quote }}
- name: SANDBOX_RUNNER_FD_LIVENESS_LIMIT
value: {{ .Values.workerSandbox.sandboxRunner.fdLivenessLimit | quote }}
- name: SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS
value: {{ $clockSkewLimit | quote }}
- name: SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS
value: {{ $clockSkewJitter | quote }}
- name: EGRESS_GATEWAY_URL
value: "http://{{ include "codeapi.fullname" . }}-egress-gateway:{{ .Values.egressGateway.service.port }}"
- name: SANDBOX_LOG_LEVEL
Expand Down Expand Up @@ -382,15 +362,14 @@ spec:
- /usr/local/bin/sandbox-runner-healthcheck.sh
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 7
timeoutSeconds: 5
readinessProbe:

@macroscopeapp macroscopeapp Bot Aug 8, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 High templates/worker-sandbox-deployment.yaml:366

The readiness probe was changed from the exec healthcheck script to httpGet on /api/v2/health, which bypasses the FD liveness check (SANDBOX_RUNNER_FD_LIVENESS_LIMIT). Kubernetes will continue routing executions to a runner whose PID 1 has hit the FD limit — /api/v2/health still returns 200 — until the slower liveness probe (30-second periods) eventually restarts it. Consider keeping the FD-aware script in the readiness probe, or incorporate that check into the HTTP endpoint.

🚀 Reply "fix it for me" or copy this AI Prompt for your agent: 
In file @helm/codeapi/templates/worker-sandbox-deployment.yaml around line 366:

The readiness probe was changed from the `exec` healthcheck script to `httpGet` on `/api/v2/health`, which bypasses the FD liveness check (`SANDBOX_RUNNER_FD_LIVENESS_LIMIT`). Kubernetes will continue routing executions to a runner whose PID 1 has hit the FD limit — `/api/v2/health` still returns 200 — until the slower liveness probe (30-second periods) eventually restarts it. Consider keeping the FD-aware script in the readiness probe, or incorporate that check into the HTTP endpoint.

exec:
command:
- /usr/local/bin/sandbox-runner-healthcheck.sh
httpGet:
path: /api/v2/health
port: sandbox
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 7
failureThreshold: 1
timeoutSeconds: 5
resources:
{{- $sandboxResources := deepCopy .Values.workerSandbox.resources }}
{{- if $useKvmDevicePlugin }}
Expand Down
9 changes: 0 additions & 9 deletions helm/codeapi/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,15 +200,6 @@ workerSandbox:
# Restart a sandbox-runner before launcher/VMM host-side fd retention can
# exhaust the process file descriptor limit. Set 0 to disable.
fdLivenessLimit: 40000
# Restart a sandbox-runner when its microVM guest wall clock differs from
# the host by this many seconds. This must remain comfortably below the
# 30-second execution-manifest clock tolerance; the chart reserves the
# five-second probe timeout and accepts at most 24. Set 0 to disable.
clockSkewLivenessLimitSeconds: 10
# Subtract a stable 0..N-second offset from each pod's clock-skew limit so
# replicas created together do not all recycle at once. Must be less than
# clockSkewLivenessLimitSeconds.
clockSkewLivenessJitterSeconds: 2
inheritSharedScheduling: true
nodeSelector: {}
tolerations: []
Expand Down
104 changes: 0 additions & 104 deletions tests/block_root_package_delivery.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,23 +36,6 @@ assert_not_contains() {
fi
}

assert_env_value() {
local file="$1"
local name="$2"
local expected="$3"
local message="$4"
if ! awk -v name="$name" -v expected="$expected" '
$0 ~ "- name: " name {
getline
if ($0 ~ "value: \\\"" expected "\\\"") found = 1
}
END { exit found ? 0 : 1 }
' "$file"; then
echo "$message" >&2
exit 1
fi
}

assert_compose_mode() {
local compose_file="$1"
local service="$2"
Expand Down Expand Up @@ -213,25 +196,6 @@ assert_not_contains \
"$TMP_DIR/helm-image.yaml" \
'app.kubernetes.io/component: package-init' \
"default Helm render must not create package-init"
assert_env_value \
"$TMP_DIR/helm-image.yaml" \
SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \
10 \
"default Helm render must keep clock skew below the manifest tolerance"
assert_env_value \
"$TMP_DIR/helm-image.yaml" \
SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS \
2 \
"default Helm render must stagger clock-skew recycling across replicas"
assert_contains \
"$TMP_DIR/helm-image.yaml" \
'^[[:space:]]+failureThreshold: 1$' \
"sandbox-runner readiness must fail immediately when clock skew is detected"

if [ "$(grep -Fc -- '- /usr/local/bin/sandbox-runner-healthcheck.sh' "$TMP_DIR/helm-image.yaml")" -lt 2 ]; then
echo "sandbox-runner liveness and readiness must both check guest clock skew" >&2
exit 1
fi

helm template codeapi "$TMP_DIR/chart" \
--set executionManifest.privateKey=test \
Expand Down Expand Up @@ -279,74 +243,6 @@ assert_contains \
'value: "0"' \
"fdLivenessLimit=0 must reach the healthcheck instead of reverting to 40000"

helm template codeapi "$TMP_DIR/chart" \
--set executionManifest.privateKey=test \
--set executionManifest.publicKey=test \
--set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=0 \
> "$TMP_DIR/helm-no-clock-skew-liveness.yaml"

assert_env_value \
"$TMP_DIR/helm-no-clock-skew-liveness.yaml" \
SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \
0 \
"clockSkewLivenessLimitSeconds=0 must reach the healthcheck instead of reverting to 10"

if helm template codeapi "$TMP_DIR/chart" \
--set executionManifest.privateKey=test \
--set executionManifest.publicKey=test \
--set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=25 \
> "$TMP_DIR/invalid-clock-skew-limit.yaml" 2> "$TMP_DIR/invalid-clock-skew-limit.log"; then
echo "clock-skew liveness limit must stay below manifest tolerance" >&2
exit 1
fi

assert_contains \
"$TMP_DIR/invalid-clock-skew-limit.log" \
'clockSkewLivenessLimitSeconds must be between 0 and 24' \
"clock-skew limit validation failed for an unexpected reason"

if helm template codeapi "$TMP_DIR/chart" \
--set executionManifest.privateKey=test \
--set executionManifest.publicKey=test \
--set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=29.5 \
> "$TMP_DIR/fractional-clock-skew-limit.yaml" 2> "$TMP_DIR/fractional-clock-skew-limit.log"; then
echo "clock-skew liveness limit must reject fractional values" >&2
exit 1
fi

assert_contains \
"$TMP_DIR/fractional-clock-skew-limit.log" \
'clockSkewLivenessLimitSeconds must be a non-negative integer' \
"fractional clock-skew limit validation failed for an unexpected reason"

if helm template codeapi "$TMP_DIR/chart" \
--set executionManifest.privateKey=test \
--set executionManifest.publicKey=test \
--set workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds=10 \
> "$TMP_DIR/invalid-clock-skew-jitter.yaml" 2> "$TMP_DIR/invalid-clock-skew-jitter.log"; then
echo "clock-skew liveness jitter must stay below the configured limit" >&2
exit 1
fi

assert_contains \
"$TMP_DIR/invalid-clock-skew-jitter.log" \
'clockSkewLivenessJitterSeconds must be non-negative and less than clockSkewLivenessLimitSeconds' \
"clock-skew jitter validation failed for an unexpected reason"

if helm template codeapi "$TMP_DIR/chart" \
--set executionManifest.privateKey=test \
--set executionManifest.publicKey=test \
--set workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds=1.5 \
> "$TMP_DIR/fractional-clock-skew-jitter.yaml" 2> "$TMP_DIR/fractional-clock-skew-jitter.log"; then
echo "clock-skew liveness jitter must reject fractional values" >&2
exit 1
fi

assert_contains \
"$TMP_DIR/fractional-clock-skew-jitter.log" \
'clockSkewLivenessJitterSeconds must be a non-negative integer' \
"fractional clock-skew jitter validation failed for an unexpected reason"

if helm template codeapi "$TMP_DIR/chart" \
--set executionManifest.privateKey=test \
--set executionManifest.publicKey=test \
Expand Down
Loading