diff --git a/helm/codeapi/README.md b/helm/codeapi/README.md index 9198b1a..3f1f1b7 100644 --- a/helm/codeapi/README.md +++ b/helm/codeapi/README.md @@ -58,16 +58,16 @@ verifier through environment variables on the api component, e.g.: ```yaml api: - extraEnv: - - name: CODEAPI_AUTH_PROVIDER - value: librechat-jwt - - name: CODEAPI_JWT_PUBLIC_KEY # single PEM/base64-DER verifier key - valueFrom: - secretKeyRef: - name: codeapi-jwt-verifier - key: public-key - - name: CODEAPI_JWT_KID - value: my-key-id + extraEnv: + - name: CODEAPI_AUTH_PROVIDER + value: librechat-jwt + - name: CODEAPI_JWT_PUBLIC_KEY # single PEM/base64-DER verifier key + valueFrom: + secretKeyRef: + name: codeapi-jwt-verifier + key: public-key + - name: CODEAPI_JWT_KID + value: my-key-id ``` `CODEAPI_JWT_PUBLIC_KEYS_DIR` (a mounted directory of PEM files) and @@ -78,6 +78,30 @@ For development only, `LOCAL_MODE=true` bypasses authentication — see **TLS to Redis.** Set `REDIS_TLS=true` via `extraEnv` on each component when your Redis (e.g. a managed cache) requires TLS. +**Redis Cluster mode (GCP Memorystore cluster, AWS ElastiCache cluster).** Set +`redis.enabled=false` (so the bundled Redis subchart doesn't take precedence) +and `redis.cluster.enabled=true`, and provide the comma-separated startup nodes +in `redis.cluster.nodes`. The chart will set `USE_REDIS_CLUSTER=true` and +populate `REDIS_HOST` with the node list on every component. For TLS with a CA +certificate (recommended for GCP Memorystore), also set: + +```yaml +redis: + enabled: false # required — otherwise the bundled subchart is used instead + cluster: + enabled: true + nodes: 'node-1:6379,node-2:6379,node-3:6379' + tls: + enabled: true + caSecretName: my-memorystore-secret # Secret that holds the CA cert + caKey: ca # Key inside the Secret + caMountPath: /etc/redis-tls/ca.crt # Mount path in each pod + useAlternativeDnsLookup: true # Required for GCP Memorystore cluster +``` + +The CA certificate is mounted from the Secret into every component pod and +passed to ioredis via `REDIS_CA`. No sidecar is needed. + **Package delivery.** KVM deployments default to `workerSandbox.packages.source=image`. Build and publish the baked runner target under the existing `workerSandbox.sandboxImage` repository and tag: @@ -105,11 +129,13 @@ override remains unchanged in either mode. ## Quick Start (Local Development) ### 1. Start Minikube + ```bash minikube start --cpus=4 --memory=8192 ``` ### 2. Build Images Inside Minikube + ```bash # Point docker to minikube's daemon eval $(minikube docker-env) @@ -124,6 +150,7 @@ docker build -t codeapi-package-init:latest -f docker/Dockerfile.package-init . ``` ### 3. Install Dependencies & Deploy + ```bash cd helm/codeapi @@ -179,6 +206,7 @@ kubectl rollout restart deployment/codeapi-sandbox-runner ``` ### 5. Access the API + ```bash # Port forward (in another terminal) kubectl port-forward svc/codeapi-api 3112:3112 @@ -192,6 +220,7 @@ curl http://localhost:3112/v1/health ## Commands Reference ### Startup + ```bash # Start minikube minikube start @@ -204,6 +233,7 @@ kubectl port-forward svc/codeapi-api 3112:3112 ``` ### Check Status + ```bash # View all pods kubectl get pods @@ -218,6 +248,7 @@ kubectl describe pod ``` ### Scaling + ```bash # Scale the sandbox execution tier kubectl scale deployment/codeapi-sandbox-runner --replicas=10 @@ -228,6 +259,7 @@ helm upgrade codeapi ./helm/codeapi -f ./helm/codeapi/values-local.yaml \ ``` ### Update After Code Changes + ```bash # Rebuild images (must be in minikube docker env) eval $(minikube docker-env) @@ -240,6 +272,7 @@ kubectl rollout restart deployment/codeapi-sandbox-runner ``` ### Teardown + ```bash # Uninstall the Helm release (removes all K8s resources) helm uninstall codeapi @@ -256,12 +289,14 @@ minikube delete ## Testing ### Health Check + ```bash curl http://localhost:3112/v1/health # Expected: OK ``` ### Execute Python Code + ```bash curl -X POST http://localhost:3112/v1/exec \ -H "Content-Type: application/json" \ @@ -270,6 +305,7 @@ curl -X POST http://localhost:3112/v1/exec \ ``` ### Verify Horizontal Scaling + ```bash # Check which service-worker processed the job kubectl logs deployment/codeapi-service-worker --tail=5 @@ -327,6 +363,7 @@ kubectl logs deployment/codeapi-service-worker --tail=5 ## Troubleshooting ### Pod stuck in `ErrImageNeverPull` + ```bash # Images must be built inside minikube's docker eval $(minikube docker-env) @@ -335,6 +372,7 @@ kubectl rollout restart deployment/ ``` ### Pod stuck in `CrashLoopBackOff` + ```bash # Check logs kubectl logs --previous @@ -342,6 +380,7 @@ kubectl describe pod ``` ### "runtime is unknown" error + ```bash # In source=pvc mode, check whether the package-init job populated the PVC: kubectl get jobs -l app.kubernetes.io/component=package-init @@ -359,12 +398,14 @@ In the default `source=image` mode, rebuild and publish package-init Job or packages PVC is rendered. ### Connection refused on port 3112 + ```bash # Make sure port-forward is running kubectl port-forward svc/codeapi-api 3112:3112 ``` ### MinIO `ImagePullBackOff` (production values) + ```bash # The Bitnami MinIO chart may reference unavailable image tags. # For local dev, values-local.yaml uses minio.useSimple=true which diff --git a/helm/codeapi/templates/_helpers.tpl b/helm/codeapi/templates/_helpers.tpl index ce81df7..e4687a4 100644 --- a/helm/codeapi/templates/_helpers.tpl +++ b/helm/codeapi/templates/_helpers.tpl @@ -152,11 +152,15 @@ app.kubernetes.io/component: tool-call-server {{- end }} {{/* -Redis host - either from subchart or external +Redis host - either from subchart or external. +In cluster mode this renders the comma-separated node list from redis.cluster.nodes +(falling back to redis.external.host for single-node external deployments). */}} {{- define "codeapi.redis.host" -}} {{- if .Values.redis.enabled }} {{- printf "%s-redis-master" .Release.Name }} +{{- else if and .Values.redis.cluster.enabled .Values.redis.cluster.nodes }} +{{- .Values.redis.cluster.nodes }} {{- else }} {{- .Values.redis.external.host }} {{- end }} @@ -168,11 +172,128 @@ Redis port {{- define "codeapi.redis.port" -}} {{- if .Values.redis.enabled }} {{- "6379" }} +{{- else if and .Values.redis.cluster.enabled .Values.redis.cluster.nodes }} +{{- $firstNode := .Values.redis.cluster.nodes | splitList "," | first }} +{{- $parts := $firstNode | splitList ":" }} +{{- if gt (len $parts) 1 }} +{{- index $parts 1 }} +{{- else }} +{{- "6379" }} +{{- end }} +{{- else }} +{{- .Values.redis.external.port | default "6379" }} +{{- end }} +{{- end }} + +{{/* +Redis host used ONLY for the wait-for-redis readiness probe. In cluster mode +codeapi.redis.host renders the full comma-separated startup-node list +("n1:6379,n2:6379,..."), which `nc -z` cannot resolve as a single +[destination] [port] pair. This always resolves to one reachable host — +the first cluster startup node when in cluster mode, otherwise the same +value as codeapi.redis.host. +*/}} +{{- define "codeapi.redis.probeHost" -}} +{{- if .Values.redis.enabled }} +{{- printf "%s-redis-master" .Release.Name }} +{{- else if and .Values.redis.cluster.enabled .Values.redis.cluster.nodes }} +{{- $firstNode := .Values.redis.cluster.nodes | splitList "," | first }} +{{- $firstNode | splitList ":" | first }} +{{- else }} +{{- .Values.redis.external.host }} +{{- end }} +{{- end }} + +{{/* +Port companion to codeapi.redis.probeHost. +*/}} +{{- define "codeapi.redis.probePort" -}} +{{- if .Values.redis.enabled }} +{{- "6379" }} +{{- else if and .Values.redis.cluster.enabled .Values.redis.cluster.nodes }} +{{- $firstNode := .Values.redis.cluster.nodes | splitList "," | first }} +{{- $parts := $firstNode | splitList ":" }} +{{- if gt (len $parts) 1 }} +{{- index $parts 1 }} +{{- else }} +{{- "6379" }} +{{- end }} {{- else }} {{- .Values.redis.external.port | default "6379" }} {{- end }} {{- end }} +{{/* +USE_REDIS_CLUSTER value – "true" when redis.cluster.enabled or when +redis.cluster.nodes contains a comma (auto-detect multiple nodes). +*/}} +{{- define "codeapi.redis.clusterEnabled" -}} +{{- if .Values.redis.cluster.enabled }} +{{- "true" }} +{{- else if and .Values.redis.cluster.nodes (contains "," .Values.redis.cluster.nodes) }} +{{- "true" }} +{{- else }} +{{- "false" }} +{{- end }} +{{- end }} + +{{/* +Emit the Redis TLS + CA environment variables and volume mount for each +component that needs it. These only apply to an external managed Redis, so +they render nothing while the bundled subchart is enabled (redis.enabled=true +always uses plain TCP to the in-cluster Bitnami Redis) or when redis.tls.enabled +is false. +Usage: {{ include "codeapi.redis.tlsEnv" . }} +*/}} +{{- define "codeapi.redis.tlsEnv" -}} +{{- if not .Values.redis.enabled }} +{{- if .Values.redis.tls.enabled }} +- name: REDIS_TLS + value: "true" +{{- if .Values.redis.tls.caSecretName }} +- name: REDIS_CA + value: {{ .Values.redis.tls.caMountPath | quote }} +{{- end }} +{{- end }} +{{- if .Values.redis.useAlternativeDnsLookup }} +- name: REDIS_USE_ALTERNATIVE_DNS_LOOKUP + value: "true" +{{- end }} +{{- end }} +{{- end }} + +{{/* +Volume definition for the Redis CA certificate secret. +Renders nothing when redis.tls.caSecretName is empty, or the bundled Redis +subchart is enabled (it never speaks TLS). +Usage: {{ include "codeapi.redis.caVolume" . }} +*/}} +{{- define "codeapi.redis.caVolume" -}} +{{- if and (not .Values.redis.enabled) .Values.redis.tls.enabled .Values.redis.tls.caSecretName }} +- name: redis-ca + secret: + secretName: {{ .Values.redis.tls.caSecretName }} + items: + - key: {{ .Values.redis.tls.caKey }} + path: ca.crt +{{- end }} +{{- end }} + +{{/* +VolumeMount for the Redis CA certificate inside a container. +Renders nothing when redis.tls.caSecretName is empty, or the bundled Redis +subchart is enabled (it never speaks TLS). +Usage: {{ include "codeapi.redis.caVolumeMount" . }} +*/}} +{{- define "codeapi.redis.caVolumeMount" -}} +{{- if and (not .Values.redis.enabled) .Values.redis.tls.enabled .Values.redis.tls.caSecretName }} +- name: redis-ca + mountPath: {{ .Values.redis.tls.caMountPath | quote }} + subPath: ca.crt + readOnly: true +{{- end }} +{{- end }} + {{/* Redis NetworkPolicy egress. Kubernetes NetworkPolicy cannot match DNS names, so external Redis can be scoped with CIDRs when available; otherwise the chart diff --git a/helm/codeapi/templates/api-deployment.yaml b/helm/codeapi/templates/api-deployment.yaml index 0c801ab..2de5257 100644 --- a/helm/codeapi/templates/api-deployment.yaml +++ b/helm/codeapi/templates/api-deployment.yaml @@ -51,7 +51,9 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password - # Service URLs + - name: USE_REDIS_CLUSTER + value: {{ include "codeapi.redis.clusterEnabled" . | quote }} + {{- include "codeapi.redis.tlsEnv" . | nindent 12 }} - name: FILE_SERVER_URL value: "http://{{ include "codeapi.fullname" . }}-file-server:{{ .Values.fileServer.service.port }}" - name: TOOL_CALL_SERVER_URL @@ -87,6 +89,14 @@ spec: periodSeconds: 10 resources: {{- toYaml .Values.api.resources | nindent 12 }} + {{- with (include "codeapi.redis.caVolumeMount" . | trim) }} + volumeMounts: + {{- . | nindent 12 }} + {{- end }} + {{- with (include "codeapi.redis.caVolume" . | trim) }} + volumes: + {{- . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/codeapi/templates/egress-gateway-deployment.yaml b/helm/codeapi/templates/egress-gateway-deployment.yaml index 6e39490..96a5e37 100644 --- a/helm/codeapi/templates/egress-gateway-deployment.yaml +++ b/helm/codeapi/templates/egress-gateway-deployment.yaml @@ -69,6 +69,9 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password + - name: USE_REDIS_CLUSTER + value: {{ include "codeapi.redis.clusterEnabled" . | quote }} + {{- include "codeapi.redis.tlsEnv" . | nindent 12 }} livenessProbe: httpGet: path: /live @@ -83,6 +86,14 @@ spec: periodSeconds: 10 resources: {{- toYaml .Values.egressGateway.resources | nindent 12 }} + {{- with (include "codeapi.redis.caVolumeMount" . | trim) }} + volumeMounts: + {{- . | nindent 12 }} + {{- end }} + {{- with (include "codeapi.redis.caVolume" . | trim) }} + volumes: + {{- . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/codeapi/templates/file-server-deployment.yaml b/helm/codeapi/templates/file-server-deployment.yaml index 1dd9516..e542cd0 100644 --- a/helm/codeapi/templates/file-server-deployment.yaml +++ b/helm/codeapi/templates/file-server-deployment.yaml @@ -110,6 +110,9 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password + - name: USE_REDIS_CLUSTER + value: {{ include "codeapi.redis.clusterEnabled" . | quote }} + {{- include "codeapi.redis.tlsEnv" . | nindent 12 }} - name: CODEAPI_INTERNAL_SERVICE_TOKEN valueFrom: secretKeyRef: @@ -135,6 +138,14 @@ spec: timeoutSeconds: 5 resources: {{- toYaml .Values.fileServer.resources | nindent 12 }} + {{- with (include "codeapi.redis.caVolumeMount" . | trim) }} + volumeMounts: + {{- . | nindent 12 }} + {{- end }} + {{- with (include "codeapi.redis.caVolume" . | trim) }} + volumes: + {{- . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/codeapi/templates/secrets.yaml b/helm/codeapi/templates/secrets.yaml index b23085e..ea661cf 100644 --- a/helm/codeapi/templates/secrets.yaml +++ b/helm/codeapi/templates/secrets.yaml @@ -29,7 +29,7 @@ stringData: {{- if .Values.redis.enabled }} redis-password: {{ .Values.redis.auth.password | quote }} {{- else }} - redis-password: {{ .Values.redis.external.password | quote }} + redis-password: {{ dig "external" "password" "" .Values.redis | quote }} {{- end }} # MinIO/S3 credentials diff --git a/helm/codeapi/templates/tool-call-server-deployment.yaml b/helm/codeapi/templates/tool-call-server-deployment.yaml index b5596c9..f648712 100644 --- a/helm/codeapi/templates/tool-call-server-deployment.yaml +++ b/helm/codeapi/templates/tool-call-server-deployment.yaml @@ -44,6 +44,9 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password + - name: USE_REDIS_CLUSTER + value: {{ include "codeapi.redis.clusterEnabled" . | quote }} + {{- include "codeapi.redis.tlsEnv" . | nindent 12 }} - name: CODEAPI_INTERNAL_SERVICE_TOKEN valueFrom: secretKeyRef: @@ -70,6 +73,14 @@ spec: periodSeconds: 10 resources: {{- toYaml .Values.toolCallServer.resources | nindent 12 }} + {{- with (include "codeapi.redis.caVolumeMount" . | trim) }} + volumeMounts: + {{- . | nindent 12 }} + {{- end }} + {{- with (include "codeapi.redis.caVolume" . | trim) }} + volumes: + {{- . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/codeapi/templates/worker-sandbox-deployment.yaml b/helm/codeapi/templates/worker-sandbox-deployment.yaml index 8a82a61..b98e06c 100644 --- a/helm/codeapi/templates/worker-sandbox-deployment.yaml +++ b/helm/codeapi/templates/worker-sandbox-deployment.yaml @@ -115,7 +115,7 @@ spec: initContainers: - name: wait-for-redis image: busybox:1.36.1 - command: ['sh', '-c', 'until nc -z {{ include "codeapi.redis.host" . }} {{ include "codeapi.redis.port" . }}; do echo waiting for redis; sleep 2; done'] + command: ['sh', '-c', 'until nc -z {{ include "codeapi.redis.probeHost" . }} {{ include "codeapi.redis.probePort" . }}; do echo waiting for redis; sleep 2; done'] - name: wait-for-sandbox-runner image: busybox:1.36.1 command: ['sh', '-c', 'until nc -z {{ include "codeapi.fullname" . }}-sandbox-runner {{ .Values.workerSandbox.sandbox.port }}; do echo waiting for sandbox-runner; sleep 2; done'] @@ -144,6 +144,9 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password + - name: USE_REDIS_CLUSTER + value: {{ include "codeapi.redis.clusterEnabled" . | quote }} + {{- include "codeapi.redis.tlsEnv" . | nindent 12 }} - name: CODEAPI_INTERNAL_SERVICE_TOKEN valueFrom: secretKeyRef: @@ -171,6 +174,10 @@ spec: {{- with .Values.workerSandbox.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} + {{- with (include "codeapi.redis.caVolumeMount" . | trim) }} + volumeMounts: + {{- . | nindent 12 }} + {{- end }} livenessProbe: httpGet: path: /health @@ -191,6 +198,10 @@ spec: limits: {{- toYaml . | nindent 14 }} {{- end }} + {{- with (include "codeapi.redis.caVolume" . | trim) }} + volumes: + {{- . | nindent 8 }} + {{- end }} {{- with $serviceWorkerNodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index c926e76..f1f19fd 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -9,37 +9,37 @@ # GLOBAL SETTINGS # ============================================================================= global: - # Image pull secrets (if using private registry) - imagePullSecrets: [] - # Storage class for persistent volumes - storageClass: "" + # Image pull secrets (if using private registry) + imagePullSecrets: [] + # Storage class for persistent volumes + storageClass: '' # Shared token for internal service-to-service requests between the public API, # file-server, tool-call-server, worker, and sandbox API. Override for every # production install. internalServiceAuth: - token: "changeme-in-production" + token: 'changeme-in-production' hardenedSandboxMode: true otel: - enabled: false - # OTLP/HTTP collector endpoint, e.g. "http://opentelemetry-collector.observability:4318". - # Required when enabled=true. - exporterOtlpEndpoint: "" - exporterOtlpTracesEndpoint: "" - # Comma-separated OTLP headers. Percent-encode commas and equals signs in - # values, matching the OTLP env-var convention. - exporterOtlpHeaders: "" - resourceAttributes: "" + enabled: false + # OTLP/HTTP collector endpoint, e.g. "http://opentelemetry-collector.observability:4318". + # Required when enabled=true. + exporterOtlpEndpoint: '' + exporterOtlpTracesEndpoint: '' + # Comma-separated OTLP headers. Percent-encode commas and equals signs in + # values, matching the OTLP env-var convention. + exporterOtlpHeaders: '' + resourceAttributes: '' # Encrypted grants/handles owned by the egress gateway. Sandbox-runner, # service-api, and service-worker never receive this secret. egressGrant: - secret: "changeme-egress-grant-secret-32-bytes-minimum" - ttlSeconds: 900 - ledgerRequired: true - ledgerTtlGraceSeconds: 300 + secret: 'changeme-egress-grant-secret-32-bytes-minimum' + ttlSeconds: 900 + ledgerRequired: true + ledgerTtlGraceSeconds: 300 # Worker signs sandbox execute requests with this private key; sandbox-runner # receives only the public verifier so a runner compromise cannot mint new @@ -51,59 +51,59 @@ egressGrant: # workerSandbox.enabled=true and these are unset. For minikube local dev, # values-local.yaml ships a test-only keypair. executionManifest: - privateKey: "" - publicKey: "" - ttlSeconds: 300 + privateKey: '' + publicKey: '' + ttlSeconds: 300 # ============================================================================= # API SERVICE (HTTP handlers, scales based on traffic) # ============================================================================= api: - enabled: true - replicaCount: 2 # Start with 2 API pods - - image: - repository: codeapi-api - tag: latest - pullPolicy: IfNotPresent # Use Always in production - - # Resource limits - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - - # Service configuration - service: - type: ClusterIP - port: 3112 - - httpJsonLimit: 50mb - - # Ingress (optional, for external access) - ingress: - enabled: false - className: "" - annotations: {} - hosts: - - host: codeapi.local - paths: - - path: / - pathType: Prefix - tls: [] - - # Horizontal Pod Autoscaler - autoscaling: - enabled: false - minReplicas: 2 - maxReplicas: 10 - targetCPUUtilizationPercentage: 70 + enabled: true + replicaCount: 2 # Start with 2 API pods - # Extra environment variables - extraEnv: [] + image: + repository: codeapi-api + tag: latest + pullPolicy: IfNotPresent # Use Always in production + + # Resource limits + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + + # Service configuration + service: + type: ClusterIP + port: 3112 + + httpJsonLimit: 50mb + + # Ingress (optional, for external access) + ingress: + enabled: false + className: '' + annotations: {} + hosts: + - host: codeapi.local + paths: + - path: / + pathType: Prefix + tls: [] + + # Horizontal Pod Autoscaler + autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + + # Extra environment variables + extraEnv: [] # ============================================================================= # WORKER-SANDBOX (Job processors + MicroVM Sandbox, scales based on queue depth) @@ -111,341 +111,367 @@ api: # Only requires /dev/kvm — no privileged mode, no Linux capabilities. # ============================================================================= workerSandbox: - enabled: true - replicaCount: 3 - - workerImage: - repository: codeapi-worker - tag: latest - pullPolicy: IfNotPresent - - sandboxImage: - # With packages.source=image, build the api/Dockerfile default (or - # sandbox-runner-baked). With source=pvc/direct mode, build - # --target sandbox-runner under the same configured repository/tag. - repository: codeapi-sandbox-runner - tag: latest - pullPolicy: IfNotPresent - - # Enable libkrun microVM isolation. Requires /dev/kvm on the node. - # Set to false to fall back to chroot + NsJail (no guest kernel). - # Override to false until the node group has nested virtualisation enabled. - kvmEnabled: true - - # KVM device group ID for legacy hostPath mode. - kvmGid: 108 - - # Custom seccomp profile (defense-in-depth on top of NsJail's inner Kafel - # policy). When enabled, pods reference Localhost: profiles/nsjail.json, - # which must be pre-installed on every node at - # /var/lib/kubelet/seccomp/profiles/nsjail.json - see seccomp/README.md - # for the DaemonSet installer pattern. When disabled, KVM-mode pods use - # RuntimeDefault (still safer than no profile) and non-KVM-mode pods use - # Unconfined (forced by NsJail's pivot_root requirement). - seccomp: - enabled: false + enabled: true + replicaCount: 3 - # Optional Kubernetes device-plugin path for non-privileged KVM access. The - # device plugin should be deployed separately; this chart only requests its - # extended resource for sandbox-runner. - kvmDevicePlugin: - enabled: false - # Extended resource advertised by your KVM device plugin, e.g. - # "devices.kubevirt.io/kvm". Required when enabled=true (install fails - # fast when unset). - resourceName: "" - - # MicroVM launcher configuration (only used when kvmEnabled: true) - launcher: - vcpus: 2 - ramMib: 2048 - logLevel: 3 - filterVsockEnotconn: true - - # Resource limits (higher because it runs code + microVM) - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 2000m - memory: 3Gi - - serviceAccount: - create: true - name: "" - annotations: {} - - sandboxServiceAccount: - create: true - name: "" - automountServiceAccountToken: false - annotations: {} - - # Optional per-deployment scheduling. Empty values fall back to the chart-wide - # nodeSelector/tolerations/affinity below. Use sandboxRunner to place only the - # untrusted-code tier on KVM-capable nodes. - serviceWorker: - # Defaults to workerSandbox.replicaCount when unset. - # Keep this as fixed HA capacity; scale sandboxRunner for execution throughput. - replicaCount: null - inheritSharedScheduling: true - nodeSelector: {} - tolerations: [] - affinity: {} - - sandboxRunner: - # Defaults to workerSandbox.replicaCount when unset. - replicaCount: null - # Restart a sandbox-runner before launcher/VMM host-side fd retention can - # exhaust the process file descriptor limit. Set 0 to disable. - fdLivenessLimit: 40000 - inheritSharedScheduling: true - nodeSelector: {} - tolerations: [] - affinity: {} - autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 20 - targetCPUUtilizationPercentage: 70 - targetMemoryUtilizationPercentage: 80 - - # Worker configuration - config: - # Throughput-mode defaults for launcher.vcpus=2. Bash/Node/Bun and - # bash-launched Python run through the "other" queue; direct Python is kept - # as a small compatibility lane. - pythonConcurrency: 1 - otherConcurrency: 8 - jobWindow: 1000 - jobTimeout: 25000 # 25 seconds - - # Health check port for the worker - healthPort: 3113 - - # Sandbox configuration (NsJail-based, uses SANDBOX_* env vars) - sandbox: - port: 2000 - maxProcessCount: 100 - runCpuTime: 15000 - runTimeout: 15000 - compileCpuTime: 10000 - compileTimeout: 10000 - outputMaxSize: 65536 - maxOpenFiles: 2048 - executeBodyLimit: 50mb - disableNetworking: true - maxConcurrentJobs: 8 - perJobUids: true - jobUidBase: 200000 - jobGidBase: 200000 - # Defaults to maxConcurrentJobs when unset. - jobUidCount: null - workspaceReaperMaxAgeSeconds: 3600 - - # Language runtime package delivery - packages: - # image (default): boot a baked /pkgs tree from a read-only ext4 block root. - # This avoids long-lived virtio-fs inode handles exhausting the launcher FD - # limit. Requires kvmEnabled=true and sandboxImage built from the - # sandbox-runner-baked target (the api/Dockerfile default). - # - # pvc: legacy compatibility mode. Mount a host/PVC package tree through - # virtio-fs and recycle runners with the FD liveness guard before exhaustion. - source: image - - # Used only when source=pvc. - persistence: - enabled: true - size: 10Gi - accessMode: ReadWriteOnce - storageClass: "" - # If you have pre-built packages, specify the existing claim - existingClaim: "" - - # Used only when source=pvc. Populates packages on first install. - # Runs as a Helm pre-install/pre-upgrade hook - # Compiles Python from source and downloads Node/Bun binaries - initJob: - enabled: true - forceRebuild: false - image: - repository: codeapi-package-init + workerImage: + repository: codeapi-worker tag: latest pullPolicy: IfNotPresent - pythonVersion: "3.14.4" - nodeVersion: "24.15.0" - bunVersion: "1.3.14" - bashPackageVersion: "5.2.0" - backoffLimit: 2 - ttlSecondsAfterFinished: 3600 - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 2000m - memory: 4Gi - # Extra environment variables for worker containers - extraEnv: [] + sandboxImage: + # With packages.source=image, build the api/Dockerfile default (or + # sandbox-runner-baked). With source=pvc/direct mode, build + # --target sandbox-runner under the same configured repository/tag. + repository: codeapi-sandbox-runner + tag: latest + pullPolicy: IfNotPresent - # Extra environment variables for the sandbox-runner container only. Keep - # control-plane, storage, Redis, and gateway encryption secrets out of this - # list; use it only for sandbox verifier/runtime knobs such as - # SANDBOX_LOG_LEVEL. Secret/token-like names are rejected by template - # validation; the manifest public verifier is chart-managed. - sandboxExtraEnv: [] + # Enable libkrun microVM isolation. Requires /dev/kvm on the node. + # Set to false to fall back to chroot + NsJail (no guest kernel). + # Override to false until the node group has nested virtualisation enabled. + kvmEnabled: true + + # KVM device group ID for legacy hostPath mode. + kvmGid: 108 + + # Custom seccomp profile (defense-in-depth on top of NsJail's inner Kafel + # policy). When enabled, pods reference Localhost: profiles/nsjail.json, + # which must be pre-installed on every node at + # /var/lib/kubelet/seccomp/profiles/nsjail.json - see seccomp/README.md + # for the DaemonSet installer pattern. When disabled, KVM-mode pods use + # RuntimeDefault (still safer than no profile) and non-KVM-mode pods use + # Unconfined (forced by NsJail's pivot_root requirement). + seccomp: + enabled: false + + # Optional Kubernetes device-plugin path for non-privileged KVM access. The + # device plugin should be deployed separately; this chart only requests its + # extended resource for sandbox-runner. + kvmDevicePlugin: + enabled: false + # Extended resource advertised by your KVM device plugin, e.g. + # "devices.kubevirt.io/kvm". Required when enabled=true (install fails + # fast when unset). + resourceName: '' + + # MicroVM launcher configuration (only used when kvmEnabled: true) + launcher: + vcpus: 2 + ramMib: 2048 + logLevel: 3 + filterVsockEnotconn: true + + # Resource limits (higher because it runs code + microVM) + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2000m + memory: 3Gi + + serviceAccount: + create: true + name: '' + annotations: {} + + sandboxServiceAccount: + create: true + name: '' + automountServiceAccountToken: false + annotations: {} + + # Optional per-deployment scheduling. Empty values fall back to the chart-wide + # nodeSelector/tolerations/affinity below. Use sandboxRunner to place only the + # untrusted-code tier on KVM-capable nodes. + serviceWorker: + # Defaults to workerSandbox.replicaCount when unset. + # Keep this as fixed HA capacity; scale sandboxRunner for execution throughput. + replicaCount: null + inheritSharedScheduling: true + nodeSelector: {} + tolerations: [] + affinity: {} + + sandboxRunner: + # Defaults to workerSandbox.replicaCount when unset. + replicaCount: null + # Restart a sandbox-runner before launcher/VMM host-side fd retention can + # exhaust the process file descriptor limit. Set 0 to disable. + fdLivenessLimit: 40000 + inheritSharedScheduling: true + nodeSelector: {} + tolerations: [] + affinity: {} + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 20 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + + # Worker configuration + config: + # Throughput-mode defaults for launcher.vcpus=2. Bash/Node/Bun and + # bash-launched Python run through the "other" queue; direct Python is kept + # as a small compatibility lane. + pythonConcurrency: 1 + otherConcurrency: 8 + jobWindow: 1000 + jobTimeout: 25000 # 25 seconds + + # Health check port for the worker + healthPort: 3113 + + # Sandbox configuration (NsJail-based, uses SANDBOX_* env vars) + sandbox: + port: 2000 + maxProcessCount: 100 + runCpuTime: 15000 + runTimeout: 15000 + compileCpuTime: 10000 + compileTimeout: 10000 + outputMaxSize: 65536 + maxOpenFiles: 2048 + executeBodyLimit: 50mb + disableNetworking: true + maxConcurrentJobs: 8 + perJobUids: true + jobUidBase: 200000 + jobGidBase: 200000 + # Defaults to maxConcurrentJobs when unset. + jobUidCount: null + workspaceReaperMaxAgeSeconds: 3600 + + # Language runtime package delivery + packages: + # image (default): boot a baked /pkgs tree from a read-only ext4 block root. + # This avoids long-lived virtio-fs inode handles exhausting the launcher FD + # limit. Requires kvmEnabled=true and sandboxImage built from the + # sandbox-runner-baked target (the api/Dockerfile default). + # + # pvc: legacy compatibility mode. Mount a host/PVC package tree through + # virtio-fs and recycle runners with the FD liveness guard before exhaustion. + source: image + + # Used only when source=pvc. + persistence: + enabled: true + size: 10Gi + accessMode: ReadWriteOnce + storageClass: '' + # If you have pre-built packages, specify the existing claim + existingClaim: '' + + # Used only when source=pvc. Populates packages on first install. + # Runs as a Helm pre-install/pre-upgrade hook + # Compiles Python from source and downloads Node/Bun binaries + initJob: + enabled: true + forceRebuild: false + image: + repository: codeapi-package-init + tag: latest + pullPolicy: IfNotPresent + pythonVersion: '3.14.4' + nodeVersion: '24.15.0' + bunVersion: '1.3.14' + bashPackageVersion: '5.2.0' + backoffLimit: 2 + ttlSecondsAfterFinished: 3600 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2000m + memory: 4Gi + + # Extra environment variables for worker containers + extraEnv: [] + + # Extra environment variables for the sandbox-runner container only. Keep + # control-plane, storage, Redis, and gateway encryption secrets out of this + # list; use it only for sandbox verifier/runtime knobs such as + # SANDBOX_LOG_LEVEL. Secret/token-like names are rejected by template + # validation; the manifest public verifier is chart-managed. + sandboxExtraEnv: [] # ============================================================================= # FILE SERVER (S3/MinIO integration, stateless) # ============================================================================= fileServer: - enabled: true - replicaCount: 1 - - image: - repository: codeapi-file-server - tag: latest - pullPolicy: IfNotPresent - - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - - service: - type: ClusterIP - port: 3000 - - # ServiceAccount for IRSA (IAM Roles for Service Accounts) on EKS - # See: documentation/services/codeapi/deployment/irsa.md - serviceAccount: - create: true - name: "" - annotations: {} - # For IRSA on EKS, add: - # annotations: - # eks.amazonaws.com/role-arn: "arn:aws:iam::ACCOUNT_ID:role/codeapi-fileserver-role" - - # S3 configuration (when using AWS S3 instead of MinIO) - # Set minio.enabled=false when using external S3 - s3: - enabled: false - endpoint: "s3.amazonaws.com" - region: "us-east-1" - bucket: "" - useSSL: true - # Use IRSA for authentication (no static credentials needed) - useIrsa: false + enabled: true + replicaCount: 1 + + image: + repository: codeapi-file-server + tag: latest + pullPolicy: IfNotPresent + + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + + service: + type: ClusterIP + port: 3000 + + # ServiceAccount for IRSA (IAM Roles for Service Accounts) on EKS + # See: documentation/services/codeapi/deployment/irsa.md + serviceAccount: + create: true + name: '' + annotations: {} + # For IRSA on EKS, add: + # annotations: + # eks.amazonaws.com/role-arn: "arn:aws:iam::ACCOUNT_ID:role/codeapi-fileserver-role" + + # S3 configuration (when using AWS S3 instead of MinIO) + # Set minio.enabled=false when using external S3 + s3: + enabled: false + endpoint: 's3.amazonaws.com' + region: 'us-east-1' + bucket: '' + useSSL: true + # Use IRSA for authentication (no static credentials needed) + useIrsa: false # ============================================================================= # TOOL CALL SERVER (programmatic tool calling, stateless) # ============================================================================= toolCallServer: - enabled: true - replicaCount: 1 - - image: - repository: codeapi-tool-call-server - tag: latest - pullPolicy: IfNotPresent - - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - - service: - type: ClusterIP - port: 3033 - - config: - requestTimeout: 300000 - sessionExpiry: 600 + enabled: true + replicaCount: 1 + + image: + repository: codeapi-tool-call-server + tag: latest + pullPolicy: IfNotPresent + + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + + service: + type: ClusterIP + port: 3033 + + config: + requestTimeout: 300000 + sessionExpiry: 600 # ============================================================================= # EGRESS GATEWAY (only sandbox-runner outbound capability endpoint) # ============================================================================= egressGateway: - enabled: true - replicaCount: 1 + enabled: true + replicaCount: 1 - image: - repository: codeapi-egress-gateway - tag: latest - pullPolicy: IfNotPresent + image: + repository: codeapi-egress-gateway + tag: latest + pullPolicy: IfNotPresent - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi - service: - type: ClusterIP - port: 3190 + service: + type: ClusterIP + port: 3190 - config: - maxToolCallBytes: 1048576 + config: + maxToolCallBytes: 1048576 # ============================================================================= # REDIS (Job queue and state store) # ============================================================================= redis: - enabled: true # Set to false if using external Redis - - # Bitnami Redis chart configuration - architecture: standalone # Use 'replication' for HA - auth: - enabled: true - password: "changeme-in-production" - - master: - persistence: - enabled: true - size: 1Gi - - # If using external Redis, configure here: - # external: - # host: redis.example.com - # port: 6379 - # password: secret + enabled: true # Set to false if using external Redis + + # Bitnami Redis chart configuration + architecture: standalone # Use 'replication' for HA + auth: + enabled: true + password: 'changeme-in-production' + + master: + persistence: + enabled: true + size: 1Gi + + # If using external Redis, configure here: + # external: + # host: redis.example.com + # port: 6379 + # password: secret + + # Redis Cluster mode (e.g. GCP Memorystore cluster, AWS ElastiCache cluster). + # When enabled the service components connect using ioredis Cluster mode. + cluster: + enabled: false + # Comma-separated list of "host:port" startup nodes for ioredis Cluster + # discovery. Populate this when redis.enabled=false and the external Redis + # is a cluster endpoint. Example: + # nodes: "node1.example.com:6379,node2.example.com:6379,node3.example.com:6379" + nodes: '' + + # TLS settings for external managed Redis (e.g. GCP Memorystore with TLS). + # Ignored when redis.enabled=true (in-cluster Redis uses plain TCP). + tls: + enabled: false + # Name of a Kubernetes Secret that holds the CA certificate. + # The Secret must contain the key specified in caKey. + caSecretName: '' + caKey: 'ca' + # Mount path inside each pod for the CA certificate file. + caMountPath: '/etc/redis-tls/ca.crt' + + # Alternative DNS lookup – set to true for GCP Memorystore cluster or + # AWS ElastiCache clusters where standard DNS resolution causes TLS + # hostname-mismatch errors. Mirrors REDIS_USE_ALTERNATIVE_DNS_LOOKUP. + useAlternativeDnsLookup: false # ============================================================================= # MINIO (S3-compatible storage) # ============================================================================= minio: - enabled: true # Set to false if using external S3/MinIO + enabled: true # Set to false if using external S3/MinIO - # Bitnami MinIO chart configuration - mode: standalone # Use 'distributed' for HA - auth: - rootUser: minioadmin - rootPassword: "changeme-in-production" + # Bitnami MinIO chart configuration + mode: standalone # Use 'distributed' for HA + auth: + rootUser: minioadmin + rootPassword: 'changeme-in-production' - persistence: - enabled: true - size: 10Gi + persistence: + enabled: true + size: 10Gi - defaultBuckets: "codeapi-files" + defaultBuckets: 'codeapi-files' - # If using external MinIO/S3: - # external: - # endpoint: s3.amazonaws.com - # accessKey: AKIAXXXXXXXX - # secretKey: xxxxxxxx - # bucket: my-bucket - # useSSL: true + # If using external MinIO/S3: + # external: + # endpoint: s3.amazonaws.com + # accessKey: AKIAXXXXXXXX + # secretKey: xxxxxxxx + # bucket: my-bucket + # useSSL: true # ============================================================================= # COMMON SETTINGS @@ -462,40 +488,40 @@ affinity: {} # Pod disruption budgets podDisruptionBudget: - api: - enabled: false - minAvailable: 1 - workerSandbox: - enabled: false - minAvailable: 2 + api: + enabled: false + minAvailable: 1 + workerSandbox: + enabled: false + minAvailable: 2 # ============================================================================= # PROMETHEUS METRICS # ============================================================================= metrics: - enabled: false # Set to true to create PodMonitors for Prometheus scraping - interval: "30s" - scrapeTimeout: "10s" + enabled: false # Set to true to create PodMonitors for Prometheus scraping + interval: '30s' + scrapeTimeout: '10s' # ============================================================================= # NETWORK POLICIES # ============================================================================= networkPolicy: - enabled: true - otel: enabled: true - port: 4318 - # Match these to your collector's namespace and pod labels. - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: observability - podSelector: - matchLabels: - app.kubernetes.io/name: opentelemetry-collector - redis: - # Used only when redis.enabled=false. If empty, NetworkPolicy permits TCP - # egress on the configured Redis port to any destination because standard - # Kubernetes NetworkPolicy cannot target redis.external.host by DNS name. - externalCIDRs: [] - toolCallServer: - allowExternalEgress: false # Allow tool call server to reach external APIs + otel: + enabled: true + port: 4318 + # Match these to your collector's namespace and pod labels. + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: opentelemetry-collector + redis: + # Used only when redis.enabled=false. If empty, NetworkPolicy permits TCP + # egress on the configured Redis port to any destination because standard + # Kubernetes NetworkPolicy cannot target redis.external.host by DNS name. + externalCIDRs: [] + toolCallServer: + allowExternalEgress: false # Allow tool call server to reach external APIs diff --git a/service/.env.example b/service/.env.example index e06ec4c..6095ca2 100644 --- a/service/.env.example +++ b/service/.env.example @@ -8,6 +8,19 @@ FILE_SERVER_PORT=3000 REDIS_HOST=localhost REDIS_PORT=6379 REDIS_PASSWORD=mysecretpassword +# Redis Cluster mode (GCP Memorystore cluster, AWS ElastiCache cluster, etc.) +# Set USE_REDIS_CLUSTER=true to enable, or provide a comma-separated list of +# host:port startup nodes in REDIS_HOST (cluster mode is then auto-detected). +# Example cluster REDIS_HOST: node1.example.com:6379,node2.example.com:6379 +# USE_REDIS_CLUSTER=false +# TLS: set REDIS_TLS=true for managed Redis with TLS (e.g. GCP Memorystore). +# REDIS_TLS=false +# CA certificate path for TLS with certificate validation (recommended over +# REDIS_TLS alone which disables cert verification). +# REDIS_CA=/path/to/redis-ca.crt +# Alternative DNS lookup required for AWS ElastiCache clusters with TLS and +# for some GCP Memorystore cluster configurations. +# REDIS_USE_ALTERNATIVE_DNS_LOOKUP=false # ----------------------------------------------------------------------------- # Stripe: https://shipfa.st/docs/features/payments diff --git a/service/src/config.ts b/service/src/config.ts index c9dcb68..b20ba42 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -4,49 +4,76 @@ import { nanoid } from 'nanoid'; import type * as t from './types'; import { Languages } from './enum'; -export const languageConfig: Record = { - [Languages.bash]: { language: 'bash', version: '5.2.0', fileName: 'script.sh' }, - [Languages.js]: { language: 'bun-js', version: '1.3.14', fileName: 'index.js' }, - [Languages.node]: { language: 'node', version: '24.15.0', fileName: 'index.js' }, - [Languages.py]: { language: 'python', version: '3.14.4', fileName: 'main.py' }, - [Languages.ts]: { language: 'bun-ts', version: '1.3.14', fileName: 'main.ts' }, +export const languageConfig: Record< + Languages | string, + t.LanguageConfig | undefined +> = { + [Languages.bash]: { + language: 'bash', + version: '5.2.0', + fileName: 'script.sh', + }, + [Languages.js]: { + language: 'bun-js', + version: '1.3.14', + fileName: 'index.js', + }, + [Languages.node]: { + language: 'node', + version: '24.15.0', + fileName: 'index.js', + }, + [Languages.py]: { + language: 'python', + version: '3.14.4', + fileName: 'main.py', + }, + [Languages.ts]: { + language: 'bun-ts', + version: '1.3.14', + fileName: 'main.ts', + }, }; const languageAliases: Record = { - // Python - python: Languages.py, - py: Languages.py, - - // JavaScript (Bun) - javascript: Languages.js, - js: Languages.js, - 'bun-js': Languages.js, - bun: Languages.js, - - // JavaScript (Node.js) - node: Languages.node, - nodejs: Languages.node, - 'node-js': Languages.node, - 'node-javascript': Languages.node, - - // TypeScript (Bun) - typescript: Languages.ts, - ts: Languages.ts, - 'bun-ts': Languages.ts, - 'bun-typescript': Languages.ts, - - // Bash - bash: Languages.bash, - sh: Languages.bash, + // Python + python: Languages.py, + py: Languages.py, + + // JavaScript (Bun) + javascript: Languages.js, + js: Languages.js, + 'bun-js': Languages.js, + bun: Languages.js, + + // JavaScript (Node.js) + node: Languages.node, + nodejs: Languages.node, + 'node-js': Languages.node, + 'node-javascript': Languages.node, + + // TypeScript (Bun) + typescript: Languages.ts, + ts: Languages.ts, + 'bun-ts': Languages.ts, + 'bun-typescript': Languages.ts, + + // Bash + bash: Languages.bash, + sh: Languages.bash, }; export function resolveLanguage(lang: string): Languages | undefined { - return languageAliases[lang.toLowerCase()]; + return languageAliases[lang.toLowerCase()]; } const defaultJobTimeoutMs = Number(process.env.JOB_TIMEOUT) || 300000; -const defaultMaxFileSize = Number(process.env.MAX_FILE_SIZE) || 25 * 1024 * 1024; -const defaultExecutionManifestTtlSeconds = Math.min(Math.ceil((defaultJobTimeoutMs + 60000) / 1000), 600); +const defaultMaxFileSize = + Number(process.env.MAX_FILE_SIZE) || 25 * 1024 * 1024; +const defaultExecutionManifestTtlSeconds = Math.min( + Math.ceil((defaultJobTimeoutMs + 60000) / 1000), + 600, +); const EGRESS_GRANT_GRACE_MS = 10 * 60 * 1000; /** Object-store listing and marker writes are metadata operations, not * checkpoint transfers. Bound each tightly so the post-exec checkpoint @@ -70,31 +97,33 @@ const POST_EXEC_CHECKPOINT_REGISTRY_COMMANDS = 6; * The two large transfers receive the configured transfer timeout; the two * metadata operations receive the smaller metadata cap. */ export function checkpointPipelineBudgetMs( - launchTimeoutMs: number, - checkpointTimeoutMs: number, + launchTimeoutMs: number, + checkpointTimeoutMs: number, ): number { - const metadataTimeoutMs = Math.min( - checkpointTimeoutMs, - CHECKPOINT_METADATA_TIMEOUT_CAP_MS, - ); - return launchTimeoutMs - + 2 * checkpointTimeoutMs - + 2 * metadataTimeoutMs - + POST_EXEC_CHECKPOINT_REGISTRY_COMMANDS - * RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS; + const metadataTimeoutMs = Math.min( + checkpointTimeoutMs, + CHECKPOINT_METADATA_TIMEOUT_CAP_MS, + ); + return ( + launchTimeoutMs + + 2 * checkpointTimeoutMs + + 2 * metadataTimeoutMs + + POST_EXEC_CHECKPOINT_REGISTRY_COMMANDS * + RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS + ); } /** BullMQ's `timestamp` is the enqueue time. Anchor the worker deadline to it * so queueing consumes the same caller-visible JOB_TIMEOUT budget instead of * granting a second full timeout after a delayed job finally starts. */ export function jobDeadlineAtMs( - enqueuedAtMs: number | undefined, - timeoutMs: number, - nowMs: number = Date.now(), + enqueuedAtMs: number | undefined, + timeoutMs: number, + nowMs: number = Date.now(), ): number { - return Number.isFinite(enqueuedAtMs) && (enqueuedAtMs as number) > 0 - ? (enqueuedAtMs as number) + timeoutMs - : nowMs + timeoutMs; + return Number.isFinite(enqueuedAtMs) && (enqueuedAtMs as number) > 0 + ? (enqueuedAtMs as number) + timeoutMs + : nowMs + timeoutMs; } /** The worker stops user work at JOB_TIMEOUT, then may still need to terminate @@ -104,298 +133,386 @@ export function jobDeadlineAtMs( export const WORKER_COMPLETION_OVERHEAD_MS = 5_000; export function jobCompletionWaitTimeoutMs( - jobTimeoutMs: number, - backendCleanupTimeoutMs: number, - egressRevokeTimeoutMs: number, + jobTimeoutMs: number, + backendCleanupTimeoutMs: number, + egressRevokeTimeoutMs: number, ): number { - return jobTimeoutMs - + backendCleanupTimeoutMs - + egressRevokeTimeoutMs - + WORKER_COMPLETION_OVERHEAD_MS; + return ( + jobTimeoutMs + + backendCleanupTimeoutMs + + egressRevokeTimeoutMs + + WORKER_COMPLETION_OVERHEAD_MS + ); } export function parseArnList(raw: string | undefined): string[] | undefined { - if (raw == null) return undefined; - const entries = raw.split(',').map((entry) => entry.trim()).filter((entry) => entry.length > 0); - return entries.length > 0 ? entries : undefined; + if (raw == null) return undefined; + const entries = raw + .split(',') + .map(entry => entry.trim()) + .filter(entry => entry.length > 0); + return entries.length > 0 ? entries : undefined; } export interface LambdaMicrovmNumericConfig { - LAMBDA_MICROVM_PORT: number; - LAMBDA_MICROVM_MAX_DURATION_SECONDS: number; - LAMBDA_MICROVM_IDLE_SECONDS: number; - LAMBDA_MICROVM_SUSPEND_SECONDS: number; - LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS: number; - LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS: number; - LAMBDA_MICROVM_HEALTH_TIMEOUT_MS: number; - LAMBDA_MICROVM_LAUNCH_TPS: number; - LAMBDA_MICROVM_TOKEN_TPS: number; + LAMBDA_MICROVM_PORT: number; + LAMBDA_MICROVM_MAX_DURATION_SECONDS: number; + LAMBDA_MICROVM_IDLE_SECONDS: number; + LAMBDA_MICROVM_SUSPEND_SECONDS: number; + LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS: number; + LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS: number; + LAMBDA_MICROVM_HEALTH_TIMEOUT_MS: number; + LAMBDA_MICROVM_LAUNCH_TPS: number; + LAMBDA_MICROVM_TOKEN_TPS: number; } type LambdaMicrovmNumericEnv = Record; interface IntegerRange { - min: number; - max?: number; + min: number; + max?: number; } -const lambdaMicrovmNumericRanges: Record = { - LAMBDA_MICROVM_PORT: { min: 1, max: 65_535 }, - LAMBDA_MICROVM_MAX_DURATION_SECONDS: { min: 1, max: 28_800 }, - LAMBDA_MICROVM_IDLE_SECONDS: { min: 60, max: 28_800 }, - LAMBDA_MICROVM_SUSPEND_SECONDS: { min: 0, max: 28_800 }, - /* Keep proxy credentials shorter than the AWS 60-minute maximum. */ - LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS: { min: 1, max: 900 }, - LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS: { min: 1 }, - LAMBDA_MICROVM_HEALTH_TIMEOUT_MS: { min: 1 }, - LAMBDA_MICROVM_LAUNCH_TPS: { min: 1 }, - LAMBDA_MICROVM_TOKEN_TPS: { min: 1 }, +const lambdaMicrovmNumericRanges: Record< + keyof LambdaMicrovmNumericConfig, + IntegerRange +> = { + LAMBDA_MICROVM_PORT: { min: 1, max: 65_535 }, + LAMBDA_MICROVM_MAX_DURATION_SECONDS: { min: 1, max: 28_800 }, + LAMBDA_MICROVM_IDLE_SECONDS: { min: 60, max: 28_800 }, + LAMBDA_MICROVM_SUSPEND_SECONDS: { min: 0, max: 28_800 }, + /* Keep proxy credentials shorter than the AWS 60-minute maximum. */ + LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS: { min: 1, max: 900 }, + LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS: { min: 1 }, + LAMBDA_MICROVM_HEALTH_TIMEOUT_MS: { min: 1 }, + LAMBDA_MICROVM_LAUNCH_TPS: { min: 1 }, + LAMBDA_MICROVM_TOKEN_TPS: { min: 1 }, }; const lambdaMicrovmNumericDefaults: LambdaMicrovmNumericConfig = { - LAMBDA_MICROVM_PORT: 8080, - LAMBDA_MICROVM_MAX_DURATION_SECONDS: 28_800, - /* 30min keeps a session's VM fully RUNNING (RAM + page cache live, ~0.3s - * follow-ups) across a realistic conversation gap before it suspends; - * 5min proved too aggressive — heavy libraries (chdb ~400MB) pay a - * 30-120s lazy rootfs re-read whenever the cache is lost. */ - LAMBDA_MICROVM_IDLE_SECONDS: 1_800, - LAMBDA_MICROVM_SUSPEND_SECONDS: 1_800, - LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS: 300, - LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS: 60_000, - LAMBDA_MICROVM_HEALTH_TIMEOUT_MS: 5_000, - LAMBDA_MICROVM_LAUNCH_TPS: 4, - LAMBDA_MICROVM_TOKEN_TPS: 8, + LAMBDA_MICROVM_PORT: 8080, + LAMBDA_MICROVM_MAX_DURATION_SECONDS: 28_800, + /* 30min keeps a session's VM fully RUNNING (RAM + page cache live, ~0.3s + * follow-ups) across a realistic conversation gap before it suspends; + * 5min proved too aggressive — heavy libraries (chdb ~400MB) pay a + * 30-120s lazy rootfs re-read whenever the cache is lost. */ + LAMBDA_MICROVM_IDLE_SECONDS: 1_800, + LAMBDA_MICROVM_SUSPEND_SECONDS: 1_800, + LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS: 300, + LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS: 60_000, + LAMBDA_MICROVM_HEALTH_TIMEOUT_MS: 5_000, + LAMBDA_MICROVM_LAUNCH_TPS: 4, + LAMBDA_MICROVM_TOKEN_TPS: 8, }; /** Parse configured values without `||` so an intentional zero survives long * enough for the range validator to accept it where AWS does (suspend duration) * and reject it everywhere else. */ export function resolveLambdaMicrovmNumericConfig( - source: LambdaMicrovmNumericEnv, + source: LambdaMicrovmNumericEnv, ): LambdaMicrovmNumericConfig { - const read = (name: keyof LambdaMicrovmNumericConfig): number => { - const raw = source[name]; - return raw == null || raw.trim() === '' ? lambdaMicrovmNumericDefaults[name] : Number(raw); - }; - return { - LAMBDA_MICROVM_PORT: read('LAMBDA_MICROVM_PORT'), - LAMBDA_MICROVM_MAX_DURATION_SECONDS: read('LAMBDA_MICROVM_MAX_DURATION_SECONDS'), - LAMBDA_MICROVM_IDLE_SECONDS: read('LAMBDA_MICROVM_IDLE_SECONDS'), - LAMBDA_MICROVM_SUSPEND_SECONDS: read('LAMBDA_MICROVM_SUSPEND_SECONDS'), - LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS: read('LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS'), - LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS: read('LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS'), - LAMBDA_MICROVM_HEALTH_TIMEOUT_MS: read('LAMBDA_MICROVM_HEALTH_TIMEOUT_MS'), - LAMBDA_MICROVM_LAUNCH_TPS: read('LAMBDA_MICROVM_LAUNCH_TPS'), - LAMBDA_MICROVM_TOKEN_TPS: read('LAMBDA_MICROVM_TOKEN_TPS'), - }; + const read = (name: keyof LambdaMicrovmNumericConfig): number => { + const raw = source[name]; + return raw == null || raw.trim() === '' + ? lambdaMicrovmNumericDefaults[name] + : Number(raw); + }; + return { + LAMBDA_MICROVM_PORT: read('LAMBDA_MICROVM_PORT'), + LAMBDA_MICROVM_MAX_DURATION_SECONDS: read( + 'LAMBDA_MICROVM_MAX_DURATION_SECONDS', + ), + LAMBDA_MICROVM_IDLE_SECONDS: read('LAMBDA_MICROVM_IDLE_SECONDS'), + LAMBDA_MICROVM_SUSPEND_SECONDS: read('LAMBDA_MICROVM_SUSPEND_SECONDS'), + LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS: read( + 'LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS', + ), + LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS: read( + 'LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS', + ), + LAMBDA_MICROVM_HEALTH_TIMEOUT_MS: read( + 'LAMBDA_MICROVM_HEALTH_TIMEOUT_MS', + ), + LAMBDA_MICROVM_LAUNCH_TPS: read('LAMBDA_MICROVM_LAUNCH_TPS'), + LAMBDA_MICROVM_TOKEN_TPS: read('LAMBDA_MICROVM_TOKEN_TPS'), + }; } /** Returns the first invalid Lambda numeric setting for fail-fast startup. */ export function lambdaMicrovmNumericConfigError( - config: LambdaMicrovmNumericConfig, + config: LambdaMicrovmNumericConfig, ): string | undefined { - for (const name of Object.keys(lambdaMicrovmNumericRanges) as Array) { - const value = config[name]; - const { min, max } = lambdaMicrovmNumericRanges[name]; - if (!Number.isSafeInteger(value) || value < min || (max != null && value > max)) { - const range = max == null ? `at least ${min}` : `between ${min} and ${max}`; - return `${name} must be a whole number ${range}`; + for (const name of Object.keys(lambdaMicrovmNumericRanges) as Array< + keyof LambdaMicrovmNumericConfig + >) { + const value = config[name]; + const { min, max } = lambdaMicrovmNumericRanges[name]; + if ( + !Number.isSafeInteger(value) || + value < min || + (max != null && value > max) + ) { + const range = + max == null ? `at least ${min}` : `between ${min} and ${max}`; + return `${name} must be a whole number ${range}`; + } } - } - return undefined; + return undefined; } -export function resolveEgressGrantTtlSeconds(rawTtlSeconds: string | undefined, jobTimeoutMs: number): number { - const defaultTtlSeconds = Math.max(1, Math.ceil((jobTimeoutMs + EGRESS_GRANT_GRACE_MS) / 1000)); - if (rawTtlSeconds == null || rawTtlSeconds.trim() === '') { - return defaultTtlSeconds; - } +export function resolveEgressGrantTtlSeconds( + rawTtlSeconds: string | undefined, + jobTimeoutMs: number, +): number { + const defaultTtlSeconds = Math.max( + 1, + Math.ceil((jobTimeoutMs + EGRESS_GRANT_GRACE_MS) / 1000), + ); + if (rawTtlSeconds == null || rawTtlSeconds.trim() === '') { + return defaultTtlSeconds; + } - const configuredTtlSeconds = Number(rawTtlSeconds); - if (!Number.isFinite(configuredTtlSeconds) || configuredTtlSeconds <= 0) { - return defaultTtlSeconds; - } + const configuredTtlSeconds = Number(rawTtlSeconds); + if (!Number.isFinite(configuredTtlSeconds) || configuredTtlSeconds <= 0) { + return defaultTtlSeconds; + } - return Math.max(1, Math.ceil(configuredTtlSeconds)); + return Math.max(1, Math.ceil(configuredTtlSeconds)); } -const lambdaMicrovmNumericConfig = resolveLambdaMicrovmNumericConfig(process.env); +const lambdaMicrovmNumericConfig = resolveLambdaMicrovmNumericConfig( + process.env, +); function configuredNumber(raw: string | undefined, fallback: number): number { - return raw == null || raw.trim() === '' ? fallback : Number(raw); + return raw == null || raw.trim() === '' ? fallback : Number(raw); } function configuredChoice( - raw: string | undefined, - name: string, - fallback: T, - allowed: readonly T[], + raw: string | undefined, + name: string, + fallback: T, + allowed: readonly T[], ): T { - if (raw == null) return fallback; - if (allowed.includes(raw as T)) return raw as T; - throw new Error(`${name} must be one of: ${allowed.join(', ')}`); + if (raw == null) return fallback; + if (allowed.includes(raw as T)) return raw as T; + throw new Error(`${name} must be one of: ${allowed.join(', ')}`); } export function resolveSandboxBackend( - raw: string | undefined, + raw: string | undefined, ): 'http' | 'lambda-microvm' { - return configuredChoice( - raw, - 'CODEAPI_SANDBOX_BACKEND', - 'http', - ['http', 'lambda-microvm'], - ); + return configuredChoice(raw, 'CODEAPI_SANDBOX_BACKEND', 'http', [ + 'http', + 'lambda-microvm', + ]); } export function resolveRuntimeSessionMode( - raw: string | undefined, + raw: string | undefined, ): 'stateless' | 'affinity' | 'strict' { - return configuredChoice( - raw, - 'CODEAPI_RUNTIME_SESSION_MODE', - 'stateless', - ['stateless', 'affinity', 'strict'], - ); + return configuredChoice(raw, 'CODEAPI_RUNTIME_SESSION_MODE', 'stateless', [ + 'stateless', + 'affinity', + 'strict', + ]); } export const env = { - PORT: process.env.SERVICE_PORT ?? 3112, - LOCAL_MODE: process.env.LOCAL_MODE === 'true', - HARDENED_SANDBOX_MODE: process.env.CODEAPI_HARDENED_SANDBOX_MODE === 'true', - INSTANCE_ID: process.env.INSTANCE_ID ?? nanoid(), - HTTP_JSON_LIMIT: process.env.CODEAPI_HTTP_JSON_LIMIT ?? '50mb', - SANDBOX_ENDPOINT: process.env.SANDBOX_ENDPOINT ?? 'http://localhost:2000/api/v2', - EGRESS_GATEWAY_URL: process.env.EGRESS_GATEWAY_URL ?? '', - FILE_SERVER_URL: process.env.FILE_SERVER_URL ?? 'http://localhost:3000', - TOOL_CALL_SERVER_URL: process.env.TOOL_CALL_SERVER_URL ?? 'http://localhost:3033', - EGRESS_GATEWAY_PORT: Number(process.env.EGRESS_GATEWAY_PORT) || 3190, - EGRESS_GATEWAY_FILE_SERVER_URL: process.env.EGRESS_GATEWAY_FILE_SERVER_URL ?? process.env.FILE_SERVER_URL ?? 'http://localhost:3000', - EGRESS_GATEWAY_TOOL_CALL_SERVER_URL: process.env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL ?? process.env.TOOL_CALL_SERVER_URL ?? 'http://localhost:3033', - EGRESS_GATEWAY_MAX_TOOL_CALL_BYTES: Number(process.env.EGRESS_GATEWAY_MAX_TOOL_CALL_BYTES) || 1024 * 1024, - EGRESS_GATEWAY_MAX_FILE_BYTES: Number(process.env.EGRESS_GATEWAY_MAX_FILE_BYTES ?? process.env.SANDBOX_MAX_FILE_SIZE) || 10_000_000, - EGRESS_GATEWAY_MAX_PATH_LENGTH: Number(process.env.EGRESS_GATEWAY_MAX_PATH_LENGTH ?? process.env.SANDBOX_MAX_PATH_LENGTH) || 256, - EGRESS_GATEWAY_MAX_NESTING_DEPTH: Number(process.env.EGRESS_GATEWAY_MAX_NESTING_DEPTH ?? process.env.SANDBOX_MAX_NESTING_DEPTH) || 10, - EGRESS_GATEWAY_REQUEST_TIMEOUT_MS: Number(process.env.EGRESS_GATEWAY_REQUEST_TIMEOUT_MS) || 30_000, - EGRESS_GATEWAY_REVOKE_TIMEOUT_MS: Number(process.env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS) || 5_000, - EGRESS_LEDGER_REQUIRED: process.env.CODEAPI_EGRESS_LEDGER_REQUIRED === 'true' || process.env.CODEAPI_HARDENED_SANDBOX_MODE === 'true', - EGRESS_LEDGER_TTL_GRACE_SECONDS: Number(process.env.CODEAPI_EGRESS_LEDGER_TTL_GRACE_SECONDS) || 300, - EGRESS_GRANT_SECRET: process.env.CODEAPI_EGRESS_GRANT_SECRET ?? '', - EGRESS_GRANT_TTL_SECONDS: resolveEgressGrantTtlSeconds(process.env.EGRESS_GRANT_TTL_SECONDS, defaultJobTimeoutMs), - PYTHON_CONCURRENCY: Number(process.env.PYTHON_CONCURRENCY) || 1, - OTHER_CONCURRENCY: Number(process.env.OTHER_CONCURRENCY) || 8, - JOB_WINDOW: Number(process.env.JOB_WINDOW) || 1000, - MAX_UPLOAD_CHECKS: Number(process.env.MAX_UPLOAD_CHECKS) || 14, - MAX_UPLOAD_WAIT: Number(process.env.MAX_UPLOAD_WAIT) || 500, - MAX_FILE_SIZE: defaultMaxFileSize, - JOB_TIMEOUT: defaultJobTimeoutMs, // 5 minutes (increased for complex matplotlib rendering) - // Execution Rate Limits - EXEC_LIMIT_WINDOW: Number(process.env.RATE_LIMIT_WINDOW) || 30 * 1000, // 30 seconds - EXEC_MAX_REQUESTS: Number(process.env.MAX_REQUESTS) || 20, // execution requests per window - // Upload Rate Limits - UPLOAD_LIMIT_WINDOW: Number(process.env.UPLOAD_LIMIT_WINDOW) || 5 * 60 * 1000, // 5 minutes - UPLOAD_MAX_REQUESTS: Number(process.env.UPLOAD_MAX_REQUESTS) || 30, // 30 uploads per 5 minutes - // Download Rate Limits - DOWNLOAD_LIMIT_WINDOW: Number(process.env.DOWNLOAD_LIMIT_WINDOW) || 60 * 1000, // 1 minute - DOWNLOAD_MAX_REQUESTS: Number(process.env.DOWNLOAD_MAX_REQUESTS) || 60, // 60 downloads per minute - // Files List Rate Limits - FETCH_LIMIT_WINDOW: Number(process.env.FETCH_LIMIT_WINDOW) || 60 * 1000, // 1 minute - FETCH_MAX_REQUESTS: Number(process.env.FETCH_MAX_REQUESTS) || 120, // 120 requests per minute - // Redis Key Cache Config - SESSION_CACHE_TTL: Number(process.env.SESSION_CACHE_TTL) || 86400, - /** Strict tenant isolation. When true, sessionKey resolution fails closed - * (500) on requests whose auth context lacks `tenantId`, instead of - * silently falling back to the `'legacy'` tenant prefix. Default OFF in - * code so single-tenant deploys without an auth tenancy concept keep - * working; multi-tenant deploys MUST set this to `true` before any tenant - * is multi-homed, otherwise a missing tenantId would silently bucket - * cross-tenant requests under the same `'legacy'` prefix. */ - TENANT_ISOLATION_STRICT: process.env.CODEAPI_TENANT_ISOLATION_STRICT === 'true', - // Signed execution manifests. Prefer private/public key mode for split-runner - // deployments so sandbox-runner receives only a verifier, not a signing secret. - EXECUTION_MANIFEST_PRIVATE_KEY: process.env.CODEAPI_EXECUTION_MANIFEST_PRIVATE_KEY ?? '', - EXECUTION_MANIFEST_PUBLIC_KEY: process.env.CODEAPI_EXECUTION_MANIFEST_PUBLIC_KEY ?? '', - // Legacy HMAC fallback for non-split deployments. Do not mount into sandbox-runner. - EXECUTION_MANIFEST_SECRET: process.env.CODEAPI_EXECUTION_MANIFEST_SECRET ?? '', - EXECUTION_MANIFEST_TTL_SECONDS: Math.min( - Number(process.env.EXECUTION_MANIFEST_TTL_SECONDS) || defaultExecutionManifestTtlSeconds, - 600, - ), - EXECUTION_MANIFEST_MAX_UPLOAD_BYTES: Number(process.env.EXECUTION_MANIFEST_MAX_UPLOAD_BYTES) || defaultMaxFileSize, - EXECUTION_MANIFEST_MAX_OUTPUT_FILES: Number(process.env.EXECUTION_MANIFEST_MAX_OUTPUT_FILES) || 50, - EXECUTION_MANIFEST_MAX_REQUESTS: Number(process.env.EXECUTION_MANIFEST_MAX_REQUESTS) || 1000, - // Redis - Alternative DNS Lookup for AWS ElastiCache TLS connections - REDIS_USE_ALTERNATIVE_DNS_LOOKUP: process.env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP === 'true', - /** - * Programmatic Tool Calling execution model. - * - `replay` (default): Temporal-style replay. Sandbox exits between round-trips; - * tool results are persisted in Redis and replayed into a fresh sandbox on each - * continuation until the code either completes or surfaces new tool calls. - * Safe to scale horizontally since all state lives in Redis. - * - `blocking`: legacy path. Sandbox process stays alive across tool round-trips - * via a long-polling HTTP callback through the Tool Call Server. Retained as - * an explicit opt-in during rollout; scheduled for removal in a follow-up. - */ - PTC_MODE: (process.env.PTC_MODE === 'blocking' ? 'blocking' : 'replay') as 'replay' | 'blocking', - PTC_DEBUG: process.env.PTC_DEBUG === 'true', - /** - * Sandbox execution backend. - * - `http` (default): POST signed execute requests to SANDBOX_ENDPOINT - * (current Kubernetes/libkrun sandbox-runner). - * - `lambda-microvm`: AWS Lambda MicroVM backend. - */ - SANDBOX_BACKEND: resolveSandboxBackend(process.env.CODEAPI_SANDBOX_BACKEND), - /** - * Runtime session affinity for stateful sandbox backends. - * - `stateless` (default): no runtime sessions; `runtime_session_hint` ignored. - * - `affinity`: stateful session reuse; contention surfaces as HTTP 409 - * rather than silently losing workspace state in a stateless execution. - * - `strict`: same serialized session semantics, and a session hint is - * required instead of degrading requests without one to stateless. - */ - RUNTIME_SESSION_MODE: resolveRuntimeSessionMode(process.env.CODEAPI_RUNTIME_SESSION_MODE), - RUNTIME_SESSION_LOCK_WAIT_MS: configuredNumber( - process.env.CODEAPI_RUNTIME_SESSION_LOCK_WAIT_MS, - 15_000, - ), - // Lambda MicroVM backend. Connector lists are comma-separated ARNs. - LAMBDA_MICROVM_IMAGE_ARN: process.env.LAMBDA_MICROVM_IMAGE_ARN ?? '', - LAMBDA_MICROVM_IMAGE_VERSION: process.env.LAMBDA_MICROVM_IMAGE_VERSION || undefined, - LAMBDA_MICROVM_EXECUTION_ROLE_ARN: process.env.LAMBDA_MICROVM_EXECUTION_ROLE_ARN || undefined, - /* Runtime VM stdout reaches CloudWatch only when RunMicrovm sends a logging - * config AND an executionRoleArn is set — pairs with the role above. */ - LAMBDA_MICROVM_LOG_GROUP: process.env.LAMBDA_MICROVM_LOG_GROUP || undefined, - LAMBDA_MICROVM_REGION: process.env.LAMBDA_MICROVM_REGION || undefined, - LAMBDA_MICROVM_INGRESS_CONNECTOR_ARNS: parseArnList(process.env.LAMBDA_MICROVM_INGRESS_CONNECTOR_ARNS), - LAMBDA_MICROVM_EGRESS_CONNECTOR_ARNS: parseArnList(process.env.LAMBDA_MICROVM_EGRESS_CONNECTOR_ARNS), - ...lambdaMicrovmNumericConfig, - /* CreateMicrovmAuthToken is minted per execute + per checkpoint; share a - * fleet-wide budget so concurrent warm-session executes queue instead of - * bursting past the AWS TPS limit. */ - LAMBDA_MICROVM_ALLOW_SHELL: process.env.LAMBDA_MICROVM_ALLOW_SHELL === 'true', - /* Session workspace checkpoints (effective only in affinity/strict modes). - * On by default so VM expiry/eviction recovery is automatic; the byte cap - * bounds tar size pulled from the VM and stored to S3. */ - SESSION_CHECKPOINTS: process.env.CODEAPI_SESSION_CHECKPOINTS !== 'false', - CHECKPOINT_MAX_BYTES: configuredNumber( - process.env.CODEAPI_CHECKPOINT_MAX_BYTES, - 512 * 1024 * 1024, - ), - CHECKPOINT_TIMEOUT_MS: configuredNumber(process.env.CODEAPI_CHECKPOINT_TIMEOUT_MS, 60_000), - CHECKPOINT_PREFIX: process.env.CODEAPI_CHECKPOINT_PREFIX ?? 'rtsx-checkpoints/', + PORT: process.env.SERVICE_PORT ?? 3112, + LOCAL_MODE: process.env.LOCAL_MODE === 'true', + HARDENED_SANDBOX_MODE: process.env.CODEAPI_HARDENED_SANDBOX_MODE === 'true', + INSTANCE_ID: process.env.INSTANCE_ID ?? nanoid(), + HTTP_JSON_LIMIT: process.env.CODEAPI_HTTP_JSON_LIMIT ?? '50mb', + SANDBOX_ENDPOINT: + process.env.SANDBOX_ENDPOINT ?? 'http://localhost:2000/api/v2', + EGRESS_GATEWAY_URL: process.env.EGRESS_GATEWAY_URL ?? '', + FILE_SERVER_URL: process.env.FILE_SERVER_URL ?? 'http://localhost:3000', + TOOL_CALL_SERVER_URL: + process.env.TOOL_CALL_SERVER_URL ?? 'http://localhost:3033', + EGRESS_GATEWAY_PORT: Number(process.env.EGRESS_GATEWAY_PORT) || 3190, + EGRESS_GATEWAY_FILE_SERVER_URL: + process.env.EGRESS_GATEWAY_FILE_SERVER_URL ?? + process.env.FILE_SERVER_URL ?? + 'http://localhost:3000', + EGRESS_GATEWAY_TOOL_CALL_SERVER_URL: + process.env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL ?? + process.env.TOOL_CALL_SERVER_URL ?? + 'http://localhost:3033', + EGRESS_GATEWAY_MAX_TOOL_CALL_BYTES: + Number(process.env.EGRESS_GATEWAY_MAX_TOOL_CALL_BYTES) || 1024 * 1024, + EGRESS_GATEWAY_MAX_FILE_BYTES: + Number( + process.env.EGRESS_GATEWAY_MAX_FILE_BYTES ?? + process.env.SANDBOX_MAX_FILE_SIZE, + ) || 10_000_000, + EGRESS_GATEWAY_MAX_PATH_LENGTH: + Number( + process.env.EGRESS_GATEWAY_MAX_PATH_LENGTH ?? + process.env.SANDBOX_MAX_PATH_LENGTH, + ) || 256, + EGRESS_GATEWAY_MAX_NESTING_DEPTH: + Number( + process.env.EGRESS_GATEWAY_MAX_NESTING_DEPTH ?? + process.env.SANDBOX_MAX_NESTING_DEPTH, + ) || 10, + EGRESS_GATEWAY_REQUEST_TIMEOUT_MS: + Number(process.env.EGRESS_GATEWAY_REQUEST_TIMEOUT_MS) || 30_000, + EGRESS_GATEWAY_REVOKE_TIMEOUT_MS: + Number(process.env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS) || 5_000, + EGRESS_LEDGER_REQUIRED: + process.env.CODEAPI_EGRESS_LEDGER_REQUIRED === 'true' || + process.env.CODEAPI_HARDENED_SANDBOX_MODE === 'true', + EGRESS_LEDGER_TTL_GRACE_SECONDS: + Number(process.env.CODEAPI_EGRESS_LEDGER_TTL_GRACE_SECONDS) || 300, + EGRESS_GRANT_SECRET: process.env.CODEAPI_EGRESS_GRANT_SECRET ?? '', + EGRESS_GRANT_TTL_SECONDS: resolveEgressGrantTtlSeconds( + process.env.EGRESS_GRANT_TTL_SECONDS, + defaultJobTimeoutMs, + ), + PYTHON_CONCURRENCY: Number(process.env.PYTHON_CONCURRENCY) || 1, + OTHER_CONCURRENCY: Number(process.env.OTHER_CONCURRENCY) || 8, + JOB_WINDOW: Number(process.env.JOB_WINDOW) || 1000, + MAX_UPLOAD_CHECKS: Number(process.env.MAX_UPLOAD_CHECKS) || 14, + MAX_UPLOAD_WAIT: Number(process.env.MAX_UPLOAD_WAIT) || 500, + MAX_FILE_SIZE: defaultMaxFileSize, + JOB_TIMEOUT: defaultJobTimeoutMs, // 5 minutes (increased for complex matplotlib rendering) + // Execution Rate Limits + EXEC_LIMIT_WINDOW: Number(process.env.RATE_LIMIT_WINDOW) || 30 * 1000, // 30 seconds + EXEC_MAX_REQUESTS: Number(process.env.MAX_REQUESTS) || 20, // execution requests per window + // Upload Rate Limits + UPLOAD_LIMIT_WINDOW: + Number(process.env.UPLOAD_LIMIT_WINDOW) || 5 * 60 * 1000, // 5 minutes + UPLOAD_MAX_REQUESTS: Number(process.env.UPLOAD_MAX_REQUESTS) || 30, // 30 uploads per 5 minutes + // Download Rate Limits + DOWNLOAD_LIMIT_WINDOW: + Number(process.env.DOWNLOAD_LIMIT_WINDOW) || 60 * 1000, // 1 minute + DOWNLOAD_MAX_REQUESTS: Number(process.env.DOWNLOAD_MAX_REQUESTS) || 60, // 60 downloads per minute + // Files List Rate Limits + FETCH_LIMIT_WINDOW: Number(process.env.FETCH_LIMIT_WINDOW) || 60 * 1000, // 1 minute + FETCH_MAX_REQUESTS: Number(process.env.FETCH_MAX_REQUESTS) || 120, // 120 requests per minute + // Redis Key Cache Config + SESSION_CACHE_TTL: Number(process.env.SESSION_CACHE_TTL) || 86400, + /** Strict tenant isolation. When true, sessionKey resolution fails closed + * (500) on requests whose auth context lacks `tenantId`, instead of + * silently falling back to the `'legacy'` tenant prefix. Default OFF in + * code so single-tenant deploys without an auth tenancy concept keep + * working; multi-tenant deploys MUST set this to `true` before any tenant + * is multi-homed, otherwise a missing tenantId would silently bucket + * cross-tenant requests under the same `'legacy'` prefix. */ + TENANT_ISOLATION_STRICT: + process.env.CODEAPI_TENANT_ISOLATION_STRICT === 'true', + // Signed execution manifests. Prefer private/public key mode for split-runner + // deployments so sandbox-runner receives only a verifier, not a signing secret. + EXECUTION_MANIFEST_PRIVATE_KEY: + process.env.CODEAPI_EXECUTION_MANIFEST_PRIVATE_KEY ?? '', + EXECUTION_MANIFEST_PUBLIC_KEY: + process.env.CODEAPI_EXECUTION_MANIFEST_PUBLIC_KEY ?? '', + // Legacy HMAC fallback for non-split deployments. Do not mount into sandbox-runner. + EXECUTION_MANIFEST_SECRET: + process.env.CODEAPI_EXECUTION_MANIFEST_SECRET ?? '', + EXECUTION_MANIFEST_TTL_SECONDS: Math.min( + Number(process.env.EXECUTION_MANIFEST_TTL_SECONDS) || + defaultExecutionManifestTtlSeconds, + 600, + ), + EXECUTION_MANIFEST_MAX_UPLOAD_BYTES: + Number(process.env.EXECUTION_MANIFEST_MAX_UPLOAD_BYTES) || + defaultMaxFileSize, + EXECUTION_MANIFEST_MAX_OUTPUT_FILES: + Number(process.env.EXECUTION_MANIFEST_MAX_OUTPUT_FILES) || 50, + EXECUTION_MANIFEST_MAX_REQUESTS: + Number(process.env.EXECUTION_MANIFEST_MAX_REQUESTS) || 1000, + // Redis - Cluster mode (GCP Memorystore cluster, AWS ElastiCache cluster, etc.) + USE_REDIS_CLUSTER: process.env.USE_REDIS_CLUSTER === 'true', + // Redis - Alternative DNS Lookup for AWS ElastiCache TLS connections + REDIS_USE_ALTERNATIVE_DNS_LOOKUP: + process.env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP === 'true', + /** + * Programmatic Tool Calling execution model. + * - `replay` (default): Temporal-style replay. Sandbox exits between round-trips; + * tool results are persisted in Redis and replayed into a fresh sandbox on each + * continuation until the code either completes or surfaces new tool calls. + * Safe to scale horizontally since all state lives in Redis. + * - `blocking`: legacy path. Sandbox process stays alive across tool round-trips + * via a long-polling HTTP callback through the Tool Call Server. Retained as + * an explicit opt-in during rollout; scheduled for removal in a follow-up. + */ + PTC_MODE: (process.env.PTC_MODE === 'blocking' ? 'blocking' : 'replay') as + | 'replay' + | 'blocking', + PTC_DEBUG: process.env.PTC_DEBUG === 'true', + /** + * Sandbox execution backend. + * - `http` (default): POST signed execute requests to SANDBOX_ENDPOINT + * (current Kubernetes/libkrun sandbox-runner). + * - `lambda-microvm`: AWS Lambda MicroVM backend. + */ + SANDBOX_BACKEND: resolveSandboxBackend(process.env.CODEAPI_SANDBOX_BACKEND), + /** + * Runtime session affinity for stateful sandbox backends. + * - `stateless` (default): no runtime sessions; `runtime_session_hint` ignored. + * - `affinity`: stateful session reuse; contention surfaces as HTTP 409 + * rather than silently losing workspace state in a stateless execution. + * - `strict`: same serialized session semantics, and a session hint is + * required instead of degrading requests without one to stateless. + */ + RUNTIME_SESSION_MODE: resolveRuntimeSessionMode( + process.env.CODEAPI_RUNTIME_SESSION_MODE, + ), + RUNTIME_SESSION_LOCK_WAIT_MS: configuredNumber( + process.env.CODEAPI_RUNTIME_SESSION_LOCK_WAIT_MS, + 15_000, + ), + // Lambda MicroVM backend. Connector lists are comma-separated ARNs. + LAMBDA_MICROVM_IMAGE_ARN: process.env.LAMBDA_MICROVM_IMAGE_ARN ?? '', + LAMBDA_MICROVM_IMAGE_VERSION: + process.env.LAMBDA_MICROVM_IMAGE_VERSION || undefined, + LAMBDA_MICROVM_EXECUTION_ROLE_ARN: + process.env.LAMBDA_MICROVM_EXECUTION_ROLE_ARN || undefined, + /* Runtime VM stdout reaches CloudWatch only when RunMicrovm sends a logging + * config AND an executionRoleArn is set — pairs with the role above. */ + LAMBDA_MICROVM_LOG_GROUP: process.env.LAMBDA_MICROVM_LOG_GROUP || undefined, + LAMBDA_MICROVM_REGION: process.env.LAMBDA_MICROVM_REGION || undefined, + LAMBDA_MICROVM_INGRESS_CONNECTOR_ARNS: parseArnList( + process.env.LAMBDA_MICROVM_INGRESS_CONNECTOR_ARNS, + ), + LAMBDA_MICROVM_EGRESS_CONNECTOR_ARNS: parseArnList( + process.env.LAMBDA_MICROVM_EGRESS_CONNECTOR_ARNS, + ), + ...lambdaMicrovmNumericConfig, + /* CreateMicrovmAuthToken is minted per execute + per checkpoint; share a + * fleet-wide budget so concurrent warm-session executes queue instead of + * bursting past the AWS TPS limit. */ + LAMBDA_MICROVM_ALLOW_SHELL: + process.env.LAMBDA_MICROVM_ALLOW_SHELL === 'true', + /* Session workspace checkpoints (effective only in affinity/strict modes). + * On by default so VM expiry/eviction recovery is automatic; the byte cap + * bounds tar size pulled from the VM and stored to S3. */ + SESSION_CHECKPOINTS: process.env.CODEAPI_SESSION_CHECKPOINTS !== 'false', + CHECKPOINT_MAX_BYTES: configuredNumber( + process.env.CODEAPI_CHECKPOINT_MAX_BYTES, + 512 * 1024 * 1024, + ), + CHECKPOINT_TIMEOUT_MS: configuredNumber( + process.env.CODEAPI_CHECKPOINT_TIMEOUT_MS, + 60_000, + ), + CHECKPOINT_PREFIX: + process.env.CODEAPI_CHECKPOINT_PREFIX ?? 'rtsx-checkpoints/', }; const default_run_memory_limit = 256 * 1024 * 1024; type PlanLimit = { - run_memory_limit?: number; - max_file_size?: number; + run_memory_limit?: number; + max_file_size?: number; }; type PlanLimits = { - default: Required; + default: Required; } & { - [key: string]: PlanLimit | undefined; + [key: string]: PlanLimit | undefined; }; /** @@ -403,26 +520,38 @@ type PlanLimits = { * JSON object keyed by the `plan_id` JWT claim. Unknown or absent plan ids * fall back to the default tier, which is the only entry defined in code. */ -export function parsePlanLimits(raw: string | undefined): Record { - if (raw == null || raw.trim() === '') { - return {}; - } - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (error) { - throw new Error(`CODEAPI_PLAN_LIMITS is not valid JSON: ${(error as Error).message}`); - } - if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { - throw new Error('CODEAPI_PLAN_LIMITS must be a JSON object keyed by plan id'); - } - return parsed as Record; +export function parsePlanLimits( + raw: string | undefined, +): Record { + if (raw == null || raw.trim() === '') { + return {}; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + throw new Error( + `CODEAPI_PLAN_LIMITS is not valid JSON: ${(error as Error).message}`, + ); + } + if ( + parsed === null || + typeof parsed !== 'object' || + Array.isArray(parsed) + ) { + throw new Error( + 'CODEAPI_PLAN_LIMITS must be a JSON object keyed by plan id', + ); + } + return parsed as Record; } export const planLimits: PlanLimits = { - ...parsePlanLimits(process.env.CODEAPI_PLAN_LIMITS), - default: { - run_memory_limit: Number(process.env.SANDBOX_RUN_MEMORY_LIMIT) || default_run_memory_limit, - max_file_size: env.MAX_FILE_SIZE, - }, + ...parsePlanLimits(process.env.CODEAPI_PLAN_LIMITS), + default: { + run_memory_limit: + Number(process.env.SANDBOX_RUN_MEMORY_LIMIT) || + default_run_memory_limit, + max_file_size: env.MAX_FILE_SIZE, + }, }; diff --git a/service/src/egress-ledger.ts b/service/src/egress-ledger.ts index 24dda87..6ceaec3 100644 --- a/service/src/egress-ledger.ts +++ b/service/src/egress-ledger.ts @@ -1,343 +1,432 @@ -import IORedis from 'ioredis'; -import type { CommonRedisOptions } from 'ioredis'; -import type * as tls from 'tls'; +import type { Redis, Cluster } from 'ioredis'; import { env } from './config'; import type { EgressGrantClaims } from './egress-grant'; import { EgressGrantError } from './egress-grant'; import logger from './logger'; -import { redisKeepAliveOptions } from './redis-options'; +import { + createRedisConnection, + isClusterMode, + type RedisClient, +} from './redis-connection'; type LedgerStatus = 'active' | 'revoked'; export interface EgressLedgerRecord { - grant_id: string; - exec_id: string; - status: LedgerStatus; - exp: number; - revoked_at?: number; - revoke_reason?: string; - input_files: EgressGrantClaims['input_files']; - read_sessions: string[]; - output_session_id: string; - max_upload_bytes: number; - max_output_files: number; - max_requests: number; - request_count: number; - read_count: number; - upload_count: number; - tool_call_count: number; - uploaded_bytes: number; - output_file_ids: string[]; + grant_id: string; + exec_id: string; + status: LedgerStatus; + exp: number; + revoked_at?: number; + revoke_reason?: string; + input_files: EgressGrantClaims['input_files']; + read_sessions: string[]; + output_session_id: string; + max_upload_bytes: number; + max_output_files: number; + max_requests: number; + request_count: number; + read_count: number; + upload_count: number; + tool_call_count: number; + uploaded_bytes: number; + output_file_ids: string[]; } -let redis: IORedis | null = null; +let redis: RedisClient | null = null; const LEDGER_MUTATION_ATTEMPTS = 32; -const LEDGER_MUTATION_POOL_SIZE = Math.max(1, Number(process.env.CODEAPI_EGRESS_LEDGER_MUTATION_CONNECTIONS) || 32); +const LEDGER_MUTATION_POOL_SIZE = Math.max( + 1, + Number(process.env.CODEAPI_EGRESS_LEDGER_MUTATION_CONNECTIONS) || 32, +); type MutationConnectionWaiter = { - resolve: (client: IORedis) => void; - reject: (error: Error) => void; + resolve: (client: RedisClient) => void; + reject: (error: Error) => void; }; -const mutationConnections = new Set(); -let idleMutationConnections: IORedis[] = []; +const mutationConnections = new Set(); +let idleMutationConnections: RedisClient[] = []; let mutationConnectionWaiters: MutationConnectionWaiter[] = []; -export function setEgressLedgerRedisForTest(client: IORedis | null): void { - resetMutationConnections(); - redis = client; +export function setEgressLedgerRedisForTest( + client: Redis | Cluster | null, +): void { + resetMutationConnections(); + redis = client; } function ledgerKey(grantId: string): string { - return `codeapi:egress:grant:${grantId}`; + return `codeapi:egress:grant:${grantId}`; } function ttlSeconds(exp: number): number { - return Math.max(1, exp - Math.floor(Date.now() / 1000) + env.EGRESS_LEDGER_TTL_GRACE_SECONDS); + return Math.max( + 1, + exp - + Math.floor(Date.now() / 1000) + + env.EGRESS_LEDGER_TTL_GRACE_SECONDS, + ); } -function redisConnection(): IORedis { - if (redis) return redis; - const retryStrategy: CommonRedisOptions['retryStrategy'] = times => { +const ledgerRetryStrategy = (times: number): number | null => { if (times > 5) return null; return 2000; - }; - redis = new IORedis({ - host: process.env.REDIS_HOST ?? 'redis', - port: Number(process.env.REDIS_PORT) || 6379, - password: process.env.REDIS_PASSWORD, - maxRetriesPerRequest: 1, - retryStrategy, - enableReadyCheck: true, - connectTimeout: 10000, - ...redisKeepAliveOptions(), - tls: process.env.REDIS_TLS === 'true' - ? { rejectUnauthorized: false } as tls.ConnectionOptions - : undefined, - ...(env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP - ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } - : {}), - }); - redis.on('error', error => logger.error('Egress ledger Redis error', { error })); - return redis; +}; + +function redisConnection(): RedisClient { + if (redis) return redis; + redis = createRedisConnection({ + maxRetriesPerRequest: 1, + retryStrategy: ledgerRetryStrategy, + enableReadyCheck: true, + }); + redis.on('error', error => + logger.error('Egress ledger Redis error', { error }), + ); + return redis; } function resetMutationConnections(): void { - const resetError = new Error('Egress ledger Redis connection reset'); - for (const waiter of mutationConnectionWaiters) { - waiter.reject(resetError); - } - mutationConnectionWaiters = []; - idleMutationConnections = []; - for (const client of mutationConnections) { - client.disconnect(); - } - mutationConnections.clear(); + const resetError = new Error('Egress ledger Redis connection reset'); + for (const waiter of mutationConnectionWaiters) { + waiter.reject(resetError); + } + mutationConnectionWaiters = []; + idleMutationConnections = []; + for (const client of mutationConnections) { + client.disconnect(); + } + mutationConnections.clear(); } -async function dedicatedMutationConnection(): Promise { - while (idleMutationConnections.length > 0) { - const client = idleMutationConnections.pop()!; - if (client.status !== 'end') { - return client; +async function dedicatedMutationConnection(): Promise { + while (idleMutationConnections.length > 0) { + const client = idleMutationConnections.pop()!; + if (client.status !== 'end') { + return client; + } + mutationConnections.delete(client); } - mutationConnections.delete(client); - } - if (mutationConnections.size < LEDGER_MUTATION_POOL_SIZE) { - return createMutationConnection(); - } + if (mutationConnections.size < LEDGER_MUTATION_POOL_SIZE) { + return createMutationConnection(); + } - return new Promise((resolve, reject) => { - mutationConnectionWaiters.push({ resolve, reject }); - }); + return new Promise((resolve, reject) => { + mutationConnectionWaiters.push({ resolve, reject }); + }); } -function createMutationConnection(): IORedis { - const client = redisConnection().duplicate(); - mutationConnections.add(client); - client.on('error', error => logger.error('Egress ledger mutation Redis error', { error })); - return client; +function createMutationConnection(): RedisClient { + // Cluster does not expose .duplicate(); create a fresh connection instead. + const client: RedisClient = isClusterMode() + ? createRedisConnection({ + maxRetriesPerRequest: 1, + retryStrategy: ledgerRetryStrategy, + enableReadyCheck: true, + }) + : (redisConnection() as Redis).duplicate(); + mutationConnections.add(client); + client.on('error', error => + logger.error('Egress ledger mutation Redis error', { error }), + ); + return client; } -function releaseMutationConnection(client: IORedis): void { - if (!mutationConnections.has(client) || client.status === 'end') { - mutationConnections.delete(client); +function releaseMutationConnection(client: RedisClient): void { + if (!mutationConnections.has(client) || client.status === 'end') { + mutationConnections.delete(client); + const waiter = mutationConnectionWaiters.shift(); + if (waiter) { + try { + waiter.resolve(createMutationConnection()); + } catch (error) { + waiter.reject( + error instanceof Error ? error : new Error(String(error)), + ); + } + } + return; + } + const waiter = mutationConnectionWaiters.shift(); if (waiter) { - try { - waiter.resolve(createMutationConnection()); - } catch (error) { - waiter.reject(error instanceof Error ? error : new Error(String(error))); - } + waiter.resolve(client); + return; } - return; - } - const waiter = mutationConnectionWaiters.shift(); - if (waiter) { - waiter.resolve(client); - return; - } - - idleMutationConnections.push(client); + idleMutationConnections.push(client); } function sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); + return new Promise(resolve => setTimeout(resolve, ms)); } export async function pingEgressLedger(): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().ping(); + if (!env.EGRESS_LEDGER_REQUIRED) return; + await redisConnection().ping(); } function recordFromGrant(grant: EgressGrantClaims): EgressLedgerRecord { - return { - grant_id: grant.grant_id, - exec_id: grant.exec_id, - status: 'active', - exp: grant.exp, - input_files: grant.input_files, - read_sessions: grant.read_sessions, - output_session_id: grant.output_session_id, - max_upload_bytes: grant.max_upload_bytes, - max_output_files: grant.max_output_files, - max_requests: grant.max_requests, - request_count: 0, - read_count: 0, - upload_count: 0, - tool_call_count: 0, - uploaded_bytes: 0, - output_file_ids: [], - }; + return { + grant_id: grant.grant_id, + exec_id: grant.exec_id, + status: 'active', + exp: grant.exp, + input_files: grant.input_files, + read_sessions: grant.read_sessions, + output_session_id: grant.output_session_id, + max_upload_bytes: grant.max_upload_bytes, + max_output_files: grant.max_output_files, + max_requests: grant.max_requests, + request_count: 0, + read_count: 0, + upload_count: 0, + tool_call_count: 0, + uploaded_bytes: 0, + output_file_ids: [], + }; } -export async function createEgressLedger(grant: EgressGrantClaims): Promise { - if (!grant.grant_id) { - throw new EgressGrantError('malformed', 'Egress grant id is required'); - } - if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().set( - ledgerKey(grant.grant_id), - JSON.stringify(recordFromGrant(grant)), - 'EX', - ttlSeconds(grant.exp), - ); +export async function createEgressLedger( + grant: EgressGrantClaims, +): Promise { + if (!grant.grant_id) { + throw new EgressGrantError('malformed', 'Egress grant id is required'); + } + if (!env.EGRESS_LEDGER_REQUIRED) return; + await redisConnection().set( + ledgerKey(grant.grant_id), + JSON.stringify(recordFromGrant(grant)), + 'EX', + ttlSeconds(grant.exp), + ); } -export async function ensureEgressLedger(grant: EgressGrantClaims): Promise { - if (!grant.grant_id) { - throw new EgressGrantError('malformed', 'Egress grant id is required'); - } - if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().set( - ledgerKey(grant.grant_id), - JSON.stringify(recordFromGrant(grant)), - 'EX', - ttlSeconds(grant.exp), - 'NX', - ); +export async function ensureEgressLedger( + grant: EgressGrantClaims, +): Promise { + if (!grant.grant_id) { + throw new EgressGrantError('malformed', 'Egress grant id is required'); + } + if (!env.EGRESS_LEDGER_REQUIRED) return; + await redisConnection().set( + ledgerKey(grant.grant_id), + JSON.stringify(recordFromGrant(grant)), + 'EX', + ttlSeconds(grant.exp), + 'NX', + ); } async function loadRecord(grantId: string): Promise { - const raw = await redisConnection().get(ledgerKey(grantId)); - if (!raw) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record is missing'); - } - return JSON.parse(raw) as EgressLedgerRecord; + const raw = await redisConnection().get(ledgerKey(grantId)); + if (!raw) { + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant ledger record is missing', + ); + } + return JSON.parse(raw) as EgressLedgerRecord; } -function assertActive(record: EgressLedgerRecord, grant: Pick): void { - if (record.grant_id !== grant.grant_id || record.exec_id !== grant.exec_id) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record does not match token'); - } - if (record.status !== 'active') { - throw new EgressGrantError('scope_mismatch', 'Egress grant has been revoked'); - } - if (record.exp <= Math.floor(Date.now() / 1000)) { - throw new EgressGrantError('expired', 'Egress grant is expired'); - } +function assertActive( + record: EgressLedgerRecord, + grant: Pick, +): void { + if ( + record.grant_id !== grant.grant_id || + record.exec_id !== grant.exec_id + ) { + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant ledger record does not match token', + ); + } + if (record.status !== 'active') { + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant has been revoked', + ); + } + if (record.exp <= Math.floor(Date.now() / 1000)) { + throw new EgressGrantError('expired', 'Egress grant is expired'); + } } async function mutateRecord( - grant: EgressGrantClaims, - mutate: (record: EgressLedgerRecord) => void, + grant: EgressGrantClaims, + mutate: (record: EgressLedgerRecord) => void, ): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) { - return recordFromGrant(grant); - } - const client = await dedicatedMutationConnection(); - const key = ledgerKey(grant.grant_id); - try { - for (let i = 0; i < LEDGER_MUTATION_ATTEMPTS; i++) { - await client.watch(key); - let record: EgressLedgerRecord; - try { - const raw = await client.get(key); - if (!raw) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record is missing'); - } - record = JSON.parse(raw) as EgressLedgerRecord; - assertActive(record, grant); - mutate(record); - if (record.request_count > record.max_requests) { - throw new EgressGrantError('scope_mismatch', 'Egress grant request budget exceeded'); + if (!env.EGRESS_LEDGER_REQUIRED) { + return recordFromGrant(grant); + } + const client = await dedicatedMutationConnection(); + const key = ledgerKey(grant.grant_id); + try { + for (let i = 0; i < LEDGER_MUTATION_ATTEMPTS; i++) { + await client.watch(key); + let record: EgressLedgerRecord; + try { + const raw = await client.get(key); + if (!raw) { + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant ledger record is missing', + ); + } + record = JSON.parse(raw) as EgressLedgerRecord; + assertActive(record, grant); + mutate(record); + if (record.request_count > record.max_requests) { + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant request budget exceeded', + ); + } + } catch (error) { + await client.unwatch().catch(unwatchError => { + logger.warn( + 'Failed to clear egress ledger WATCH after rejected mutation', + { error: unwatchError }, + ); + }); + throw error; + } + const result = await client + .multi() + .set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)) + .exec(); + if (result) return record; + if (i < LEDGER_MUTATION_ATTEMPTS - 1) { + await sleep(Math.min(25, i + 1)); + } } - } catch (error) { - await client.unwatch().catch(unwatchError => { - logger.warn('Failed to clear egress ledger WATCH after rejected mutation', { error: unwatchError }); + } finally { + await client.unwatch().catch(error => { + logger.warn( + 'Failed to clear egress ledger WATCH before returning mutation connection', + { error }, + ); }); - throw error; - } - const result = await client.multi() - .set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)) - .exec(); - if (result) return record; - if (i < LEDGER_MUTATION_ATTEMPTS - 1) { - await sleep(Math.min(25, i + 1)); - } + releaseMutationConnection(client); } - } finally { - await client.unwatch().catch(error => { - logger.warn('Failed to clear egress ledger WATCH before returning mutation connection', { error }); - }); - releaseMutationConnection(client); - } - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger update conflicted'); + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant ledger update conflicted', + ); } -export async function assertEgressGrantActive(grant: EgressGrantClaims): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) return recordFromGrant(grant); - const record = await loadRecord(grant.grant_id); - assertActive(record, grant); - return record; +export async function assertEgressGrantActive( + grant: EgressGrantClaims, +): Promise { + if (!env.EGRESS_LEDGER_REQUIRED) return recordFromGrant(grant); + const record = await loadRecord(grant.grant_id); + assertActive(record, grant); + return record; } -export async function recordEgressRead(grant: EgressGrantClaims): Promise { - await mutateRecord(grant, record => { - record.request_count += 1; - record.read_count += 1; - }); +export async function recordEgressRead( + grant: EgressGrantClaims, +): Promise { + await mutateRecord(grant, record => { + record.request_count += 1; + record.read_count += 1; + }); } export async function reserveEgressUpload(args: { - grant: EgressGrantClaims; - fileId: string; - bytes: number; + grant: EgressGrantClaims; + fileId: string; + bytes: number; }): Promise { - await mutateRecord(args.grant, record => { - if (args.bytes > Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES)) { - throw new EgressGrantError('scope_mismatch', 'Upload exceeds per-file egress byte limit'); - } - if (record.output_file_ids.includes(args.fileId)) { - throw new EgressGrantError('scope_mismatch', 'Output file id has already been used for this grant'); - } - if (record.output_file_ids.length >= record.max_output_files) { - throw new EgressGrantError('scope_mismatch', 'Output file count budget exceeded'); - } - const aggregateLimit = Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES) * record.max_output_files; - if (record.uploaded_bytes + args.bytes > aggregateLimit) { - throw new EgressGrantError('scope_mismatch', 'Aggregate upload byte budget exceeded'); - } - record.request_count += 1; - record.upload_count += 1; - record.uploaded_bytes += args.bytes; - record.output_file_ids.push(args.fileId); - }); + await mutateRecord(args.grant, record => { + if ( + args.bytes > + Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES) + ) { + throw new EgressGrantError( + 'scope_mismatch', + 'Upload exceeds per-file egress byte limit', + ); + } + if (record.output_file_ids.includes(args.fileId)) { + throw new EgressGrantError( + 'scope_mismatch', + 'Output file id has already been used for this grant', + ); + } + if (record.output_file_ids.length >= record.max_output_files) { + throw new EgressGrantError( + 'scope_mismatch', + 'Output file count budget exceeded', + ); + } + const aggregateLimit = + Math.min( + record.max_upload_bytes, + env.EGRESS_GATEWAY_MAX_FILE_BYTES, + ) * record.max_output_files; + if (record.uploaded_bytes + args.bytes > aggregateLimit) { + throw new EgressGrantError( + 'scope_mismatch', + 'Aggregate upload byte budget exceeded', + ); + } + record.request_count += 1; + record.upload_count += 1; + record.uploaded_bytes += args.bytes; + record.output_file_ids.push(args.fileId); + }); } export async function releaseEgressUpload(args: { - grant: EgressGrantClaims; - fileId: string; - bytes: number; + grant: EgressGrantClaims; + fileId: string; + bytes: number; }): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) return; - await mutateRecord(args.grant, record => { - record.uploaded_bytes = Math.max(0, record.uploaded_bytes - args.bytes); - record.upload_count = Math.max(0, record.upload_count - 1); - record.request_count = Math.max(0, record.request_count - 1); - record.output_file_ids = record.output_file_ids.filter(id => id !== args.fileId); - }); + if (!env.EGRESS_LEDGER_REQUIRED) return; + await mutateRecord(args.grant, record => { + record.uploaded_bytes = Math.max(0, record.uploaded_bytes - args.bytes); + record.upload_count = Math.max(0, record.upload_count - 1); + record.request_count = Math.max(0, record.request_count - 1); + record.output_file_ids = record.output_file_ids.filter( + id => id !== args.fileId, + ); + }); } -export async function recordEgressToolCall(grantId: string | undefined, executionId: string): Promise { - if (!env.EGRESS_LEDGER_REQUIRED || !grantId) return; - const grant = { grant_id: grantId, exec_id: executionId } as EgressGrantClaims; - await mutateRecord(grant, record => { - record.request_count += 1; - record.tool_call_count += 1; - }); +export async function recordEgressToolCall( + grantId: string | undefined, + executionId: string, +): Promise { + if (!env.EGRESS_LEDGER_REQUIRED || !grantId) return; + const grant = { + grant_id: grantId, + exec_id: executionId, + } as EgressGrantClaims; + await mutateRecord(grant, record => { + record.request_count += 1; + record.tool_call_count += 1; + }); } -export async function revokeEgressLedger(grantId: string, reason: string): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) return; - const key = ledgerKey(grantId); - const raw = await redisConnection().get(key); - if (!raw) return; - const record = JSON.parse(raw) as EgressLedgerRecord; - record.status = 'revoked'; - record.revoked_at = Math.floor(Date.now() / 1000); - record.revoke_reason = reason; - await redisConnection().set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)); +export async function revokeEgressLedger( + grantId: string, + reason: string, +): Promise { + if (!env.EGRESS_LEDGER_REQUIRED) return; + const key = ledgerKey(grantId); + const raw = await redisConnection().get(key); + if (!raw) return; + const record = JSON.parse(raw) as EgressLedgerRecord; + record.status = 'revoked'; + record.revoked_at = Math.floor(Date.now() / 1000); + record.revoke_reason = reason; + await redisConnection().set( + key, + JSON.stringify(record), + 'EX', + ttlSeconds(record.exp), + ); } diff --git a/service/src/file-server.ts b/service/src/file-server.ts index 6d226c0..a2dbcfd 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -1,6 +1,5 @@ import b from 'busboy'; import path from 'path'; -import IORedis from 'ioredis'; import express from 'express'; import { Client } from 'minio'; import { nanoid } from 'nanoid'; @@ -8,15 +7,17 @@ import { PassThrough } from 'stream'; import { pipeline } from 'stream/promises'; import type { BucketItem, BucketItemStat, ClientOptions } from 'minio'; import type { Readable } from 'stream'; -import type * as tls from 'tls'; import type * as t from './types'; import { metricsHandler, fileUploads, fileDownloads } from './metrics'; import { httpMetricsMiddleware } from './middleware/httpMetrics'; -import { internalServiceAuthEnabled, requireInternalServiceAuth } from './internal-service-auth'; +import { + internalServiceAuthEnabled, + requireInternalServiceAuth, +} from './internal-service-auth'; import { shutdownTelemetry, traceHttpRequest } from './telemetry'; import logger from './fileServerLogger'; import { env } from './config'; -import { redisKeepAliveOptions } from './redis-options'; +import { createRedisConnection } from './redis-connection'; const { INSTANCE_ID } = env; @@ -27,155 +28,162 @@ app.use(httpMetricsMiddleware); const bucketName = process.env.MINIO_BUCKET ?? 'test-bucket'; -type IamProviderModule = { IamAwsProvider?: new (opts: object) => unknown; default?: new (opts: object) => unknown }; +type IamProviderModule = { + IamAwsProvider?: new (opts: object) => unknown; + default?: new (opts: object) => unknown; +}; async function createMinioClient(): Promise { - const irsaExplicit = process.env.MINIO_USE_IRSA?.toLowerCase() === 'true'; - const irsaEnvVars = Boolean(process.env.AWS_WEB_IDENTITY_TOKEN_FILE) && Boolean(process.env.AWS_ROLE_ARN); - const useIrsa = irsaExplicit || irsaEnvVars; - - const baseConfig: ClientOptions = { - endPoint: process.env.MINIO_ENDPOINT ?? 'localhost', - port: process.env.MINIO_NO_PORT?.toLowerCase() === 'true' ? undefined : parseInt(process.env.MINIO_PORT ?? '9000'), - useSSL: process.env.MINIO_USE_SSL?.toLowerCase() === 'true', - region: process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1', - }; - - if (useIrsa) { - logger.info('Using IRSA (IamAwsProvider) for S3 authentication', { - tokenFile: process.env.AWS_WEB_IDENTITY_TOKEN_FILE, - roleArn: process.env.AWS_ROLE_ARN, - region: baseConfig.region, - }); + const irsaExplicit = process.env.MINIO_USE_IRSA?.toLowerCase() === 'true'; + const irsaEnvVars = + Boolean(process.env.AWS_WEB_IDENTITY_TOKEN_FILE) && + Boolean(process.env.AWS_ROLE_ARN); + const useIrsa = irsaExplicit || irsaEnvVars; + + const baseConfig: ClientOptions = { + endPoint: process.env.MINIO_ENDPOINT ?? 'localhost', + port: + process.env.MINIO_NO_PORT?.toLowerCase() === 'true' + ? undefined + : parseInt(process.env.MINIO_PORT ?? '9000'), + useSSL: process.env.MINIO_USE_SSL?.toLowerCase() === 'true', + region: + process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1', + }; - /** IamAwsProvider exists in minio 8.0.6+ but isn't exported from main module - * Try multiple import paths for compatibility with different runtimes (bun, ts-node, node) - */ - let IamAwsProviderClass: new (opts: object) => unknown; - try { - const mod = await import('minio/dist/main/IamAwsProvider.js') as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (primaryError) { - try { - // Fallback for bun: resolve path using require if available (CJS context) - let resolvePath = 'node_modules/minio/'; + if (useIrsa) { + logger.info('Using IRSA (IamAwsProvider) for S3 authentication', { + tokenFile: process.env.AWS_WEB_IDENTITY_TOKEN_FILE, + roleArn: process.env.AWS_ROLE_ARN, + region: baseConfig.region, + }); + + /** IamAwsProvider exists in minio 8.0.6+ but isn't exported from main module + * Try multiple import paths for compatibility with different runtimes (bun, ts-node, node) + */ + let IamAwsProviderClass: new (opts: object) => unknown; try { - // eslint-disable-next-line @typescript-eslint/no-require-imports - resolvePath = require.resolve('minio').replace(/dist\/.*$/, ''); - } catch { - // require.resolve not available (ESM context), use default path + const mod = + (await import('minio/dist/main/IamAwsProvider.js')) as IamProviderModule; + IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; + } catch (primaryError) { + try { + // Fallback for bun: resolve path using require if available (CJS context) + let resolvePath = 'node_modules/minio/'; + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + resolvePath = require + .resolve('minio') + .replace(/dist\/.*$/, ''); + } catch { + // require.resolve not available (ESM context), use default path + } + const mod = (await import( + `${resolvePath}dist/main/IamAwsProvider.js` + )) as IamProviderModule; + IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; + } catch (fallbackError) { + logger.error('Failed to load IamAwsProvider', { + primaryError, + fallbackError, + }); + throw new Error( + 'Could not load IamAwsProvider for IRSA authentication. Ensure minio >= 8.0.6 is installed.', + ); + } } - const mod = await import(`${resolvePath}dist/main/IamAwsProvider.js`) as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (fallbackError) { - logger.error('Failed to load IamAwsProvider', { primaryError, fallbackError }); - throw new Error('Could not load IamAwsProvider for IRSA authentication. Ensure minio >= 8.0.6 is installed.'); - } - } - const credentialsProvider = new IamAwsProviderClass({}); + const credentialsProvider = new IamAwsProviderClass({}); + return new Client({ + ...baseConfig, + credentialsProvider: + credentialsProvider as ClientOptions['credentialsProvider'], + }); + } + + logger.info('Using explicit credentials for MinIO/S3 authentication'); return new Client({ - ...baseConfig, - credentialsProvider: credentialsProvider as ClientOptions['credentialsProvider'], + ...baseConfig, + accessKey: process.env.MINIO_ACCESS_KEY ?? '', + secretKey: process.env.MINIO_SECRET_KEY ?? '', + sessionToken: process.env.MINIO_SESSION_TOKEN, }); - } - - logger.info('Using explicit credentials for MinIO/S3 authentication'); - return new Client({ - ...baseConfig, - accessKey: process.env.MINIO_ACCESS_KEY ?? '', - secretKey: process.env.MINIO_SECRET_KEY ?? '', - sessionToken: process.env.MINIO_SESSION_TOKEN, - }); } let minioClient: Client; let storageInitialized = false; -const useAltDnsLookup = process.env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP === 'true'; - -const redisClient = new IORedis({ - host: process.env.REDIS_HOST ?? 'redis', - port: Number(process.env.REDIS_PORT) || 6379, - password: process.env.REDIS_PASSWORD, - enableReadyCheck: false, - tls: process.env.REDIS_TLS === 'true' ? { - // For self-signed certificates - rejectUnauthorized: false - } as tls.ConnectionOptions : undefined, - connectTimeout: 10000, - ...redisKeepAliveOptions(), - maxRetriesPerRequest: 3, - retryStrategy(times: number): number { - const delay = Math.min(times * 500, 2000); - return delay; - }, - reconnectOnError(err: Error): boolean { - const targetError = 'READONLY'; - if (err.message.includes(targetError)) { - return true; - } - return false; - }, - // Alternative DNS lookup for AWS ElastiCache TLS connections - ...(useAltDnsLookup - ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } - : {}) +const redisClient = createRedisConnection({ + enableReadyCheck: false, + maxRetriesPerRequest: 3, + retryStrategy(times: number): number { + return Math.min(times * 500, 2000); + }, + reconnectOnError(err: Error): boolean { + return err.message.includes('READONLY'); + }, }); -redisClient.on('error', (err) => { - logger.error('Redis Client Error', { error: err }); +redisClient.on('error', err => { + logger.error('Redis Client Error', { error: err }); }); redisClient.on('connect', () => { - logger.info('Redis Client Connected', { - host: process.env.REDIS_HOST, - port: process.env.REDIS_PORT - }); + logger.info('Redis Client Connected', { + host: process.env.REDIS_HOST, + port: process.env.REDIS_PORT, + }); }); redisClient.on('ready', () => { - logger.info('Redis Client Ready'); + logger.info('Redis Client Ready'); }); -const minioRegion = process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1'; +const minioRegion = + process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1'; async function ensureBucketExists(retries = 10, delay = 1000): Promise { - for (let attempt = 1; attempt <= retries; attempt++) { - try { - const exists = await minioClient.bucketExists(bucketName); - if (exists) { - logger.info('Bucket already exists'); - return; - } - await minioClient.makeBucket(bucketName, minioRegion); - logger.info('Bucket created successfully'); - return; - } catch (err: unknown) { - const error = err as { code?: string; message?: string }; - if (error.code === 'BucketAlreadyOwnedByYou') { - logger.info('Bucket already exists'); - return; - } - - if (attempt < retries) { - const backoff = delay * Math.pow(2, attempt - 1); - logger.warn(`MinIO not ready, retrying in ${backoff}ms (attempt ${attempt}/${retries})`, { error: error.message }); - await new Promise(resolve => setTimeout(resolve, backoff)); - } else { - logger.error('Failed to ensure bucket exists after all retries', { error }); - throw err; - } + for (let attempt = 1; attempt <= retries; attempt++) { + try { + const exists = await minioClient.bucketExists(bucketName); + if (exists) { + logger.info('Bucket already exists'); + return; + } + await minioClient.makeBucket(bucketName, minioRegion); + logger.info('Bucket created successfully'); + return; + } catch (err: unknown) { + const error = err as { code?: string; message?: string }; + if (error.code === 'BucketAlreadyOwnedByYou') { + logger.info('Bucket already exists'); + return; + } + + if (attempt < retries) { + const backoff = delay * Math.pow(2, attempt - 1); + logger.warn( + `MinIO not ready, retrying in ${backoff}ms (attempt ${attempt}/${retries})`, + { error: error.message }, + ); + await new Promise(resolve => setTimeout(resolve, backoff)); + } else { + logger.error( + 'Failed to ensure bucket exists after all retries', + { error }, + ); + throw err; + } + } } - } } async function initializeStorage(): Promise { - minioClient = await createMinioClient(); - await ensureBucketExists(); - storageInitialized = true; - logger.info('Storage initialization complete'); + minioClient = await createMinioClient(); + await ensureBucketExists(); + storageInitialized = true; + logger.info('Storage initialization complete'); } /** @@ -186,243 +194,322 @@ async function initializeStorage(): Promise { * resolve with `empty: true`; non-empty streams resolve with a PassThrough * that replays the peeked first chunk and the rest of the upstream. */ -function peekStreamForEmpty(input: Readable): Promise<{ empty: true } | { empty: false; body: Readable }> { - return new Promise((resolve, reject) => { - let settled = false; - const cleanup = (): void => { - input.removeListener('data', onData); - input.removeListener('end', onEnd); - input.removeListener('error', onError); - }; - function onError(err: Error): void { - if (settled) return; - settled = true; - cleanup(); - reject(err); - } - function onEnd(): void { - if (settled) return; - settled = true; - cleanup(); - resolve({ empty: true }); - } - function onData(firstChunk: Buffer | string): void { - if (settled) return; - settled = true; - cleanup(); - input.pause(); - const passthrough = new PassThrough(); - const buf = Buffer.isBuffer(firstChunk) ? firstChunk : Buffer.from(firstChunk); - passthrough.write(buf); - input.pipe(passthrough); - input.once('error', (err) => passthrough.destroy(err)); - resolve({ empty: false, body: passthrough }); - } - input.once('data', onData); - input.once('end', onEnd); - input.once('error', onError); - }); +function peekStreamForEmpty( + input: Readable, +): Promise<{ empty: true } | { empty: false; body: Readable }> { + return new Promise((resolve, reject) => { + let settled = false; + const cleanup = (): void => { + input.removeListener('data', onData); + input.removeListener('end', onEnd); + input.removeListener('error', onError); + }; + function onError(err: Error): void { + if (settled) return; + settled = true; + cleanup(); + reject(err); + } + function onEnd(): void { + if (settled) return; + settled = true; + cleanup(); + resolve({ empty: true }); + } + function onData(firstChunk: Buffer | string): void { + if (settled) return; + settled = true; + cleanup(); + input.pause(); + const passthrough = new PassThrough(); + const buf = Buffer.isBuffer(firstChunk) + ? firstChunk + : Buffer.from(firstChunk); + passthrough.write(buf); + input.pipe(passthrough); + input.once('error', err => passthrough.destroy(err)); + resolve({ empty: false, body: passthrough }); + } + input.once('data', onData); + input.once('end', onEnd); + input.once('error', onError); + }); } async function uploadFile( - session_id: string, - fileStream: Readable, - filename: string, - mimetype: string, - existingFileId?: string, - readOnly = false, + session_id: string, + fileStream: Readable, + filename: string, + mimetype: string, + existingFileId?: string, + readOnly = false, ): Promise { - const fileId = existingFileId ?? nanoid(); - const fileExtension = path.extname(filename); - const objectName = `${session_id}/${fileId}${fileExtension}`; - - const encodedFilename = Buffer.from(filename).toString('base64'); - - /* `X-Amz-Meta-Read-Only: true` declares this file as infrastructure the - * uploader doesn't want surfaced as a generated artifact downstream - * (e.g. skill files primed by LibreChat). Stored as an MinIO/S3 user - * metadata header so it persists with the object and is exposed on - * `getObject` / `statObject` without a separate Redis lookup. */ - const metaData: Record = { - 'Content-Type': mimetype, - 'X-Amz-Meta-Original-Filename': encodedFilename, - 'X-Amz-Meta-Original-Filename-Encoded': 'base64', - }; - if (readOnly) { - metaData['X-Amz-Meta-Read-Only'] = 'true'; - } - - /* Note: this returns UploadedObjectInfo */ - const sessionKey = await redisClient.get(`session:${session_id}`); - const peeked = await peekStreamForEmpty(fileStream); - if (peeked.empty) { - /* Empty file: explicit single PUT with size=0 — multipart fails with - * "You must specify at least one part" on zero-byte streams. */ - await minioClient.putObject(bucketName, objectName, Buffer.alloc(0), 0, metaData); - } else { - await minioClient.putObject(bucketName, objectName, peeked.body, undefined, metaData); - } - logger.info(`[${INSTANCE_ID}] File ID: ${fileId} | Filename: ${filename} | Session key: ${sessionKey}`); - await redisClient.set(`upload:${sessionKey}${session_id}${fileId}`, 'true', 'EX', env.SESSION_CACHE_TTL); - fileUploads.inc(); - - return { - filename, - fileId, - }; + const fileId = existingFileId ?? nanoid(); + const fileExtension = path.extname(filename); + const objectName = `${session_id}/${fileId}${fileExtension}`; + + const encodedFilename = Buffer.from(filename).toString('base64'); + + /* `X-Amz-Meta-Read-Only: true` declares this file as infrastructure the + * uploader doesn't want surfaced as a generated artifact downstream + * (e.g. skill files primed by LibreChat). Stored as an MinIO/S3 user + * metadata header so it persists with the object and is exposed on + * `getObject` / `statObject` without a separate Redis lookup. */ + const metaData: Record = { + 'Content-Type': mimetype, + 'X-Amz-Meta-Original-Filename': encodedFilename, + 'X-Amz-Meta-Original-Filename-Encoded': 'base64', + }; + if (readOnly) { + metaData['X-Amz-Meta-Read-Only'] = 'true'; + } + + /* Note: this returns UploadedObjectInfo */ + const sessionKey = await redisClient.get(`session:${session_id}`); + const peeked = await peekStreamForEmpty(fileStream); + if (peeked.empty) { + /* Empty file: explicit single PUT with size=0 — multipart fails with + * "You must specify at least one part" on zero-byte streams. */ + await minioClient.putObject( + bucketName, + objectName, + Buffer.alloc(0), + 0, + metaData, + ); + } else { + await minioClient.putObject( + bucketName, + objectName, + peeked.body, + undefined, + metaData, + ); + } + logger.info( + `[${INSTANCE_ID}] File ID: ${fileId} | Filename: ${filename} | Session key: ${sessionKey}`, + ); + await redisClient.set( + `upload:${sessionKey}${session_id}${fileId}`, + 'true', + 'EX', + env.SESSION_CACHE_TTL, + ); + fileUploads.inc(); + + return { + filename, + fileId, + }; } app.get('/metrics', metricsHandler); app.get('/health', (_req: express.Request, res: express.Response) => { - res.status(200).json({ status: 'ok' }); + res.status(200).json({ status: 'ok' }); }); app.get('/ready', async (_req: express.Request, res: express.Response) => { - const checks: { redis?: string; s3?: string; storage?: string } = {}; - let healthy = true; - - if (!storageInitialized) { - checks.storage = 'initializing'; - healthy = false; - } - - try { - await redisClient.ping(); - checks.redis = 'ok'; - } catch (error) { - logger.error('Readiness check failed - Redis:', { error }); - checks.redis = 'error'; - healthy = false; - } - - if (storageInitialized) { + const checks: { redis?: string; s3?: string; storage?: string } = {}; + let healthy = true; + + if (!storageInitialized) { + checks.storage = 'initializing'; + healthy = false; + } + try { - await minioClient.bucketExists(bucketName); - checks.s3 = 'ok'; + await redisClient.ping(); + checks.redis = 'ok'; } catch (error) { - logger.error('Readiness check failed - S3:', { error }); - checks.s3 = 'error'; - healthy = false; + logger.error('Readiness check failed - Redis:', { error }); + checks.redis = 'error'; + healthy = false; + } + + if (storageInitialized) { + try { + await minioClient.bucketExists(bucketName); + checks.s3 = 'ok'; + } catch (error) { + logger.error('Readiness check failed - S3:', { error }); + checks.s3 = 'error'; + healthy = false; + } + } else { + checks.s3 = 'pending'; + } + + if (healthy) { + res.status(200).json({ status: 'ready', checks }); + } else { + res.status(503).json({ status: 'not ready', checks }); } - } else { - checks.s3 = 'pending'; - } - - if (healthy) { - res.status(200).json({ status: 'ready', checks }); - } else { - res.status(503).json({ status: 'not ready', checks }); - } }); if (!internalServiceAuthEnabled()) { - logger.warn('CODEAPI_INTERNAL_SERVICE_TOKEN is not set; file object routes are unauthenticated'); + logger.warn( + 'CODEAPI_INTERNAL_SERVICE_TOKEN is not set; file object routes are unauthenticated', + ); } app.use('/sessions', requireInternalServiceAuth); -app.post('/sessions/:session_id/objects', async (req: express.Request, res: express.Response) => { - const { session_id } = req.params; - /** Request-level X-Read-Only flag — applies to every file in this batch. - * See `uploadFile` for semantics (infrastructure inputs that callers - * should not surface as generated artifacts). */ - const readOnlyHeader = req.headers['x-read-only']; - const readOnly = typeof readOnlyHeader === 'string' && readOnlyHeader.toLowerCase() === 'true'; - /** busboy with proper charset handling and preservePath so subdirectory - * components survive (e.g. `pptx/editing.md`); default strips to basename. */ - const busboy = b({ - headers: req.headers, - defCharset: 'utf8', - defParamCharset: 'utf8', - preservePath: true, - }); - const uploadPromises: Promise[] = []; - - busboy.on('file', (fieldname: string, file: Readable, info: b.FileInfo) => { - const { filename: combinedFilename, encoding: _e, mimeType } = info; - - // Handle the filename properly - it might be URL encoded - let decodedFilename: string; - try { - decodedFilename = decodeURIComponent(combinedFilename); - } catch (err) { - // If decoding fails, use the original filename - logger.warn(`Failed to decode filename, using original: ${combinedFilename}`, { error: err }); - decodedFilename = combinedFilename; - } - - const [fileId, ...filenameParts] = decodedFilename.split('___'); - const filename = filenameParts.join('___'); - - logger.info(`[${INSTANCE_ID}] Processing file: ${filename} with ID: ${fileId}`); - - const uploadPromise = uploadFile(session_id, file, filename, mimeType, fileId, readOnly).catch(err => { - logger.error(`[${INSTANCE_ID}] Error uploading file ${filename}:`, { error: err }); - return null; - }); - uploadPromises.push(uploadPromise); - }); - - busboy.on('finish', async () => { - try { - const results = await Promise.all(uploadPromises); - const successfulUploads = results.filter((result): result is t.UploadResult => result !== null); - - logger.info(`[${INSTANCE_ID}] Successfully uploaded ${successfulUploads.length} files for session ${session_id}`); +app.post( + '/sessions/:session_id/objects', + async (req: express.Request, res: express.Response) => { + const { session_id } = req.params; + /** Request-level X-Read-Only flag — applies to every file in this batch. + * See `uploadFile` for semantics (infrastructure inputs that callers + * should not surface as generated artifacts). */ + const readOnlyHeader = req.headers['x-read-only']; + const readOnly = + typeof readOnlyHeader === 'string' && + readOnlyHeader.toLowerCase() === 'true'; + /** busboy with proper charset handling and preservePath so subdirectory + * components survive (e.g. `pptx/editing.md`); default strips to basename. */ + const busboy = b({ + headers: req.headers, + defCharset: 'utf8', + defParamCharset: 'utf8', + preservePath: true, + }); + const uploadPromises: Promise[] = []; + + busboy.on( + 'file', + (fieldname: string, file: Readable, info: b.FileInfo) => { + const { + filename: combinedFilename, + encoding: _e, + mimeType, + } = info; + + // Handle the filename properly - it might be URL encoded + let decodedFilename: string; + try { + decodedFilename = decodeURIComponent(combinedFilename); + } catch (err) { + // If decoding fails, use the original filename + logger.warn( + `Failed to decode filename, using original: ${combinedFilename}`, + { error: err }, + ); + decodedFilename = combinedFilename; + } + + const [fileId, ...filenameParts] = decodedFilename.split('___'); + const filename = filenameParts.join('___'); + + logger.info( + `[${INSTANCE_ID}] Processing file: ${filename} with ID: ${fileId}`, + ); + + const uploadPromise = uploadFile( + session_id, + file, + filename, + mimeType, + fileId, + readOnly, + ).catch(err => { + logger.error( + `[${INSTANCE_ID}] Error uploading file ${filename}:`, + { error: err }, + ); + return null; + }); + uploadPromises.push(uploadPromise); + }, + ); + + busboy.on('finish', async () => { + try { + const results = await Promise.all(uploadPromises); + const successfulUploads = results.filter( + (result): result is t.UploadResult => result !== null, + ); + + logger.info( + `[${INSTANCE_ID}] Successfully uploaded ${successfulUploads.length} files for session ${session_id}`, + ); + + return res.status(200).json({ + message: 'success', + storage_session_id: session_id, + files: successfulUploads, + }); + } catch (err) { + logger.error('Error processing uploads:', { error: err }); + return res.status(500).send('Error uploading files.'); + } + }); - return res.status(200).json({ - message: 'success', - storage_session_id: session_id, - files: successfulUploads - }); - } catch (err) { - logger.error('Error processing uploads:', { error: err }); - return res.status(500).send('Error uploading files.'); - } - }); + busboy.on('error', error => { + logger.error( + `[${INSTANCE_ID}] Busboy error for session_id ${session_id}:`, + error, + ); + res.status(500).json({ error: 'Error processing upload' }); + }); - busboy.on('error', (error) => { - logger.error(`[${INSTANCE_ID}] Busboy error for session_id ${session_id}:`, error); - res.status(500).json({ error: 'Error processing upload' }); - }); + await pipeline(req, busboy); + }, +); + +app.put( + '/sessions/:session_id/objects/:fileId', + async (req: express.Request, res: express.Response) => { + const { session_id, fileId } = req.params; + // Decode the filename from the header if it's URL encoded + const originalFilename = req.headers['x-original-filename'] as string; + let decodedFilename = ''; + + if (originalFilename) { + try { + decodedFilename = decodeURIComponent(originalFilename); + } catch (err) { + // If decoding fails, use the original filename + logger.warn( + `Failed to decode filename header, using original: ${originalFilename}`, + { error: err }, + ); + decodedFilename = originalFilename; + } + } - await pipeline(req, busboy); -}); + const mimeType = req.headers['content-type'] as string; + const readOnlyHeader = req.headers['x-read-only']; + const readOnly = + typeof readOnlyHeader === 'string' && + readOnlyHeader.toLowerCase() === 'true'; -app.put('/sessions/:session_id/objects/:fileId', async (req: express.Request, res: express.Response) => { - const { session_id, fileId } = req.params; - // Decode the filename from the header if it's URL encoded - const originalFilename = req.headers['x-original-filename'] as string; - let decodedFilename = ''; + if (!decodedFilename || !mimeType) { + return res.status(400).json({ error: 'Missing required headers' }); + } - if (originalFilename) { - try { - decodedFilename = decodeURIComponent(originalFilename); - } catch (err) { - // If decoding fails, use the original filename - logger.warn(`Failed to decode filename header, using original: ${originalFilename}`, { error: err }); - decodedFilename = originalFilename; - } - } - - const mimeType = req.headers['content-type'] as string; - const readOnlyHeader = req.headers['x-read-only']; - const readOnly = typeof readOnlyHeader === 'string' && readOnlyHeader.toLowerCase() === 'true'; - - if (!decodedFilename || !mimeType) { - return res.status(400).json({ error: 'Missing required headers' }); - } - - try { - const result = await uploadFile(session_id, req, decodedFilename, mimeType, fileId, readOnly); - logger.info(`[${INSTANCE_ID}] File uploaded successfully: ${result.filename}`); - return res.status(200).json(result); - } catch (err) { - logger.error(`[${INSTANCE_ID}] Error uploading file ${decodedFilename}:`, { error: err }); - return res.status(500).json({ error: 'Error uploading file.' }); - } -}); + try { + const result = await uploadFile( + session_id, + req, + decodedFilename, + mimeType, + fileId, + readOnly, + ); + logger.info( + `[${INSTANCE_ID}] File uploaded successfully: ${result.filename}`, + ); + return res.status(200).json(result); + } catch (err) { + logger.error( + `[${INSTANCE_ID}] Error uploading file ${decodedFilename}:`, + { error: err }, + ); + return res.status(500).json({ error: 'Error uploading file.' }); + } + }, +); /** * Single-object metadata lookup. Returns the JSON shape callers @@ -434,289 +521,371 @@ app.put('/sessions/:session_id/objects/:fileId', async (req: express.Request, re * service-api expose only the metadata variant under sessionAuth * without conflating with the internal-only binary download. */ -app.get('/sessions/:session_id/objects/:objectId/metadata', async (req, res) => { - const { session_id, objectId } = req.params; - - try { - const stream = minioClient.listObjects(bucketName, `${session_id}/${objectId}`, true); - let objectName = ''; - - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { - objectName = obj.name; - break; - } - } +app.get( + '/sessions/:session_id/objects/:objectId/metadata', + async (req, res) => { + const { session_id, objectId } = req.params; - if (!objectName) { - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found', - session_id, - objectId, - }); - } - - const stat: Partial = await minioClient.statObject(bucketName, objectName); - const originalFilename = decodeOriginalFilename(stat.metaData, path.basename(objectName)); - - return res.status(200).json({ - name: objectName, - originalFilename, - size: stat.size, - lastModified: stat.lastModified, - etag: stat.etag, - contentType: stat.metaData?.['content-type'] ?? 'application/octet-stream', - readOnly: stat.metaData?.['read-only'] === 'true', - }); - } catch (err) { - logger.error('Error fetching object metadata:', { error: err, session_id, objectId, bucketName }); - return res.status(500).json({ - error: 'Error fetching object metadata', - details: (err as Error | undefined)?.message, - }); - } -}); + try { + const stream = minioClient.listObjects( + bucketName, + `${session_id}/${objectId}`, + true, + ); + let objectName = ''; + + for await (const obj of stream) { + if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { + objectName = obj.name; + break; + } + } + + if (!objectName) { + return res.status(404).json({ + error: 'File not found', + details: 'No matching file found', + session_id, + objectId, + }); + } + + const stat: Partial = await minioClient.statObject( + bucketName, + objectName, + ); + const originalFilename = decodeOriginalFilename( + stat.metaData, + path.basename(objectName), + ); + + return res.status(200).json({ + name: objectName, + originalFilename, + size: stat.size, + lastModified: stat.lastModified, + etag: stat.etag, + contentType: + stat.metaData?.['content-type'] ?? + 'application/octet-stream', + readOnly: stat.metaData?.['read-only'] === 'true', + }); + } catch (err) { + logger.error('Error fetching object metadata:', { + error: err, + session_id, + objectId, + bucketName, + }); + return res.status(500).json({ + error: 'Error fetching object metadata', + details: (err as Error | undefined)?.message, + }); + } + }, +); app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { - const { session_id, objectId } = req.params; - - try { - // List objects to find the correct file with extension - const stream = minioClient.listObjects(bucketName, `${session_id}/${objectId}`, true); - let objectName = ''; - - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { - logger.warn('File not found', { session_id, objectId, bucketName }); - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found', - session_id, - objectId, - bucketName - }); - } + const { session_id, objectId } = req.params; - logger.info(`[${INSTANCE_ID}] Attempting to download: ${objectName}`); + try { + // List objects to find the correct file with extension + const stream = minioClient.listObjects( + bucketName, + `${session_id}/${objectId}`, + true, + ); + let objectName = ''; + + for await (const obj of stream) { + if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { + objectName = obj.name; + break; + } + } - const stat: Partial = await minioClient.statObject(bucketName, objectName); + if (!objectName) { + logger.warn('File not found', { session_id, objectId, bucketName }); + return res.status(404).json({ + error: 'File not found', + details: 'No matching file found', + session_id, + objectId, + bucketName, + }); + } - let originalFilename = path.basename(objectName); - if (stat.metaData?.['original-filename-encoded'] === 'base64' && stat.metaData['original-filename'] != null) { - try { - originalFilename = Buffer.from(stat.metaData['original-filename'], 'base64').toString('utf8'); - } catch (err) { - logger.warn('Failed to decode filename from metadata, using fallback', { error: err }); - originalFilename = stat.metaData['original-filename'] ?? path.basename(objectName); - } - } else if (stat.metaData?.['original-filename'] != null) { - originalFilename = stat.metaData['original-filename']; - } + logger.info(`[${INSTANCE_ID}] Attempting to download: ${objectName}`); + + const stat: Partial = await minioClient.statObject( + bucketName, + objectName, + ); + + let originalFilename = path.basename(objectName); + if ( + stat.metaData?.['original-filename-encoded'] === 'base64' && + stat.metaData['original-filename'] != null + ) { + try { + originalFilename = Buffer.from( + stat.metaData['original-filename'], + 'base64', + ).toString('utf8'); + } catch (err) { + logger.warn( + 'Failed to decode filename from metadata, using fallback', + { error: err }, + ); + originalFilename = + stat.metaData['original-filename'] ?? + path.basename(objectName); + } + } else if (stat.metaData?.['original-filename'] != null) { + originalFilename = stat.metaData['original-filename']; + } - logger.info(`[${INSTANCE_ID}] File found: ${objectName}`); + logger.info(`[${INSTANCE_ID}] File found: ${objectName}`); - // Explicitly remove problematic headers that might be duplicated - res.removeHeader('Transfer-Encoding'); - res.removeHeader('Date'); + // Explicitly remove problematic headers that might be duplicated + res.removeHeader('Transfer-Encoding'); + res.removeHeader('Date'); - const encodedFilename = encodeURIComponent(originalFilename); - res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${encodedFilename}`); - if (stat.metaData?.['content-type'] != null) { - res.setHeader('Content-Type', stat.metaData['content-type']); - } - /* Surface the read-only flag on download so the sandbox can plumb it - * onto its in-memory file metadata without a separate metadata fetch. - * MinIO normalizes `X-Amz-Meta-Read-Only` to `read-only` in stat.metaData. */ - if (stat.metaData?.['read-only'] === 'true') { - res.setHeader('X-Read-Only', 'true'); - } + const encodedFilename = encodeURIComponent(originalFilename); + res.setHeader( + 'Content-Disposition', + `attachment; filename*=UTF-8''${encodedFilename}`, + ); + if (stat.metaData?.['content-type'] != null) { + res.setHeader('Content-Type', stat.metaData['content-type']); + } + /* Surface the read-only flag on download so the sandbox can plumb it + * onto its in-memory file metadata without a separate metadata fetch. + * MinIO normalizes `X-Amz-Meta-Read-Only` to `read-only` in stat.metaData. */ + if (stat.metaData?.['read-only'] === 'true') { + res.setHeader('X-Read-Only', 'true'); + } - const dataStream = await minioClient.getObject(bucketName, objectName); - fileDownloads.inc(); + const dataStream = await minioClient.getObject(bucketName, objectName); + fileDownloads.inc(); - dataStream.on('data', (chunk) => { - res.write(chunk); - }); + dataStream.on('data', chunk => { + res.write(chunk); + }); - dataStream.on('end', () => { - res.end(); - }); + dataStream.on('end', () => { + res.end(); + }); - dataStream.on('error', (err) => { - logger.error('Error streaming file:', { error: err, session_id, objectId, bucketName }); - // Only send error if headers haven't been sent yet - if (!res.headersSent) { - res.status(500).json({ - error: 'Error streaming file', - details: err.message + dataStream.on('error', err => { + logger.error('Error streaming file:', { + error: err, + session_id, + objectId, + bucketName, + }); + // Only send error if headers haven't been sent yet + if (!res.headersSent) { + res.status(500).json({ + error: 'Error streaming file', + details: err.message, + }); + } else { + res.end(); + } }); - } else { - res.end(); - } - }); - } catch (err) { - logger.error('Error downloading file:', { error: err, session_id, objectId, bucketName }); - return res.status(500).json({ - error: 'Error downloading file', - details: (err as Error | undefined)?.message, - session_id, - objectId, - bucketName - }); - } + } catch (err) { + logger.error('Error downloading file:', { + error: err, + session_id, + objectId, + bucketName, + }); + return res.status(500).json({ + error: 'Error downloading file', + details: (err as Error | undefined)?.message, + session_id, + objectId, + bucketName, + }); + } }); /** * Decodes the original filename from metadata. * Handles both base64-encoded and plain text filenames for consistency. */ -function decodeOriginalFilename(metadata: Record | undefined, fallbackName: string): string { - if (!metadata) return fallbackName; +function decodeOriginalFilename( + metadata: Record | undefined, + fallbackName: string, +): string { + if (!metadata) return fallbackName; - const encodedFilename = metadata['original-filename']; - const encodingType = metadata['original-filename-encoded']; + const encodedFilename = metadata['original-filename']; + const encodingType = metadata['original-filename-encoded']; - if (encodedFilename && encodingType === 'base64') { - try { - return Buffer.from(encodedFilename, 'base64').toString('utf8'); - } catch { - return encodedFilename; + if (encodedFilename && encodingType === 'base64') { + try { + return Buffer.from(encodedFilename, 'base64').toString('utf8'); + } catch { + return encodedFilename; + } } - } - return encodedFilename || fallbackName; + return encodedFilename || fallbackName; } /** * Extracts session_id and file_id from object name (format: {session_id}/{file_id}.ext) */ -function parseObjectName(objectName: string | undefined): { session_id: string; file_id: string } | null { - if (objectName == null || objectName === '') return null; - const parts = objectName.split('/'); - if (parts.length < 2) return null; - const session_id = parts[0]; - const fileNameWithExt = parts[1]; - // Remove extension to get file_id - const file_id = fileNameWithExt.replace(/\.[^.]+$/, ''); - return { session_id, file_id }; +function parseObjectName( + objectName: string | undefined, +): { session_id: string; file_id: string } | null { + if (objectName == null || objectName === '') return null; + const parts = objectName.split('/'); + if (parts.length < 2) return null; + const session_id = parts[0]; + const fileNameWithExt = parts[1]; + // Remove extension to get file_id + const file_id = fileNameWithExt.replace(/\.[^.]+$/, ''); + return { session_id, file_id }; } -const detailLevels: Record Promise> | undefined> = { - simple: async (obj: BucketItem): Promise> => obj.name ?? '', - summary: async (obj: BucketItem): Promise> => ({ - name: obj.name, - size: obj.size, - lastModified: obj.lastModified, - etag: obj.etag - }), - full: async (obj: BucketItem): Promise> => { - const stat = await minioClient.statObject(bucketName, obj.name ?? ''); - // Decode original filename for consistent plain text response - const originalFilename = decodeOriginalFilename(stat.metaData, path.basename(obj.name ?? '')); - return { - name: obj.name, - size: obj.size, - lastModified: obj.lastModified, - etag: obj.etag, - metadata: { - ...stat.metaData, - // Provide decoded filename for client convenience (standardized) - 'original-filename': originalFilename, - 'original-filename-encoded': 'none' // Indicate it's already decoded - }, - versionId: stat.versionId, - contentType: stat.metaData['content-type'] ?? 'application/octet-stream' - }; - }, - // New normalized detail level - returns self-contained file references - // Ideal for clients that need to pass files to subsequent requests - normalized: async (obj: BucketItem): Promise> => { - const stat = await minioClient.statObject(bucketName, obj.name ?? ''); - const originalFilename = decodeOriginalFilename(stat.metaData, path.basename(obj.name ?? '')); - const parsed = parseObjectName(obj.name); - - const result: Record = { - id: parsed?.file_id ?? path.basename(obj.name ?? '').replace(/\.[^.]+$/, ''), - name: originalFilename, - storage_session_id: parsed?.session_id ?? '', - size: obj.size, - contentType: stat.metaData['content-type'] ?? 'application/octet-stream', - lastModified: obj.lastModified, - }; - if (stat.metaData['read-only'] === 'true') { - result.read_only = true; - } - return result; - } +const detailLevels: Record< + t.DetailLevel | string, + ( + obj: BucketItem, + ) => Promise> | undefined +> = { + simple: async (obj: BucketItem): Promise> => + obj.name ?? '', + summary: async (obj: BucketItem): Promise> => ({ + name: obj.name, + size: obj.size, + lastModified: obj.lastModified, + etag: obj.etag, + }), + full: async (obj: BucketItem): Promise> => { + const stat = await minioClient.statObject(bucketName, obj.name ?? ''); + // Decode original filename for consistent plain text response + const originalFilename = decodeOriginalFilename( + stat.metaData, + path.basename(obj.name ?? ''), + ); + return { + name: obj.name, + size: obj.size, + lastModified: obj.lastModified, + etag: obj.etag, + metadata: { + ...stat.metaData, + // Provide decoded filename for client convenience (standardized) + 'original-filename': originalFilename, + 'original-filename-encoded': 'none', // Indicate it's already decoded + }, + versionId: stat.versionId, + contentType: + stat.metaData['content-type'] ?? 'application/octet-stream', + }; + }, + // New normalized detail level - returns self-contained file references + // Ideal for clients that need to pass files to subsequent requests + normalized: async (obj: BucketItem): Promise> => { + const stat = await minioClient.statObject(bucketName, obj.name ?? ''); + const originalFilename = decodeOriginalFilename( + stat.metaData, + path.basename(obj.name ?? ''), + ); + const parsed = parseObjectName(obj.name); + + const result: Record = { + id: + parsed?.file_id ?? + path.basename(obj.name ?? '').replace(/\.[^.]+$/, ''), + name: originalFilename, + storage_session_id: parsed?.session_id ?? '', + size: obj.size, + contentType: + stat.metaData['content-type'] ?? 'application/octet-stream', + lastModified: obj.lastModified, + }; + if (stat.metaData['read-only'] === 'true') { + result.read_only = true; + } + return result; + }, }; app.get('/sessions/:session_id/objects', async (req, res) => { - const { session_id } = req.params; - const { detail = 'simple' } = req.query; + const { session_id } = req.params; + const { detail = 'simple' } = req.query; - try { - const stream = minioClient.listObjects(bucketName, session_id, true); - const objects: (t.ObjectTypes | Partial | undefined)[] = []; + try { + const stream = minioClient.listObjects(bucketName, session_id, true); + const objects: (t.ObjectTypes | Partial | undefined)[] = + []; - const getDetail = detailLevels[detail as string] ?? detailLevels.simple; + const getDetail = detailLevels[detail as string] ?? detailLevels.simple; - for await (const obj of stream) { - objects.push(await getDetail(obj)); - } + for await (const obj of stream) { + objects.push(await getDetail(obj)); + } - res.json(objects); - } catch (err) { - logger.error('Error listing objects:', { error: err, session_id }); - return res.status(500).send('Error listing objects'); - } + res.json(objects); + } catch (err) { + logger.error('Error listing objects:', { error: err, session_id }); + return res.status(500).send('Error listing objects'); + } }); app.delete('/sessions/:session_id/objects/:fileId', async (req, res) => { - const { session_id, fileId } = req.params; + const { session_id, fileId } = req.params; - try { - const stream = minioClient.listObjects(bucketName, `${session_id}/${fileId}`, true); - let objectName = ''; + try { + const stream = minioClient.listObjects( + bucketName, + `${session_id}/${fileId}`, + true, + ); + let objectName = ''; + + for await (const obj of stream) { + if (obj.name.startsWith(`${session_id}/${fileId}`) === true) { + objectName = obj.name; + break; + } + } - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${fileId}`) === true) { - objectName = obj.name; - break; - } - } + if (!objectName) { + logger.warn('File not found for deletion', { + session_id, + fileId, + bucketName, + }); + return res.status(404).json({ + error: 'File not found', + details: 'No matching file found for deletion', + session_id, + fileId, + bucketName, + }); + } - if (!objectName) { - logger.warn('File not found for deletion', { session_id, fileId, bucketName }); - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found for deletion', - session_id, - fileId, - bucketName - }); + await minioClient.removeObject(bucketName, objectName); + logger.info( + `[${INSTANCE_ID}] File deleted successfully: ${objectName}`, + ); + return res.status(200).json({ + message: 'File deleted successfully', + session_id, + fileId, + }); + } catch (err) { + logger.error('Error deleting file:', err); + return res.status(500).json({ + error: 'Error deleting file', + }); } - - await minioClient.removeObject(bucketName, objectName); - logger.info(`[${INSTANCE_ID}] File deleted successfully: ${objectName}`); - return res.status(200).json({ - message: 'File deleted successfully', - session_id, - fileId - }); - - } catch (err) { - logger.error('Error deleting file:', err); - return res.status(500).json({ - error: 'Error deleting file', - }); - } }); const port = Number(process.env.FILE_SERVER_PORT ?? 3000); @@ -729,50 +898,58 @@ let server: ReturnType | undefined; let shuttingDown = false; async function startServer(): Promise { - try { - await initializeStorage(); - const onListen = () => { - logger.info(`[${INSTANCE_ID}] Server running on ${host ?? '*'}:${port}`); - }; - server = host ? app.listen(port, host, onListen) : app.listen(port, onListen); - } catch (err) { - logger.error('Critical: Could not initialize storage', { error: err }); - process.exit(1); - } + try { + await initializeStorage(); + const onListen = () => { + logger.info( + `[${INSTANCE_ID}] Server running on ${host ?? '*'}:${port}`, + ); + }; + server = host + ? app.listen(port, host, onListen) + : app.listen(port, onListen); + } catch (err) { + logger.error('Critical: Could not initialize storage', { error: err }); + process.exit(1); + } } function closeHttpServer(): Promise { - if (!server) return Promise.resolve(); - return new Promise((resolve, reject) => { - server?.close((error) => { - if (error) reject(error); - else resolve(); + if (!server) return Promise.resolve(); + return new Promise((resolve, reject) => { + server?.close(error => { + if (error) reject(error); + else resolve(); + }); }); - }); } async function shutdown(): Promise { - if (shuttingDown) return; - shuttingDown = true; - logger.info(`[${INSTANCE_ID}] Shutting down file server...`); - try { - await closeHttpServer(); - await redisClient.quit(); - try { - await shutdownTelemetry(); - } catch (telemetryError) { - logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { error: telemetryError }); - } - process.exit(0); - } catch (error) { - logger.error(`[${INSTANCE_ID}] File server shutdown failed`, { error }); + if (shuttingDown) return; + shuttingDown = true; + logger.info(`[${INSTANCE_ID}] Shutting down file server...`); try { - await shutdownTelemetry(); - } catch (telemetryError) { - logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { error: telemetryError }); + await closeHttpServer(); + await redisClient.quit(); + try { + await shutdownTelemetry(); + } catch (telemetryError) { + logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { + error: telemetryError, + }); + } + process.exit(0); + } catch (error) { + logger.error(`[${INSTANCE_ID}] File server shutdown failed`, { error }); + try { + await shutdownTelemetry(); + } catch (telemetryError) { + logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { + error: telemetryError, + }); + } + process.exit(1); } - process.exit(1); - } } startServer(); @@ -780,10 +957,10 @@ startServer(); process.on('SIGTERM', () => void shutdown()); process.on('SIGINT', () => void shutdown()); -process.on('uncaughtException', (error) => { - logger.error('Uncaught Exception', { error }); +process.on('uncaughtException', error => { + logger.error('Uncaught Exception', { error }); }); process.on('unhandledRejection', (reason, promise) => { - logger.error('Unhandled Rejection', { reason, promise }); + logger.error('Unhandled Rejection', { reason, promise }); }); diff --git a/service/src/middleware/limits.ts b/service/src/middleware/limits.ts index b16ed19..92d5726 100644 --- a/service/src/middleware/limits.ts +++ b/service/src/middleware/limits.ts @@ -11,194 +11,224 @@ import { getExecutionIdentity } from '../execution-identity'; import logger from '../logger'; type RedisCommandTarget = { - call?: (command: string, ...args: (string | number | Buffer)[]) => Promise; + call?: ( + command: string, + ...args: (string | number | Buffer)[] + ) => Promise; }; let redisCommands: RedisCommandTarget | undefined; async function getRedisCommands(): Promise { - if (redisCommands) return redisCommands; - const { connection } = await import('../queue'); - return connection; + if (redisCommands) return redisCommands; + const { connection } = await import('../queue'); + return connection; } -const sendCommand: SendCommandFn = async (command: string, ...args: (string | number | Buffer)[]): Promise => { - const target = await getRedisCommands(); - if (typeof target.call === 'function') { - const result = await target.call(command, ...args); +const sendCommand: SendCommandFn = async ( + command: string, + ...args: (string | number | Buffer)[] +): Promise => { + const target = await getRedisCommands(); + const commandMethod = (target as Record)[ + command.toLowerCase() + ]; + if (typeof commandMethod !== 'function') { + if (typeof target.call === 'function') { + const result = await target.call(command, ...args); + return result as RedisReply; + } + throw new Error(`Redis command method unavailable: ${command}`); + } + const result = await commandMethod.apply(target, args); return result as RedisReply; - } - const commandMethod = (target as Record)[command.toLowerCase()]; - if (typeof commandMethod !== 'function') { - throw new Error(`Redis command method unavailable: ${command}`); - } - const result = await commandMethod.apply(target, args); - return result as RedisReply; }; type RequestWithRateLimit = Request & { - rateLimit?: { - limit: number; - used: number; - remaining: number; - resetTime?: Date; - }; + rateLimit?: { + limit: number; + used: number; + remaining: number; + resetTime?: Date; + }; }; type RateLimitResponseOptions = { - message: string; - structuredBody?: boolean; - logRejections?: boolean; + message: string; + structuredBody?: boolean; + logRejections?: boolean; }; const unknownPrincipal = 'unknown'; -const keySegment = (value: string | undefined, fallback = unknownPrincipal): string => { - const trimmed = value?.trim(); - return trimmed ? trimmed.replace(/:/g, '_') : fallback; +const keySegment = ( + value: string | undefined, + fallback = unknownPrincipal, +): string => { + const trimmed = value?.trim(); + return trimmed ? trimmed.replace(/:/g, '_') : fallback; }; const hashLabel = (value: string | undefined): string | undefined => { - const trimmed = value?.trim(); - if (!trimmed) return undefined; - return createHash('sha256').update(trimmed).digest('hex').slice(0, 12); + const trimmed = value?.trim(); + if (!trimmed) return undefined; + return createHash('sha256').update(trimmed).digest('hex').slice(0, 12); }; export function setRateLimitRedisForTests(client?: RedisCommandTarget): void { - redisCommands = client; + redisCommands = client; } export function retryAfterSeconds(req: Request, windowMs: number): number { - const resetTime = (req as RequestWithRateLimit).rateLimit?.resetTime; - if (resetTime instanceof Date) { - return Math.max(1, Math.ceil((resetTime.getTime() - Date.now()) / 1000)); - } - return Math.max(1, Math.ceil(windowMs / 1000)); + const resetTime = (req as RequestWithRateLimit).rateLimit?.resetTime; + if (resetTime instanceof Date) { + return Math.max( + 1, + Math.ceil((resetTime.getTime() - Date.now()) / 1000), + ); + } + return Math.max(1, Math.ceil(windowMs / 1000)); } -export function rateLimitResponseBody(message: string, retryAfter: number): { - error: 'rate_limited'; - message: string; - retry_after_seconds: number; +export function rateLimitResponseBody( + message: string, + retryAfter: number, +): { + error: 'rate_limited'; + message: string; + retry_after_seconds: number; } { - const retryAfterLabel = retryAfter === 1 ? '1 second' : `${retryAfter} seconds`; - return { - error: 'rate_limited', - message: `${message} Please retry in ${retryAfterLabel}.`, - retry_after_seconds: retryAfter, - }; + const retryAfterLabel = + retryAfter === 1 ? '1 second' : `${retryAfter} seconds`; + return { + error: 'rate_limited', + message: `${message} Please retry in ${retryAfterLabel}.`, + retry_after_seconds: retryAfter, + }; } export const keyGenerator = (req: Request): string => { - const authReq = req as AuthenticatedRequest; - const identity = getExecutionIdentity(authReq); - if (identity.canonicalUserId) { - return `${keySegment(identity.storageNamespace, 'legacy')}:user:${keySegment(identity.canonicalUserId)}`; - } - return `ip:${keySegment(req.ip)}`; + const authReq = req as AuthenticatedRequest; + const identity = getExecutionIdentity(authReq); + if (identity.canonicalUserId) { + return `${keySegment(identity.storageNamespace, 'legacy')}:user:${keySegment(identity.canonicalUserId)}`; + } + return `ip:${keySegment(req.ip)}`; }; const buildRateLimiter = ( - prefix: string, - windowMs: number, - max: number, - options: RateLimitResponseOptions + prefix: string, + windowMs: number, + max: number, + options: RateLimitResponseOptions, ): RateLimitRequestHandler => { - return rateLimitFactory({ - windowMs, - max, - standardHeaders: 'draft-6', - legacyHeaders: false, - store: new RateLimitRedisStore({ - sendCommand, - prefix: `${prefix}:` - }), - keyGenerator, - handler: (req: Request, res: Response) => { - const retryAfter = retryAfterSeconds(req, windowMs); - const rateLimit = (req as RequestWithRateLimit).rateLimit; - res.setHeader('Retry-After', retryAfter.toString()); - res.setHeader('RateLimit-Limit', (rateLimit?.limit ?? max).toString()); - res.setHeader('RateLimit-Remaining', '0'); - res.setHeader('RateLimit-Reset', retryAfter.toString()); - - if (options.logRejections) { - const authReq = req as AuthenticatedRequest; - const principal = authReq.codeApiPrincipal; - const identity = getExecutionIdentity(authReq); - const hasIdentity = Boolean(identity.canonicalUserId); - logger.warn('CodeAPI rate limit rejected', { - limiter: prefix, - path: req.originalUrl || req.path, - retryAfterSeconds: retryAfter, - limit: rateLimit?.limit ?? max, - windowMs, - principalSource: hasIdentity ? identity.principalSource : undefined, - tenantHash: hasIdentity ? hashLabel(identity.storageNamespace) : undefined, - userHash: hasIdentity ? hashLabel(identity.canonicalUserId) : undefined, - credentialHash: hashLabel(principal?.credentialId), - }); - } - - if (options.structuredBody) { - return res.status(429).json(rateLimitResponseBody(options.message, retryAfter)); - } - const retryAfterLabel = retryAfter === 1 ? '1 second' : `${retryAfter} seconds`; - res.status(429).json({ - error: `${options.message} Please retry in ${retryAfterLabel}.`, - }); - }, - }); + return rateLimitFactory({ + windowMs, + max, + standardHeaders: 'draft-6', + legacyHeaders: false, + store: new RateLimitRedisStore({ + sendCommand, + prefix: `${prefix}:`, + }), + keyGenerator, + handler: (req: Request, res: Response) => { + const retryAfter = retryAfterSeconds(req, windowMs); + const rateLimit = (req as RequestWithRateLimit).rateLimit; + res.setHeader('Retry-After', retryAfter.toString()); + res.setHeader( + 'RateLimit-Limit', + (rateLimit?.limit ?? max).toString(), + ); + res.setHeader('RateLimit-Remaining', '0'); + res.setHeader('RateLimit-Reset', retryAfter.toString()); + + if (options.logRejections) { + const authReq = req as AuthenticatedRequest; + const principal = authReq.codeApiPrincipal; + const identity = getExecutionIdentity(authReq); + const hasIdentity = Boolean(identity.canonicalUserId); + logger.warn('CodeAPI rate limit rejected', { + limiter: prefix, + path: req.originalUrl || req.path, + retryAfterSeconds: retryAfter, + limit: rateLimit?.limit ?? max, + windowMs, + principalSource: hasIdentity + ? identity.principalSource + : undefined, + tenantHash: hasIdentity + ? hashLabel(identity.storageNamespace) + : undefined, + userHash: hasIdentity + ? hashLabel(identity.canonicalUserId) + : undefined, + credentialHash: hashLabel(principal?.credentialId), + }); + } + + if (options.structuredBody) { + return res + .status(429) + .json(rateLimitResponseBody(options.message, retryAfter)); + } + const retryAfterLabel = + retryAfter === 1 ? '1 second' : `${retryAfter} seconds`; + res.status(429).json({ + error: `${options.message} Please retry in ${retryAfterLabel}.`, + }); + }, + }); }; export const createRateLimiter = ( - prefix: string, - windowMs: number, - max: number, - options: RateLimitResponseOptions + prefix: string, + windowMs: number, + max: number, + options: RateLimitResponseOptions, ): RateLimitRequestHandler => { - let limiter: RateLimitRequestHandler | undefined; - const getLimiter = (): RateLimitRequestHandler => { - limiter ??= buildRateLimiter(prefix, windowMs, max, options); - return limiter; - }; - - const lazyLimiter = ((req: Request, res: Response, next: NextFunction) => { - return getLimiter()(req, res, next); - }) as RateLimitRequestHandler; - lazyLimiter.resetKey = (key: string) => getLimiter().resetKey(key); - lazyLimiter.getKey = (key: string) => getLimiter().getKey(key); - return lazyLimiter; + let limiter: RateLimitRequestHandler | undefined; + const getLimiter = (): RateLimitRequestHandler => { + limiter ??= buildRateLimiter(prefix, windowMs, max, options); + return limiter; + }; + + const lazyLimiter = ((req: Request, res: Response, next: NextFunction) => { + return getLimiter()(req, res, next); + }) as RateLimitRequestHandler; + lazyLimiter.resetKey = (key: string) => getLimiter().resetKey(key); + lazyLimiter.getKey = (key: string) => getLimiter().getKey(key); + return lazyLimiter; }; export const executionLimiter = createRateLimiter( - 'exec', - env.EXEC_LIMIT_WINDOW, - env.EXEC_MAX_REQUESTS, - { - message: 'Too many CodeAPI execution requests.', - structuredBody: true, - logRejections: true, - } + 'exec', + env.EXEC_LIMIT_WINDOW, + env.EXEC_MAX_REQUESTS, + { + message: 'Too many CodeAPI execution requests.', + structuredBody: true, + logRejections: true, + }, ); export const uploadLimiter = createRateLimiter( - 'upload', - env.UPLOAD_LIMIT_WINDOW, - env.UPLOAD_MAX_REQUESTS, - { message: 'Too many file uploads.' } + 'upload', + env.UPLOAD_LIMIT_WINDOW, + env.UPLOAD_MAX_REQUESTS, + { message: 'Too many file uploads.' }, ); export const downloadLimiter = createRateLimiter( - 'download', - env.DOWNLOAD_LIMIT_WINDOW, - env.DOWNLOAD_MAX_REQUESTS, - { message: 'Too many file downloads.' } + 'download', + env.DOWNLOAD_LIMIT_WINDOW, + env.DOWNLOAD_MAX_REQUESTS, + { message: 'Too many file downloads.' }, ); export const fetchLimiter = createRateLimiter( - 'fetch', - env.FETCH_LIMIT_WINDOW, - env.FETCH_MAX_REQUESTS, - { message: 'Too many file list requests.' } + 'fetch', + env.FETCH_LIMIT_WINDOW, + env.FETCH_MAX_REQUESTS, + { message: 'Too many file list requests.' }, ); diff --git a/service/src/queue.ts b/service/src/queue.ts index c6f3226..ea2b2ff 100644 --- a/service/src/queue.ts +++ b/service/src/queue.ts @@ -1,105 +1,114 @@ // src/queue.ts -import IORedis from 'ioredis'; import { Queue, QueueEvents } from 'bullmq'; import { setMaxListeners } from 'events'; import type { CommonRedisOptions } from 'ioredis'; -import type * as tls from 'tls'; import type * as t from './types'; import { Jobs, Queues } from './enum'; import { env } from './config'; import logger from './logger'; -import { redisKeepAliveOptions } from './redis-options'; -import { bullmqQueueJobs, registerBullmqQueueMetricsCollector } from './metrics'; +import { + bullmqQueueJobs, + registerBullmqQueueMetricsCollector, +} from './metrics'; +import { createRedisConnection, bullmqPrefix } from './redis-connection'; const MAX_RECONNECT_ATTEMPTS = 5; const RECONNECT_DELAY = 2000; -const retryStrategy: CommonRedisOptions['retryStrategy'] = (times) => { - if (times > MAX_RECONNECT_ATTEMPTS) { - logger.error(`Failed to connect to Redis after ${times} attempts`); - return null; - } - logger.warn(`Retrying Redis connection attempt ${times}`); - return RECONNECT_DELAY; +const retryStrategy: CommonRedisOptions['retryStrategy'] = times => { + if (times > MAX_RECONNECT_ATTEMPTS) { + logger.error(`Failed to connect to Redis after ${times} attempts`); + return null; + } + logger.warn(`Retrying Redis connection attempt ${times}`); + return RECONNECT_DELAY; }; -const reconnectOnError: CommonRedisOptions['reconnectOnError'] = (err) => { - logger.error('Redis connection error:', err); - const targetError = 'READONLY'; - if (err.message.includes(targetError)) { - return true; - } - return false; +const reconnectOnError: CommonRedisOptions['reconnectOnError'] = err => { + logger.error('Redis connection error:', err); + const targetError = 'READONLY'; + if (err.message.includes(targetError)) { + return true; + } + return false; }; -const connection = new IORedis({ - host: process.env.REDIS_HOST ?? 'redis', - port: Number(process.env.REDIS_PORT) || 6379, - password: process.env.REDIS_PASSWORD, - maxRetriesPerRequest: null, - retryStrategy, - reconnectOnError, - enableReadyCheck: true, - connectTimeout: 10000, - disconnectTimeout: 2000, - ...redisKeepAliveOptions(), - tls: process.env.REDIS_TLS === 'true' ? { - rejectUnauthorized: false - } as tls.ConnectionOptions : undefined, - // Alternative DNS lookup for AWS ElastiCache TLS connections - ...(env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP - ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } - : {}) +const connection = createRedisConnection({ + maxRetriesPerRequest: null, + retryStrategy, + reconnectOnError, + enableReadyCheck: true, + disconnectTimeout: 2000, }); +const prefix = bullmqPrefix(); + // Global queues - no INSTANCE_ID prefix // This enables horizontal scaling where any worker can process any job -const pyQueue = new Queue(Queues.python, { connection }); -const otherQueue = new Queue(Queues.other, { connection }); +const pyQueue = new Queue(Queues.python, { + connection, + prefix, +}); +const otherQueue = new Queue( + Queues.other, + { connection, prefix }, +); -const pyQueueEvents = new QueueEvents(Queues.python, { connection }); -const otherQueueEvents = new QueueEvents(Queues.other, { connection }); +const pyQueueEvents = new QueueEvents(Queues.python, { connection, prefix }); +const otherQueueEvents = new QueueEvents(Queues.other, { connection, prefix }); const queueMetricStates = ['waiting', 'active', 'delayed'] as const; const queueMetricSources = [ - { name: Queues.python, queue: pyQueue }, - { name: Queues.other, queue: otherQueue }, + { name: Queues.python, queue: pyQueue }, + { name: Queues.other, queue: otherQueue }, ] as const; const QUEUE_METRICS_TIMEOUT_MS = 1000; -async function withTimeout(promise: Promise, timeoutMs: number, message: string): Promise { - let timeout: ReturnType | undefined; - void promise.catch(() => undefined); - const timeoutPromise = new Promise((_, reject) => { - timeout = setTimeout(() => reject(new Error(message)), timeoutMs); - }); - try { - return await Promise.race([promise, timeoutPromise]); - } finally { - if (timeout !== undefined) { - clearTimeout(timeout); +async function withTimeout( + promise: Promise, + timeoutMs: number, + message: string, +): Promise { + let timeout: ReturnType | undefined; + void promise.catch(() => undefined); + const timeoutPromise = new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(message)), timeoutMs); + }); + try { + return await Promise.race([promise, timeoutPromise]); + } finally { + if (timeout !== undefined) { + clearTimeout(timeout); + } } - } } registerBullmqQueueMetricsCollector(async () => { - await Promise.all(queueMetricSources.map(async ({ name, queue }) => { - try { - const counts = await withTimeout( - queue.getJobCounts(...queueMetricStates), - QUEUE_METRICS_TIMEOUT_MS, - `Timed out collecting BullMQ queue metrics for ${name}`, - ); - for (const state of queueMetricStates) { - bullmqQueueJobs.set({ queue: name, state }, counts[state] ?? 0); - } - } catch (error) { - logger.warn('Failed to collect BullMQ queue metrics', { queue: name, error }); - for (const state of queueMetricStates) { - bullmqQueueJobs.remove({ queue: name, state }); - } - } - })); + await Promise.all( + queueMetricSources.map(async ({ name, queue }) => { + try { + const counts = await withTimeout( + queue.getJobCounts(...queueMetricStates), + QUEUE_METRICS_TIMEOUT_MS, + `Timed out collecting BullMQ queue metrics for ${name}`, + ); + for (const state of queueMetricStates) { + bullmqQueueJobs.set( + { queue: name, state }, + counts[state] ?? 0, + ); + } + } catch (error) { + logger.warn('Failed to collect BullMQ queue metrics', { + queue: name, + error, + }); + for (const state of queueMetricStates) { + bullmqQueueJobs.remove({ queue: name, state }); + } + } + }), + ); }); /* job.waitUntilFinished() attaches a short-lived `closing` listener to the diff --git a/service/src/redis-connection.test.ts b/service/src/redis-connection.test.ts new file mode 100644 index 0000000..4632c53 --- /dev/null +++ b/service/src/redis-connection.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, test, afterEach } from 'bun:test'; +import { + parseRedisNodes, + isClusterMode, + buildTlsOptions, + bullmqPrefix, +} from './redis-connection'; + +// ────────────────────────────────────────────────────────────────────────────── +// We test the pure helper functions from redis-connection.ts without +// establishing any real network connection. ioredis constructors are NOT +// called in these tests. +// ────────────────────────────────────────────────────────────────────────────── + +// Preserve original env so each test can start clean +const ORIGINAL_ENV: Record = {}; +for (const key of Object.keys(process.env)) { + if (key.startsWith('REDIS_') || key === 'USE_REDIS_CLUSTER') { + ORIGINAL_ENV[key] = process.env[key]; + } +} + +function resetEnv(overrides: Record = {}): void { + for (const key of Object.keys(process.env)) { + if (key.startsWith('REDIS_') || key === 'USE_REDIS_CLUSTER') { + delete process.env[key]; + } + } + for (const [k, v] of Object.entries(overrides)) { + if (v !== undefined) process.env[k] = v; + } +} + +afterEach(() => { + resetEnv(ORIGINAL_ENV); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// parseRedisNodes +// ────────────────────────────────────────────────────────────────────────────── + +describe('parseRedisNodes', () => { + test('single host uses REDIS_PORT default 6379', () => { + resetEnv({ REDIS_HOST: 'myhost' }); + expect(parseRedisNodes()).toEqual([{ host: 'myhost', port: 6379 }]); + }); + + test('single host with embedded port', () => { + resetEnv({ REDIS_HOST: 'myhost:6380' }); + expect(parseRedisNodes()).toEqual([{ host: 'myhost', port: 6380 }]); + }); + + test('comma-separated hosts without ports', () => { + resetEnv({ REDIS_HOST: 'node1,node2,node3', REDIS_PORT: '6380' }); + expect(parseRedisNodes()).toEqual([ + { host: 'node1', port: 6380 }, + { host: 'node2', port: 6380 }, + { host: 'node3', port: 6380 }, + ]); + }); + + test('comma-separated hosts with embedded ports', () => { + resetEnv({ REDIS_HOST: 'node1:6379,node2:6380,node3:6381' }); + expect(parseRedisNodes()).toEqual([ + { host: 'node1', port: 6379 }, + { host: 'node2', port: 6380 }, + { host: 'node3', port: 6381 }, + ]); + }); + + test('trims whitespace around entries', () => { + resetEnv({ REDIS_HOST: ' node1 , node2 ' }); + expect(parseRedisNodes()).toEqual([ + { host: 'node1', port: 6379 }, + { host: 'node2', port: 6379 }, + ]); + }); + + test('defaults to "redis" when REDIS_HOST is unset', () => { + resetEnv({}); + expect(parseRedisNodes()).toEqual([{ host: 'redis', port: 6379 }]); + }); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// isClusterMode +// ────────────────────────────────────────────────────────────────────────────── + +describe('isClusterMode', () => { + test('false by default', () => { + resetEnv({ REDIS_HOST: 'redis' }); + expect(isClusterMode()).toBe(false); + }); + + test('true when USE_REDIS_CLUSTER=true', () => { + resetEnv({ REDIS_HOST: 'redis', USE_REDIS_CLUSTER: 'true' }); + expect(isClusterMode()).toBe(true); + }); + + test('true when REDIS_HOST contains a comma', () => { + resetEnv({ REDIS_HOST: 'node1,node2' }); + expect(isClusterMode()).toBe(true); + }); + + test('false when REDIS_HOST is a single host', () => { + resetEnv({ + REDIS_HOST: 'single-host:6379', + USE_REDIS_CLUSTER: 'false', + }); + expect(isClusterMode()).toBe(false); + }); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// buildTlsOptions +// ────────────────────────────────────────────────────────────────────────────── + +describe('buildTlsOptions', () => { + test('returns undefined when neither REDIS_TLS nor REDIS_CA is set', () => { + resetEnv({}); + expect(buildTlsOptions()).toBeUndefined(); + }); + + test('returns { rejectUnauthorized: false } when REDIS_TLS=true and no CA', () => { + resetEnv({ REDIS_TLS: 'true' }); + expect(buildTlsOptions()).toEqual({ rejectUnauthorized: false }); + }); + + test('returns { ca } when REDIS_CA points to an existing file', () => { + const tmpFile = '/tmp/test-redis-ca.pem'; + const pem = + '-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n'; + const { writeFileSync, unlinkSync } = + require('fs') as typeof import('fs'); + writeFileSync(tmpFile, pem, 'utf8'); + + resetEnv({ REDIS_CA: tmpFile }); + try { + expect(buildTlsOptions()).toEqual({ ca: pem }); + } finally { + unlinkSync(tmpFile); + } + }); + + test('CA takes precedence over REDIS_TLS=true', () => { + const tmpFile = '/tmp/test-redis-ca2.pem'; + const pem = + '-----BEGIN CERTIFICATE-----\ndata\n-----END CERTIFICATE-----\n'; + const { writeFileSync, unlinkSync } = + require('fs') as typeof import('fs'); + writeFileSync(tmpFile, pem, 'utf8'); + + resetEnv({ REDIS_CA: tmpFile, REDIS_TLS: 'true' }); + try { + expect(buildTlsOptions()).toEqual({ ca: pem }); + } finally { + unlinkSync(tmpFile); + } + }); + + test('returns undefined when REDIS_CA file does not exist', () => { + resetEnv({ REDIS_CA: '/tmp/does-not-exist-ca-xyz.pem' }); + expect(buildTlsOptions()).toBeUndefined(); + }); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// bullmqPrefix +// ────────────────────────────────────────────────────────────────────────────── + +describe('bullmqPrefix', () => { + test('returns undefined in standalone mode', () => { + resetEnv({ REDIS_HOST: 'redis' }); + expect(bullmqPrefix()).toBeUndefined(); + }); + + test('returns "{codeapi}" in cluster mode via USE_REDIS_CLUSTER', () => { + resetEnv({ USE_REDIS_CLUSTER: 'true' }); + expect(bullmqPrefix()).toBe('{codeapi}'); + }); + + test('returns "{codeapi}" when REDIS_HOST contains multiple nodes', () => { + resetEnv({ REDIS_HOST: 'node1,node2,node3' }); + expect(bullmqPrefix()).toBe('{codeapi}'); + }); +}); diff --git a/service/src/redis-connection.ts b/service/src/redis-connection.ts new file mode 100644 index 0000000..5e88e31 --- /dev/null +++ b/service/src/redis-connection.ts @@ -0,0 +1,221 @@ +import { existsSync, readFileSync } from 'fs'; +import IORedis, { Cluster } from 'ioredis'; +import type { + Redis, + RedisOptions, + ClusterOptions, + CommonRedisOptions, +} from 'ioredis'; +import { redisKeepAliveOptions } from './redis-options'; +import logger from './logger'; + +export type RedisClient = Redis | Cluster; + +const MAX_CLUSTER_RECONNECT_ATTEMPTS = 5; + +/** + * Parse REDIS_HOST into an array of {host, port} startup nodes. + * Accepts comma-separated entries in "host:port" or plain "host" format. + * Falls back to REDIS_PORT (default 6379) when no port is embedded. + */ +export function parseRedisNodes(): Array<{ host: string; port: number }> { + const defaultPort = Number(process.env.REDIS_PORT) || 6379; + const raw = process.env.REDIS_HOST ?? 'redis'; + return raw.split(',').map(entry => { + const trimmed = entry.trim(); + const colonIdx = trimmed.lastIndexOf(':'); + if (colonIdx > 0) { + const potentialPort = Number(trimmed.slice(colonIdx + 1)); + if (Number.isInteger(potentialPort) && potentialPort > 0) { + return { + host: trimmed.slice(0, colonIdx), + port: potentialPort, + }; + } + } + return { host: trimmed, port: defaultPort }; + }); +} + +/** + * Returns true when Redis Cluster mode is active. + * Cluster mode is enabled when USE_REDIS_CLUSTER=true or REDIS_HOST contains + * a comma-separated list of nodes. + */ +export function isClusterMode(): boolean { + return ( + process.env.USE_REDIS_CLUSTER === 'true' || + (process.env.REDIS_HOST ?? '').includes(',') + ); +} + +/** + * Read and return the PEM-encoded CA certificate from the path given by the + * REDIS_CA environment variable. Returns null when the variable is unset or + * the file cannot be read. + */ +function readCACert(): string | null { + const caPath = process.env.REDIS_CA; + if (!caPath) return null; + try { + if (!existsSync(caPath)) { + logger.warn(`Redis CA certificate file not found: ${caPath}`); + return null; + } + return readFileSync(caPath, 'utf8'); + } catch (error) { + logger.error(`Failed to read Redis CA certificate: ${caPath}`, { + error, + }); + return null; + } +} + +/** + * Build the TLS options for an ioredis connection: + * - REDIS_CA points to a file → use that CA (certificate fully validated). + * - REDIS_TLS=true without a CA → disable certificate validation (backward-compatible). + * - Neither set → no TLS. + */ +export function buildTlsOptions(): Record | undefined { + const ca = readCACert(); + if (ca) return { ca }; + if (process.env.REDIS_TLS === 'true') return { rejectUnauthorized: false }; + return undefined; +} + +/** + * Returns the BullMQ `prefix` required in cluster mode so that all queue keys + * land in the same hash slot. Returns undefined in standalone mode (no prefix). + */ +export function bullmqPrefix(): string | undefined { + return isClusterMode() ? '{codeapi}' : undefined; +} + +/** + * Wrap an id in a Redis Cluster hash tag so every key built from it — no + * matter the prefix — hashes to the same slot. Required for any multi-key + * Lua script or MULTI/EXEC transaction that touches several keys sharing + * this id; harmless in standalone mode. Redis Cluster hashes only the + * substring between the first `{` and the following `}` in a key. + */ +export function hashTag(id: string): string { + return `{${id}}`; +} + +/** Reverse of `hashTag`: strips the surrounding `{}` if present, otherwise + * returns the input unchanged. Used to recover the raw id from a key that + * was built with `hashTag`. */ +export function stripHashTag(raw: string): string { + return raw.startsWith('{') && raw.endsWith('}') ? raw.slice(1, -1) : raw; +} + +/** Default cap on keys returned by `scanKeys` in a single call, bounding + * memory usage on a pathological keyspace. */ +export const SCAN_KEYS_DEFAULT_LIMIT = 10_000; + +/** Iterate matching Redis keys with SCAN instead of the blocking KEYS + * command. Stops collecting once `limit` keys have been gathered. In + * cluster mode, SCAN is issued against every master node individually + * because `Cluster` has no top-level `scanStream`; each node owns a + * disjoint set of hash slots so there are no duplicates across nodes. */ +export async function scanKeys( + client: RedisClient, + match: string, + count = 200, + limit = SCAN_KEYS_DEFAULT_LIMIT, +): Promise { + const out: string[] = []; + + const collect = async (node: Redis): Promise => { + const stream = node.scanStream({ match, count }); + for await (const batch of stream as AsyncIterable) { + for (const key of batch) { + out.push(key); + if (out.length >= limit) { + stream.destroy(); + logger.warn( + 'scanKeys hit limit; remaining keys deferred to next pass', + { match, limit }, + ); + return; + } + } + } + }; + + if (client instanceof Cluster) { + for (const node of client.nodes('master')) { + if (out.length >= limit) break; + await collect(node); + } + } else { + await collect(client); + } + + return out; +} + +type ConnectionOverrides = Partial< + Pick< + CommonRedisOptions, + | 'maxRetriesPerRequest' + | 'enableReadyCheck' + | 'retryStrategy' + | 'reconnectOnError' + | 'disconnectTimeout' + > +>; + +/** + * Create an ioredis client – standalone `Redis` or `Cluster` – based on the + * current environment configuration. + * + * Callers supply per-client `overrides` (retry strategy, maxRetriesPerRequest, + * enableReadyCheck, …) to preserve each component's existing connection semantics. + * + * In cluster mode the overrides are forwarded to `redisOptions` (applied + * per-node). The cluster-level reconnect loop is handled independently via + * `clusterRetryStrategy`. + */ +export function createRedisConnection( + overrides: ConnectionOverrides, +): RedisClient { + const tls = buildTlsOptions() as RedisOptions['tls']; + const dnsLookup: ClusterOptions['dnsLookup'] | undefined = + process.env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP === 'true' + ? (address, callback) => callback(null, address) + : undefined; + + const baseOptions: RedisOptions = { + password: process.env.REDIS_PASSWORD, + connectTimeout: 10000, + ...redisKeepAliveOptions(), + ...(tls !== undefined ? { tls } : {}), + ...(dnsLookup ? { dnsLookup } : {}), + ...overrides, + }; + + if (isClusterMode()) { + const nodes = parseRedisNodes(); + return new Cluster(nodes, { + ...(dnsLookup ? { dnsLookup } : {}), + redisOptions: baseOptions, + clusterRetryStrategy(times) { + if (times > MAX_CLUSTER_RECONNECT_ATTEMPTS) { + logger.error( + `Redis cluster giving up after ${times} reconnection attempts`, + ); + return null; + } + const base = Math.min(2 ** times * 100, 3000); + const jitter = Math.floor(Math.random() * Math.min(base, 1000)); + return Math.min(base + jitter, 3000); + }, + enableOfflineQueue: true, + }); + } + + const [{ host, port }] = parseRedisNodes(); + return new IORedis({ host, port, ...baseOptions }); +} diff --git a/service/src/runtime-session/registry.ts b/service/src/runtime-session/registry.ts index f8d6650..4fb90b2 100644 --- a/service/src/runtime-session/registry.ts +++ b/service/src/runtime-session/registry.ts @@ -1,10 +1,8 @@ import { nanoid } from 'nanoid'; import type { Redis } from 'ioredis'; +import type { RedisClient } from '../redis-connection'; import { connection } from '../queue'; -import { - env, - RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS, -} from '../config'; +import { env, RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS } from '../config'; import logger from '../logger'; export { RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS } from '../config'; @@ -24,36 +22,41 @@ export { RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS } from '../config'; * subset that ioredis-mock supports (see replay-state.ts). */ -export type RuntimeSessionState = 'PENDING' | 'RUNNING' | 'SUSPENDED' | 'TERMINATING' | 'TERMINATED'; +export type RuntimeSessionState = + | 'PENDING' + | 'RUNNING' + | 'SUSPENDED' + | 'TERMINATING' + | 'TERMINATED'; export interface RuntimeSessionRecord { - runtime_session_id: string; - tenant_id: string; - canonical_user_id: string; - microvm_id?: string; - endpoint?: string; - port?: number; - image_arn?: string; - image_version?: string; - /** Fingerprint of every immutable launch/security input. A deploy that - * changes one must not reuse a VM launched under the old policy. */ - launch_fingerprint?: string; - /** Exact provider idempotency token for this persisted launch intent. - * Optional so workers can safely replay records written before this field - * existed. */ - launch_client_token?: string; - /** Exact fingerprint of the RunMicrovm request paired with - * `launch_client_token`. Unlike `launch_fingerprint`, this preserves - * wire-level distinctions such as connector ordering for safe replay. */ - launch_request_fingerprint?: string; - state: RuntimeSessionState; - generation: number; - launched_at?: number; - last_seen_at: number; - hard_deadline_at?: number; - workspace_checkpoint?: string; - checkpointed_at?: number; - last_error?: string; + runtime_session_id: string; + tenant_id: string; + canonical_user_id: string; + microvm_id?: string; + endpoint?: string; + port?: number; + image_arn?: string; + image_version?: string; + /** Fingerprint of every immutable launch/security input. A deploy that + * changes one must not reuse a VM launched under the old policy. */ + launch_fingerprint?: string; + /** Exact provider idempotency token for this persisted launch intent. + * Optional so workers can safely replay records written before this field + * existed. */ + launch_client_token?: string; + /** Exact fingerprint of the RunMicrovm request paired with + * `launch_client_token`. Unlike `launch_fingerprint`, this preserves + * wire-level distinctions such as connector ordering for safe replay. */ + launch_request_fingerprint?: string; + state: RuntimeSessionState; + generation: number; + launched_at?: number; + last_seen_at: number; + hard_deadline_at?: number; + workspace_checkpoint?: string; + checkpointed_at?: number; + last_error?: string; } const SESS_PREFIX = 'rtsx:sess:'; @@ -73,90 +76,99 @@ const CKPT_SEQ_PREFIX = 'rtsx:ckptseq:'; export const RUNTIME_SESSION_REDIS_CLEANUP_TIMEOUT_MS = 2_000; export interface RuntimeSessionRedisCommandOptions { - signal?: AbortSignal; - timeoutMs?: number; + signal?: AbortSignal; + timeoutMs?: number; } export interface RuntimeSessionLockRenewOptions extends RuntimeSessionRedisCommandOptions { - /** Positive evidence from a late token-mismatch must still fence the holder. */ - onLateLost?: () => void; + /** Positive evidence from a late token-mismatch must still fence the holder. */ + onLateLost?: () => void; } class RuntimeSessionRedisTimeoutError extends Error {} function registryAbortReason(signal: AbortSignal): Error { - return signal.reason instanceof Error - ? signal.reason - : new Error('Runtime session registry command aborted'); + return signal.reason instanceof Error + ? signal.reason + : new Error('Runtime session registry command aborted'); } async function runRegistryCommand( - label: string, - operation: () => Promise, - options: RuntimeSessionRedisCommandOptions = {}, - onLateValue?: (value: T) => void | Promise, + label: string, + operation: () => Promise, + options: RuntimeSessionRedisCommandOptions = {}, + onLateValue?: (value: T) => void | Promise, ): Promise { - const timeoutMs = options.timeoutMs ?? RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS; - if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { - throw new Error('Runtime session Redis command timeout must be positive'); - } - - /* Check before constructing the command. Promise racing cannot remove an - * ioredis command from its command/offline queues, so an already-expired job - * must not enqueue another late mutation. */ - options.signal?.throwIfAborted(); - const command = operation(); - - return new Promise((resolve, reject) => { - let settled = false; - const runBestEffort = ( - callback?: (() => void | Promise), - ): void => { - if (!callback) return; - try { - void Promise.resolve(callback()).catch(() => {}); - } catch { - // Best-effort cleanup must not replace the caller's primary outcome. - } - }; - const finish = (callback: () => void): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - options.signal?.removeEventListener('abort', onAbort); - callback(); - }; - const onAbort = (): void => { - finish(() => reject(registryAbortReason(options.signal as AbortSignal))); - }; - const timer = setTimeout(() => { - finish(() => reject(new RuntimeSessionRedisTimeoutError( - `${label} timed out after ${timeoutMs}ms`, - ))); - }, timeoutMs); - timer.unref?.(); - options.signal?.addEventListener('abort', onAbort, { once: true }); - /* Abort can race the pre-enqueue check and listener registration. Signals - * do not replay an already-fired event to a newly-added listener. */ - if (options.signal?.aborted) { - onAbort(); + const timeoutMs = + options.timeoutMs ?? RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS; + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error( + 'Runtime session Redis command timeout must be positive', + ); } - /* Keep handlers attached after the caller-side deadline wins. ioredis may - * settle the abandoned command when the connection recovers; consuming - * that result/rejection avoids an unhandled rejection while the fencing - * rules above make a late mutation harmless. */ - command.then( - (value) => { - if (settled) { - runBestEffort(onLateValue ? () => onLateValue(value) : undefined); - return; + /* Check before constructing the command. Promise racing cannot remove an + * ioredis command from its command/offline queues, so an already-expired job + * must not enqueue another late mutation. */ + options.signal?.throwIfAborted(); + const command = operation(); + + return new Promise((resolve, reject) => { + let settled = false; + const runBestEffort = (callback?: () => void | Promise): void => { + if (!callback) return; + try { + void Promise.resolve(callback()).catch(() => {}); + } catch { + // Best-effort cleanup must not replace the caller's primary outcome. + } + }; + const finish = (callback: () => void): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + options.signal?.removeEventListener('abort', onAbort); + callback(); + }; + const onAbort = (): void => { + finish(() => + reject(registryAbortReason(options.signal as AbortSignal)), + ); + }; + const timer = setTimeout(() => { + finish(() => + reject( + new RuntimeSessionRedisTimeoutError( + `${label} timed out after ${timeoutMs}ms`, + ), + ), + ); + }, timeoutMs); + timer.unref?.(); + options.signal?.addEventListener('abort', onAbort, { once: true }); + /* Abort can race the pre-enqueue check and listener registration. Signals + * do not replay an already-fired event to a newly-added listener. */ + if (options.signal?.aborted) { + onAbort(); } - finish(() => resolve(value)); - }, - error => finish(() => reject(error)), - ); - }); + + /* Keep handlers attached after the caller-side deadline wins. ioredis may + * settle the abandoned command when the connection recovers; consuming + * that result/rejection avoids an unhandled rejection while the fencing + * rules above make a late mutation harmless. */ + command.then( + value => { + if (settled) { + runBestEffort( + onLateValue ? () => onLateValue(value) : undefined, + ); + return; + } + finish(() => resolve(value)); + }, + error => finish(() => reject(error)), + ); + }); } /** The session lock is held across the WHOLE `executeSession` critical path @@ -169,85 +181,95 @@ async function runRegistryCommand( * one heartbeat interval plus a stalled event loop — it already covers a normal * relaunch (execute + launch + health + the checkpoint I/Os) with headroom. */ export const RUNTIME_SESSION_LOCK_TTL_MS = - env.JOB_TIMEOUT + - 2 * env.LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS + - env.LAMBDA_MICROVM_HEALTH_TIMEOUT_MS + - 4 * env.CHECKPOINT_TIMEOUT_MS + - 60_000; + env.JOB_TIMEOUT + + 2 * env.LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS + + env.LAMBDA_MICROVM_HEALTH_TIMEOUT_MS + + 4 * env.CHECKPOINT_TIMEOUT_MS + + 60_000; const MAX_MICROVM_DURATION_SECONDS = 28_800; -export const RUNTIME_SESSION_RECORD_TTL_SECONDS = MAX_MICROVM_DURATION_SECONDS + 600; - -type RedisWithScripts = Redis & { - releaseRuntimeSessionLockScript(lockKey: string, token: string): Promise; - renewRuntimeSessionLockScript(lockKey: string, token: string, ttlMs: string): Promise; - writeRuntimeSessionRecordScript( - sessKey: string, - lockKey: string, - token: string, - recordJson: string, - ttlSeconds: string, - ): Promise; - removeRuntimeSessionScript( - sessKey: string, - lockKey: string, - token: string, - ): Promise; - allocateCheckpointSequenceScript( - sequenceKey: string, - retainedMax: string, - ttlSeconds: string, - ): Promise; - allocateRuntimeSessionGenerationScript( - generationKey: string, - initialGeneration: string, - ttlSeconds: string, - ): Promise; +export const RUNTIME_SESSION_RECORD_TTL_SECONDS = + MAX_MICROVM_DURATION_SECONDS + 600; + +type RedisWithScripts = RedisClient & { + releaseRuntimeSessionLockScript( + lockKey: string, + token: string, + ): Promise; + renewRuntimeSessionLockScript( + lockKey: string, + token: string, + ttlMs: string, + ): Promise; + writeRuntimeSessionRecordScript( + sessKey: string, + lockKey: string, + token: string, + recordJson: string, + ttlSeconds: string, + ): Promise; + removeRuntimeSessionScript( + sessKey: string, + lockKey: string, + token: string, + ): Promise; + allocateCheckpointSequenceScript( + sequenceKey: string, + retainedMax: string, + ttlSeconds: string, + ): Promise; + allocateRuntimeSessionGenerationScript( + generationKey: string, + initialGeneration: string, + ttlSeconds: string, + ): Promise; }; -const SCRIPTS_REGISTERED = Symbol.for('runtime-session-registry.scriptsRegistered'); - -function registerScripts(client: Redis): RedisWithScripts { - const tagged = client as Redis & { [SCRIPTS_REGISTERED]?: true }; - if (tagged[SCRIPTS_REGISTERED]) return client as RedisWithScripts; - client.defineCommand('releaseRuntimeSessionLockScript', { - numberOfKeys: 1, - lua: "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end", - }); - client.defineCommand('renewRuntimeSessionLockScript', { - numberOfKeys: 1, - lua: "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('pexpire', KEYS[1], ARGV[2]) else return 0 end", - }); - client.defineCommand('writeRuntimeSessionRecordScript', { - numberOfKeys: 2, - lua: `if redis.call('get', KEYS[2]) == ARGV[1] then +const SCRIPTS_REGISTERED = Symbol.for( + 'runtime-session-registry.scriptsRegistered', +); + +function registerScripts(client: RedisClient): RedisWithScripts { + const tagged = client as RedisClient & { [SCRIPTS_REGISTERED]?: true }; + if (tagged[SCRIPTS_REGISTERED]) return client as RedisWithScripts; + client.defineCommand('releaseRuntimeSessionLockScript', { + numberOfKeys: 1, + lua: "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end", + }); + client.defineCommand('renewRuntimeSessionLockScript', { + numberOfKeys: 1, + lua: "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('pexpire', KEYS[1], ARGV[2]) else return 0 end", + }); + client.defineCommand('writeRuntimeSessionRecordScript', { + numberOfKeys: 2, + lua: `if redis.call('get', KEYS[2]) == ARGV[1] then redis.call('set', KEYS[1], ARGV[2], 'EX', ARGV[3]) return 1 else return 0 end`, - }); - client.defineCommand('removeRuntimeSessionScript', { - numberOfKeys: 2, - lua: `if redis.call('get', KEYS[2]) == ARGV[1] then + }); + client.defineCommand('removeRuntimeSessionScript', { + numberOfKeys: 2, + lua: `if redis.call('get', KEYS[2]) == ARGV[1] then redis.call('del', KEYS[1]) return 1 else return 0 end`, - }); - client.defineCommand('allocateCheckpointSequenceScript', { - numberOfKeys: 1, - lua: `local current = tonumber(redis.call('get', KEYS[1]) or '0') + }); + client.defineCommand('allocateCheckpointSequenceScript', { + numberOfKeys: 1, + lua: `local current = tonumber(redis.call('get', KEYS[1]) or '0') local retained = tonumber(ARGV[1]) if retained > current then current = retained end local sequence = current + 1 redis.call('set', KEYS[1], tostring(sequence), 'EX', ARGV[2]) return sequence`, - }); - client.defineCommand('allocateRuntimeSessionGenerationScript', { - numberOfKeys: 1, - lua: `local current = redis.call('get', KEYS[1]) or '0' + }); + client.defineCommand('allocateRuntimeSessionGenerationScript', { + numberOfKeys: 1, + lua: `local current = redis.call('get', KEYS[1]) or '0' local initial = ARGV[1] local generation local below_initial = string.len(current) < string.len(initial) @@ -261,169 +283,192 @@ else generation = redis.call('get', KEYS[1]) end return generation`, - }); - tagged[SCRIPTS_REGISTERED] = true; - return client as RedisWithScripts; + }); + tagged[SCRIPTS_REGISTERED] = true; + return client as RedisWithScripts; } let redis: RedisWithScripts = registerScripts(connection); /** Test seam mirroring replay-state.ts: swap in ioredis-mock per test. */ export function setRedisForTests(client: Redis): void { - redis = registerScripts(client); + redis = registerScripts(client); } export function resetRedisForTests(): void { - redis = registerScripts(connection); + redis = registerScripts(connection); } export async function acquireRuntimeSessionLock( - runtimeSessionId: string, - ttlMs: number = RUNTIME_SESSION_LOCK_TTL_MS, - options: RuntimeSessionRedisCommandOptions = {}, + runtimeSessionId: string, + ttlMs: number = RUNTIME_SESSION_LOCK_TTL_MS, + options: RuntimeSessionRedisCommandOptions = {}, ): Promise { - const token = nanoid(); - const releaseAmbiguousAcquire = (): Promise => - releaseRuntimeSessionLock(runtimeSessionId, token); - let result: 'OK' | null; - try { - result = await runRegistryCommand( - 'Runtime session lock acquire', - () => redis.set(`${LOCK_PREFIX}${runtimeSessionId}`, token, 'PX', ttlMs, 'NX'), - options, - /* A caller-side deadline cannot cancel ioredis. The original SET may - * have succeeded even if an automatic replay eventually resolves null, - * so CAS-release our unique token after every late settlement. */ - releaseAmbiguousAcquire, - ); - } catch (error) { - /* Redis writes are ambiguous on every error, not only a local timeout: - * the SET may have applied before the response was lost. Queue a release - * immediately; it is ordered after the SET on this shared client. */ + const token = nanoid(); + const releaseAmbiguousAcquire = (): Promise => + releaseRuntimeSessionLock(runtimeSessionId, token); + let result: 'OK' | null; + try { + result = await runRegistryCommand( + 'Runtime session lock acquire', + () => + redis.set( + `${LOCK_PREFIX}${runtimeSessionId}`, + token, + 'PX', + ttlMs, + 'NX', + ), + options, + /* A caller-side deadline cannot cancel ioredis. The original SET may + * have succeeded even if an automatic replay eventually resolves null, + * so CAS-release our unique token after every late settlement. */ + releaseAmbiguousAcquire, + ); + } catch (error) { + /* Redis writes are ambiguous on every error, not only a local timeout: + * the SET may have applied before the response was lost. Queue a release + * immediately; it is ordered after the SET on this shared client. */ + void releaseAmbiguousAcquire().catch(() => {}); + throw error; + } + if (result === 'OK') return token; + /* A reconnect/replay can resolve null inside the caller's budget even + * though the original SET succeeded and only its response was lost. A + * token-CAS release is also safe for ordinary lock contention. */ void releaseAmbiguousAcquire().catch(() => {}); - throw error; - } - if (result === 'OK') return token; - /* A reconnect/replay can resolve null inside the caller's budget even - * though the original SET succeeded and only its response was lost. A - * token-CAS release is also safe for ordinary lock contention. */ - void releaseAmbiguousAcquire().catch(() => {}); - return null; + return null; } /** Polls for the session mutex; returns null once `waitMs` is exhausted. * Stateful callers surface contention as HTTP 409 instead of executing cold. */ export async function waitForRuntimeSessionLock( - runtimeSessionId: string, - args: { waitMs: number; pollMs?: number; ttlMs?: number; signal?: AbortSignal }, + runtimeSessionId: string, + args: { + waitMs: number; + pollMs?: number; + ttlMs?: number; + signal?: AbortSignal; + }, ): Promise { - const pollMs = args.pollMs ?? 250; - const deadline = Date.now() + args.waitMs; - let firstAttempt = true; - for (;;) { - args.signal?.throwIfAborted(); - const remainingWaitMs = deadline - Date.now(); - /* `waitMs: 0` disables contention polling; it must not disable the one - * uncontended SET needed to start every session execution. Give that - * immediate attempt the normal Redis command bound. Positive wait budgets - * continue to bound the SET itself by their remaining time. */ - if (!firstAttempt && remainingWaitMs <= 0) return null; - const commandTimeoutMs = args.waitMs > 0 - ? Math.max(1, remainingWaitMs) - : RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS; - firstAttempt = false; - let token: string | null; - try { - token = await acquireRuntimeSessionLock( - runtimeSessionId, - args.ttlMs, - { - signal: args.signal, - timeoutMs: commandTimeoutMs, - }, - ); - } catch (error) { - /* This command's timeout is the remaining lock-wait budget. Preserve the - * documented contention contract while allowing caller aborts and actual - * Redis failures to propagate. */ - if (error instanceof RuntimeSessionRedisTimeoutError) return null; - throw error; - } - if (args.signal?.aborted) { - if (token != null) await releaseRuntimeSessionLock(runtimeSessionId, token); - args.signal.throwIfAborted(); + const pollMs = args.pollMs ?? 250; + const deadline = Date.now() + args.waitMs; + let firstAttempt = true; + for (;;) { + args.signal?.throwIfAborted(); + const remainingWaitMs = deadline - Date.now(); + /* `waitMs: 0` disables contention polling; it must not disable the one + * uncontended SET needed to start every session execution. Give that + * immediate attempt the normal Redis command bound. Positive wait budgets + * continue to bound the SET itself by their remaining time. */ + if (!firstAttempt && remainingWaitMs <= 0) return null; + const commandTimeoutMs = + args.waitMs > 0 + ? Math.max(1, remainingWaitMs) + : RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS; + firstAttempt = false; + let token: string | null; + try { + token = await acquireRuntimeSessionLock( + runtimeSessionId, + args.ttlMs, + { + signal: args.signal, + timeoutMs: commandTimeoutMs, + }, + ); + } catch (error) { + /* This command's timeout is the remaining lock-wait budget. Preserve the + * documented contention contract while allowing caller aborts and actual + * Redis failures to propagate. */ + if (error instanceof RuntimeSessionRedisTimeoutError) return null; + throw error; + } + if (args.signal?.aborted) { + if (token != null) + await releaseRuntimeSessionLock(runtimeSessionId, token); + args.signal.throwIfAborted(); + } + if (token != null) return token; + if (args.waitMs <= 0) return null; + if (Date.now() + pollMs > deadline) return null; + await new Promise((resolve, reject) => { + let settled = false; + const timer = setTimeout(() => { + if (settled) return; + settled = true; + args.signal?.removeEventListener('abort', onAbort); + resolve(); + }, pollMs); + const onAbort = (): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + args.signal?.removeEventListener('abort', onAbort); + reject( + args.signal?.reason instanceof Error + ? args.signal.reason + : new Error('Runtime session lock wait aborted'), + ); + }; + args.signal?.addEventListener('abort', onAbort, { once: true }); + /* Abort can land between the loop's pre-sleep check and listener + * registration; signals do not replay that event. */ + if (args.signal?.aborted) onAbort(); + }); } - if (token != null) return token; - if (args.waitMs <= 0) return null; - if (Date.now() + pollMs > deadline) return null; - await new Promise((resolve, reject) => { - let settled = false; - const timer = setTimeout(() => { - if (settled) return; - settled = true; - args.signal?.removeEventListener('abort', onAbort); - resolve(); - }, pollMs); - const onAbort = (): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - args.signal?.removeEventListener('abort', onAbort); - reject(args.signal?.reason instanceof Error - ? args.signal.reason - : new Error('Runtime session lock wait aborted')); - }; - args.signal?.addEventListener('abort', onAbort, { once: true }); - /* Abort can land between the loop's pre-sleep check and listener - * registration; signals do not replay that event. */ - if (args.signal?.aborted) onAbort(); - }); - } } export async function releaseRuntimeSessionLock( - runtimeSessionId: string, - token: string, - options: RuntimeSessionRedisCommandOptions = {}, + runtimeSessionId: string, + token: string, + options: RuntimeSessionRedisCommandOptions = {}, ): Promise { - const retryDelaysMs = [50, 200] as const; - /* Release is called from finally blocks after the job signal may already be - * aborted. Give the whole cleanup sequence its own short absolute budget - * instead of inheriting that expired signal or resetting a timeout per retry. */ - const cleanupTimeoutMs = options.timeoutMs ?? RUNTIME_SESSION_REDIS_CLEANUP_TIMEOUT_MS; - const deadlineAtMs = Date.now() + cleanupTimeoutMs; - let lastError: unknown; - for (let attempt = 0; attempt <= retryDelaysMs.length; attempt += 1) { - const remainingMs = deadlineAtMs - Date.now(); - if (remainingMs <= 0 || options.signal?.aborted) break; - try { - await runRegistryCommand( - 'Runtime session lock release', - () => redis.releaseRuntimeSessionLockScript(`${LOCK_PREFIX}${runtimeSessionId}`, token), - { signal: options.signal, timeoutMs: remainingMs }, - ); - return; - } catch (error) { - lastError = error; - if (attempt < retryDelaysMs.length) { - const retryDelayMs = Math.min( - retryDelaysMs[attempt], - Math.max(0, deadlineAtMs - Date.now()), - ); - if (retryDelayMs > 0) { - await new Promise(resolve => setTimeout(resolve, retryDelayMs)); + const retryDelaysMs = [50, 200] as const; + /* Release is called from finally blocks after the job signal may already be + * aborted. Give the whole cleanup sequence its own short absolute budget + * instead of inheriting that expired signal or resetting a timeout per retry. */ + const cleanupTimeoutMs = + options.timeoutMs ?? RUNTIME_SESSION_REDIS_CLEANUP_TIMEOUT_MS; + const deadlineAtMs = Date.now() + cleanupTimeoutMs; + let lastError: unknown; + for (let attempt = 0; attempt <= retryDelaysMs.length; attempt += 1) { + const remainingMs = deadlineAtMs - Date.now(); + if (remainingMs <= 0 || options.signal?.aborted) break; + try { + await runRegistryCommand( + 'Runtime session lock release', + () => + redis.releaseRuntimeSessionLockScript( + `${LOCK_PREFIX}${runtimeSessionId}`, + token, + ), + { signal: options.signal, timeoutMs: remainingMs }, + ); + return; + } catch (error) { + lastError = error; + if (attempt < retryDelaysMs.length) { + const retryDelayMs = Math.min( + retryDelaysMs[attempt], + Math.max(0, deadlineAtMs - Date.now()), + ); + if (retryDelayMs > 0) { + await new Promise(resolve => + setTimeout(resolve, retryDelayMs), + ); + } + } } - } } - } - /* Release runs from finally blocks. Do not mask either a successful execute - * or its primary error, but heal brief Redis failovers before accepting that - * the token-guarded lease must age out. The Lua release is idempotent, so a - * retry is safe even if Redis deleted the key but lost the first response. */ - logger.warn('Failed to release runtime session lock after retries', { - runtimeSessionId, - err: lastError, - }); + /* Release runs from finally blocks. Do not mask either a successful execute + * or its primary error, but heal brief Redis failovers before accepting that + * the token-guarded lease must age out. The Lua release is idempotent, so a + * retry is safe even if Redis deleted the key but lost the first response. */ + logger.warn('Failed to release runtime session lock after retries', { + runtimeSessionId, + err: lastError, + }); } /** Fenced heartbeat: extend the lock's TTL only while we still hold the token. @@ -439,80 +484,88 @@ export async function releaseRuntimeSessionLock( export type LockRenewal = 'held' | 'lost' | 'error'; export async function renewRuntimeSessionLock( - runtimeSessionId: string, - token: string, - ttlMs: number = RUNTIME_SESSION_LOCK_TTL_MS, - options: RuntimeSessionLockRenewOptions = {}, + runtimeSessionId: string, + token: string, + ttlMs: number = RUNTIME_SESSION_LOCK_TTL_MS, + options: RuntimeSessionLockRenewOptions = {}, ): Promise { - try { - const result = await runRegistryCommand( - 'Runtime session lock renewal', - () => redis.renewRuntimeSessionLockScript( - `${LOCK_PREFIX}${runtimeSessionId}`, - token, - String(ttlMs), - ), - options, - result => { - if (result !== 1) options.onLateLost?.(); - }, - ); - return result === 1 ? 'held' : 'lost'; - } catch (err) { - logger.warn('Failed to renew runtime session lock', { runtimeSessionId, err }); - return 'error'; - } + try { + const result = await runRegistryCommand( + 'Runtime session lock renewal', + () => + redis.renewRuntimeSessionLockScript( + `${LOCK_PREFIX}${runtimeSessionId}`, + token, + String(ttlMs), + ), + options, + result => { + if (result !== 1) options.onLateLost?.(); + }, + ); + return result === 1 ? 'held' : 'lost'; + } catch (err) { + logger.warn('Failed to renew runtime session lock', { + runtimeSessionId, + err, + }); + return 'error'; + } } export async function readRuntimeSessionRecord( - runtimeSessionId: string, - options: RuntimeSessionRedisCommandOptions = {}, + runtimeSessionId: string, + options: RuntimeSessionRedisCommandOptions = {}, ): Promise { - const data = await runRegistryCommand( - 'Runtime session record read', - () => redis.get(`${SESS_PREFIX}${runtimeSessionId}`), - options, - ); - if (data == null) return null; - /* Treat a corrupt/incompatible record as missing so a single bad key can't - * wedge every request for the session until it is manually deleted. */ - try { - return JSON.parse(data) as RuntimeSessionRecord; - } catch (err) { - logger.warn('Discarding malformed runtime session record', { runtimeSessionId, err }); - return null; - } + const data = await runRegistryCommand( + 'Runtime session record read', + () => redis.get(`${SESS_PREFIX}${runtimeSessionId}`), + options, + ); + if (data == null) return null; + /* Treat a corrupt/incompatible record as missing so a single bad key can't + * wedge every request for the session until it is manually deleted. */ + try { + return JSON.parse(data) as RuntimeSessionRecord; + } catch (err) { + logger.warn('Discarding malformed runtime session record', { + runtimeSessionId, + err, + }); + return null; + } } /** Fenced write: persists the record only while `lockToken` still holds the * session mutex. Returns false when ownership is lost or cannot be confirmed, * so callers fail closed and tear down any potentially concurrent VM. */ export async function writeRuntimeSessionRecord( - record: RuntimeSessionRecord, - lockToken: string, - ttlSeconds: number = RUNTIME_SESSION_RECORD_TTL_SECONDS, - options: RuntimeSessionRedisCommandOptions = {}, + record: RuntimeSessionRecord, + lockToken: string, + ttlSeconds: number = RUNTIME_SESSION_RECORD_TTL_SECONDS, + options: RuntimeSessionRedisCommandOptions = {}, ): Promise { - try { - const result = await runRegistryCommand( - 'Runtime session record write', - () => redis.writeRuntimeSessionRecordScript( - `${SESS_PREFIX}${record.runtime_session_id}`, - `${LOCK_PREFIX}${record.runtime_session_id}`, - lockToken, - JSON.stringify(record), - String(ttlSeconds), - ), - options, - ); - return result === 1; - } catch { - /* Every failed write is ambiguous once issued: Redis may have applied it, - * or may later confirm that our token was already lost. Even a concurrent - * caller abort cannot prove ownership. Report fencing so callers use their - * existing teardown path instead of leaving a mutated VM reusable. */ - return false; - } + try { + const result = await runRegistryCommand( + 'Runtime session record write', + () => + redis.writeRuntimeSessionRecordScript( + `${SESS_PREFIX}${record.runtime_session_id}`, + `${LOCK_PREFIX}${record.runtime_session_id}`, + lockToken, + JSON.stringify(record), + String(ttlSeconds), + ), + options, + ); + return result === 1; + } catch { + /* Every failed write is ambiguous once issued: Redis may have applied it, + * or may later confirm that our token was already lost. Even a concurrent + * caller abort cannot prove ownership. Report fencing so callers use their + * existing teardown path instead of leaving a mutated VM reusable. */ + return false; + } } /** Monotonic generation for launch fencing: allocated while holding the lock, @@ -521,28 +574,33 @@ export async function writeRuntimeSessionRecord( * counter into a collision-resistant namespace while remaining compatible * with older workers, which derive the provider token from this number. */ export async function allocateRuntimeSessionGeneration( - runtimeSessionId: string, - initialGeneration = 1, - options: RuntimeSessionRedisCommandOptions = {}, + runtimeSessionId: string, + initialGeneration = 1, + options: RuntimeSessionRedisCommandOptions = {}, ): Promise { - if (!Number.isSafeInteger(initialGeneration) || initialGeneration < 1) { - throw new Error('Runtime session generation must be a positive safe integer'); - } - const key = `${GEN_PREFIX}${runtimeSessionId}`; - const rawGeneration = await runRegistryCommand( - 'Runtime session generation allocation', - () => redis.allocateRuntimeSessionGenerationScript( - key, - String(initialGeneration), - String(RUNTIME_SESSION_RECORD_TTL_SECONDS), - ), - options, - ); - const generation = Number(rawGeneration); - if (!Number.isSafeInteger(generation) || generation < 1) { - throw new Error('Runtime session generation exceeded the safe integer range'); - } - return generation; + if (!Number.isSafeInteger(initialGeneration) || initialGeneration < 1) { + throw new Error( + 'Runtime session generation must be a positive safe integer', + ); + } + const key = `${GEN_PREFIX}${runtimeSessionId}`; + const rawGeneration = await runRegistryCommand( + 'Runtime session generation allocation', + () => + redis.allocateRuntimeSessionGenerationScript( + key, + String(initialGeneration), + String(RUNTIME_SESSION_RECORD_TTL_SECONDS), + ), + options, + ); + const generation = Number(rawGeneration); + if (!Number.isSafeInteger(generation) || generation < 1) { + throw new Error( + 'Runtime session generation exceeded the safe integer range', + ); + } + return generation; } /** Atomically reserves a monotonic checkpoint sequence above both the Redis @@ -552,37 +610,39 @@ export async function allocateRuntimeSessionGeneration( * session lock can consume a distinct sequence, but can never reset the * counter and overwrite the newer holder's immutable object key. */ export async function allocateCheckpointSequence( - runtimeSessionId: string, - retainedMax = 0, - options: RuntimeSessionRedisCommandOptions = {}, + runtimeSessionId: string, + retainedMax = 0, + options: RuntimeSessionRedisCommandOptions = {}, ): Promise { - const key = `${CKPT_SEQ_PREFIX}${runtimeSessionId}`; - return runRegistryCommand( - 'Runtime session checkpoint sequence allocation', - () => redis.allocateCheckpointSequenceScript( - key, - String(retainedMax), - String(RUNTIME_SESSION_RECORD_TTL_SECONDS), - ), - options, - ); + const key = `${CKPT_SEQ_PREFIX}${runtimeSessionId}`; + return runRegistryCommand( + 'Runtime session checkpoint sequence allocation', + () => + redis.allocateCheckpointSequenceScript( + key, + String(retainedMax), + String(RUNTIME_SESSION_RECORD_TTL_SECONDS), + ), + options, + ); } /** Fenced removal: deletes the record while the caller holds the mutex. * Returns false when fenced. */ export async function removeRuntimeSession( - runtimeSessionId: string, - lockToken: string, - options: RuntimeSessionRedisCommandOptions = {}, + runtimeSessionId: string, + lockToken: string, + options: RuntimeSessionRedisCommandOptions = {}, ): Promise { - const result = await runRegistryCommand( - 'Runtime session record removal', - () => redis.removeRuntimeSessionScript( - `${SESS_PREFIX}${runtimeSessionId}`, - `${LOCK_PREFIX}${runtimeSessionId}`, - lockToken, - ), - options, - ); - return result === 1; + const result = await runRegistryCommand( + 'Runtime session record removal', + () => + redis.removeRuntimeSessionScript( + `${SESS_PREFIX}${runtimeSessionId}`, + `${LOCK_PREFIX}${runtimeSessionId}`, + lockToken, + ), + options, + ); + return result === 1; } diff --git a/service/src/runtime-session/throttle.ts b/service/src/runtime-session/throttle.ts index 3a28468..f9b938e 100644 --- a/service/src/runtime-session/throttle.ts +++ b/service/src/runtime-session/throttle.ts @@ -1,4 +1,5 @@ import type { Redis } from 'ioredis'; +import type { RedisClient } from '../redis-connection'; import { connection } from '../queue'; /** @@ -18,76 +19,80 @@ const POISON_PREFIX = 'rtsx:tps:poison:'; const BUCKET_TTL_MS = 2_000; const DEFAULT_POISON_MS = 2_000; -let redis: Redis = connection; +let redis: RedisClient = connection; export function setRedisForTests(client: Redis): void { - redis = client; + redis = client; } export function resetRedisForTests(): void { - redis = connection; + redis = connection; } export class MicrovmOpThrottledError extends Error { - constructor(public readonly op: ThrottledOp, budgetMs: number) { - super(`Lambda MicroVM ${op} budget exhausted after ${budgetMs}ms of throttling`); - this.name = 'MicrovmOpThrottledError'; - } + constructor( + public readonly op: ThrottledOp, + budgetMs: number, + ) { + super( + `Lambda MicroVM ${op} budget exhausted after ${budgetMs}ms of throttling`, + ); + this.name = 'MicrovmOpThrottledError'; + } } export interface OpBudgetOptions { - limitPerSecond: number; - /** Total time the caller is willing to wait for a slot. */ - budgetMs: number; - /** Optional caller-owned absolute deadline. Supplying it lets multiple - * launch attempts and every Redis leg consume one shared budget instead of - * starting a fresh `budgetMs` window. */ - deadlineAtMs?: number; - now?: () => number; - sleep?: (ms: number, signal?: AbortSignal) => Promise; - signal?: AbortSignal; + limitPerSecond: number; + /** Total time the caller is willing to wait for a slot. */ + budgetMs: number; + /** Optional caller-owned absolute deadline. Supplying it lets multiple + * launch attempts and every Redis leg consume one shared budget instead of + * starting a fresh `budgetMs` window. */ + deadlineAtMs?: number; + now?: () => number; + sleep?: (ms: number, signal?: AbortSignal) => Promise; + signal?: AbortSignal; } export interface PoisonOpBucketOptions { - deadlineAtMs?: number; - now?: () => number; - signal?: AbortSignal; + deadlineAtMs?: number; + now?: () => number; + signal?: AbortSignal; } function abortReason(signal: AbortSignal): Error { - return signal.reason instanceof Error - ? signal.reason - : new Error('Lambda MicroVM operation budget aborted'); + return signal.reason instanceof Error + ? signal.reason + : new Error('Lambda MicroVM operation budget aborted'); } -const defaultSleep = (ms: number, signal?: AbortSignal): Promise => new Promise( - (resolve, reject) => { - if (signal?.aborted) { - reject(abortReason(signal)); - return; - } - let settled = false; - const timer = setTimeout(() => { - if (settled) return; - settled = true; - signal?.removeEventListener('abort', onAbort); - resolve(); - }, ms); - const onAbort = (): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - signal?.removeEventListener('abort', onAbort); - reject(abortReason(signal as AbortSignal)); - }; - signal?.addEventListener('abort', onAbort, { once: true }); - /* Abort can land between the preflight check and registration. */ - if (signal?.aborted) onAbort(); - }, -); +const defaultSleep = (ms: number, signal?: AbortSignal): Promise => + new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(abortReason(signal)); + return; + } + let settled = false; + const timer = setTimeout(() => { + if (settled) return; + settled = true; + signal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + const onAbort = (): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + reject(abortReason(signal as AbortSignal)); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + /* Abort can land between the preflight check and registration. */ + if (signal?.aborted) onAbort(); + }); function throwIfAborted(signal?: AbortSignal): void { - if (signal?.aborted) throw abortReason(signal); + if (signal?.aborted) throw abortReason(signal); } /** ioredis cannot cancel one queued command. This bounds the caller's await, @@ -95,62 +100,66 @@ function throwIfAborted(signal?: AbortSignal): void { * throttle mutations' conservative ambiguity: a late INCR consumes capacity, * while a late expiry or poison only reduces load. */ function runRedisWithinDeadline( - operation: () => Promise, - options: { - deadlineAtMs?: number; - now: () => number; - signal?: AbortSignal; - timeoutError: () => Error; - }, + operation: () => Promise, + options: { + deadlineAtMs?: number; + now: () => number; + signal?: AbortSignal; + timeoutError: () => Error; + }, ): Promise { - throwIfAborted(options.signal); - const remainingMs = options.deadlineAtMs == null - ? undefined - : options.deadlineAtMs - options.now(); - if (remainingMs != null && remainingMs <= 0) { - return Promise.reject(options.timeoutError()); - } - - /* Construct only after the preflight checks: an already-expired caller must - * not enqueue a mutation that can execute after Redis recovers. */ - const command = operation(); - if (remainingMs == null && options.signal == null) return command; - - return new Promise((resolve, reject) => { - let settled = false; - let timer: ReturnType | undefined; - const finish = (callback: () => void): void => { - if (settled) return; - settled = true; - if (timer !== undefined) clearTimeout(timer); - options.signal?.removeEventListener('abort', onAbort); - callback(); - }; - const onAbort = (): void => { - finish(() => reject(abortReason(options.signal as AbortSignal))); - }; - if (remainingMs != null) { - timer = setTimeout(() => { - finish(() => reject(options.timeoutError())); - }, remainingMs); - timer.unref?.(); + throwIfAborted(options.signal); + const remainingMs = + options.deadlineAtMs == null + ? undefined + : options.deadlineAtMs - options.now(); + if (remainingMs != null && remainingMs <= 0) { + return Promise.reject(options.timeoutError()); } - options.signal?.addEventListener('abort', onAbort, { once: true }); - /* Abort may race the preflight check and listener registration. */ - if (options.signal?.aborted) onAbort(); - - command.then( - (value) => { - if (settled) return; - if (options.deadlineAtMs != null && options.now() >= options.deadlineAtMs) { - finish(() => reject(options.timeoutError())); - return; + + /* Construct only after the preflight checks: an already-expired caller must + * not enqueue a mutation that can execute after Redis recovers. */ + const command = operation(); + if (remainingMs == null && options.signal == null) return command; + + return new Promise((resolve, reject) => { + let settled = false; + let timer: ReturnType | undefined; + const finish = (callback: () => void): void => { + if (settled) return; + settled = true; + if (timer !== undefined) clearTimeout(timer); + options.signal?.removeEventListener('abort', onAbort); + callback(); + }; + const onAbort = (): void => { + finish(() => reject(abortReason(options.signal as AbortSignal))); + }; + if (remainingMs != null) { + timer = setTimeout(() => { + finish(() => reject(options.timeoutError())); + }, remainingMs); + timer.unref?.(); } - finish(() => resolve(value)); - }, - error => finish(() => reject(error)), - ); - }); + options.signal?.addEventListener('abort', onAbort, { once: true }); + /* Abort may race the preflight check and listener registration. */ + if (options.signal?.aborted) onAbort(); + + command.then( + value => { + if (settled) return; + if ( + options.deadlineAtMs != null && + options.now() >= options.deadlineAtMs + ) { + finish(() => reject(options.timeoutError())); + return; + } + finish(() => resolve(value)); + }, + error => finish(() => reject(error)), + ); + }); } /** @@ -158,70 +167,76 @@ function runRedisWithinDeadline( * boundaries until `budgetMs` is exhausted. Throws MicrovmOpThrottledError * when no slot frees up in time. */ -export async function acquireOpBudget(op: ThrottledOp, options: OpBudgetOptions): Promise { - const now = options.now ?? Date.now; - const sleep = options.sleep ?? defaultSleep; - const deadline = options.deadlineAtMs ?? now() + options.budgetMs; - const redisOptions = { - deadlineAtMs: deadline, - now, - signal: options.signal, - timeoutError: () => new MicrovmOpThrottledError(op, options.budgetMs), - }; - - for (;;) { - throwIfAborted(options.signal); - const poisoned = await runRedisWithinDeadline( - () => redis.pttl(`${POISON_PREFIX}${op}`), - redisOptions, - ); - throwIfAborted(options.signal); - if (poisoned > 0) { - if (now() + poisoned > deadline) throw new MicrovmOpThrottledError(op, options.budgetMs); - await sleep(poisoned, options.signal); - continue; - } +export async function acquireOpBudget( + op: ThrottledOp, + options: OpBudgetOptions, +): Promise { + const now = options.now ?? Date.now; + const sleep = options.sleep ?? defaultSleep; + const deadline = options.deadlineAtMs ?? now() + options.budgetMs; + const redisOptions = { + deadlineAtMs: deadline, + now, + signal: options.signal, + timeoutError: () => new MicrovmOpThrottledError(op, options.budgetMs), + }; - const nowMs = now(); - const second = Math.floor(nowMs / 1_000); - const key = `${BUCKET_PREFIX}${op}:${second}`; - const count = await runRedisWithinDeadline( - () => redis.incr(key), - redisOptions, - ); - throwIfAborted(options.signal); - if (count === 1) { - await runRedisWithinDeadline( - () => redis.pexpire(key, BUCKET_TTL_MS), - redisOptions, - ); - throwIfAborted(options.signal); + for (;;) { + throwIfAborted(options.signal); + const poisoned = await runRedisWithinDeadline( + () => redis.pttl(`${POISON_PREFIX}${op}`), + redisOptions, + ); + throwIfAborted(options.signal); + if (poisoned > 0) { + if (now() + poisoned > deadline) + throw new MicrovmOpThrottledError(op, options.budgetMs); + await sleep(poisoned, options.signal); + continue; + } + + const nowMs = now(); + const second = Math.floor(nowMs / 1_000); + const key = `${BUCKET_PREFIX}${op}:${second}`; + const count = await runRedisWithinDeadline( + () => redis.incr(key), + redisOptions, + ); + throwIfAborted(options.signal); + if (count === 1) { + await runRedisWithinDeadline( + () => redis.pexpire(key, BUCKET_TTL_MS), + redisOptions, + ); + throwIfAborted(options.signal); + } + if (count <= options.limitPerSecond) return; + + const nextSecondMs = (second + 1) * 1_000 - nowMs; + const jitter = Math.floor(Math.random() * 100); + const waitMs = nextSecondMs + jitter; + if (nowMs + waitMs > deadline) + throw new MicrovmOpThrottledError(op, options.budgetMs); + await sleep(waitMs, options.signal); } - if (count <= options.limitPerSecond) return; - - const nextSecondMs = (second + 1) * 1_000 - nowMs; - const jitter = Math.floor(Math.random() * 100); - const waitMs = nextSecondMs + jitter; - if (nowMs + waitMs > deadline) throw new MicrovmOpThrottledError(op, options.budgetMs); - await sleep(waitMs, options.signal); - } } /** Called when the SDK reports ThrottlingException/TooManyRequests: back the * whole fleet off `op` briefly instead of hammering per-second buckets. */ export async function poisonOpBucket( - op: ThrottledOp, - durationMs: number = DEFAULT_POISON_MS, - options: PoisonOpBucketOptions = {}, + op: ThrottledOp, + durationMs: number = DEFAULT_POISON_MS, + options: PoisonOpBucketOptions = {}, ): Promise { - const now = options.now ?? Date.now; - await runRedisWithinDeadline( - () => redis.set(`${POISON_PREFIX}${op}`, '1', 'PX', durationMs), - { - deadlineAtMs: options.deadlineAtMs, - now, - signal: options.signal, - timeoutError: () => new Error(`Lambda MicroVM ${op} poison deadline exceeded`), - }, - ); + const now = options.now ?? Date.now; + await runRedisWithinDeadline( + () => redis.set(`${POISON_PREFIX}${op}`, '1', 'PX', durationMs), + { + deadlineAtMs: options.deadlineAtMs, + now, + signal: options.signal, + timeoutError: () => + new Error(`Lambda MicroVM ${op} poison deadline exceeded`), + }, + ); } diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 562e06d..6f67e8b 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -2,11 +2,16 @@ * Redis-backed state for Programmatic Tool Calling (PTC) replay mode. * * Owns every key the replay path mutates: - * - `exec_state:` ExecutionState (request inputs + bookkeeping) - * - `tool_history:` Hash of cached tool results, keyed by call_id - * - `exec_lock:` Per-execution mutex protecting the above pair - * - `exec_result:` Blocking-mode terminal result (kept here so - * blocking and replay share one cleanup path) + * - `exec_state:{}` ExecutionState (request inputs + bookkeeping) + * - `tool_history:{}` Hash of cached tool results, keyed by call_id + * - `exec_lock:{}` Per-execution mutex protecting the above pair + * - `exec_result:{}` Blocking-mode terminal result (kept here so + * blocking and replay share one cleanup path) + * + * The `{}` hash tag (see `hashTag` in `redis-connection.ts`) makes every + * key for a given execution land on the same Redis Cluster slot, which + * `commitToolHistoryAndState`'s MULTI/EXEC transaction and the + * `setExecutionResultScript`/`setExecutionErrorScript` Lua scripts require. * * The router (`programmatic-router.ts`) orchestrates HTTP routing and sandbox * iteration; this module owns everything between the router and Redis. All @@ -26,12 +31,18 @@ import type * as t from '../types'; import type { LCTool } from '../preamble'; import { connection } from '../queue'; import { env } from '../config'; +import { + hashTag, + scanKeys, + stripHashTag, + type RedisClient, +} from '../redis-connection'; import { internalServiceHeaders } from '../internal-service-auth'; import logger from '../logger'; import { - ptcReplayHistorySize, - ptcReplayHistoryEntries, - ptcReplayStaleCleanups, + ptcReplayHistorySize, + ptcReplayHistoryEntries, + ptcReplayStaleCleanups, } from '../metrics'; // --------------------------------------------------------------------------- @@ -54,7 +65,10 @@ export const MAX_EXECUTION_STATE_BYTES = 10_000_000; * expire while the holder is still executing and a second continuation * would be able to mutate `tool_history`/`exec_state` concurrently. Floor * at 10 minutes so short job timeouts don't produce a tiny lock window. */ -export const REPLAY_LOCK_TTL_MS = Math.max(10 * 60 * 1000, env.JOB_TIMEOUT * 2 + 30_000); +export const REPLAY_LOCK_TTL_MS = Math.max( + 10 * 60 * 1000, + env.JOB_TIMEOUT * 2 + 30_000, +); /** Per-entry cap for a single serialized tool result (JSON bytes). Keeps the * Redis hash and the `_ptc_history.json` injected into the sandbox bounded @@ -88,85 +102,85 @@ export const CALL_ID_RE = /^call_\d{3,6}$/; // --------------------------------------------------------------------------- export interface ExecutionState { - execution_id: string; - session_id: string; - sessionKey?: string; - userId: string; - tenantId?: string; - canonicalUserId?: string; - orgId?: string; - serviceId?: string; - externalUserId?: string; - principalSource?: string; - authContextHash?: string; - /** Optional even though current construction sites always set a string, - * because `ExecutionState` is JSON-deserialized from Redis and entries - * persisted by prior binaries (or future schema migrations) can omit - * it. The two continuation paths intentionally bypass the apiKeyId - * ownership check when the persisted value is missing — see - * `checkContinuationPreconditions` and the blocking continuation auth - * in `programmatic-router.ts`. Tighten on both paths in a follow-up - * after one `EXECUTION_STATE_TTL` window post a trusted-source - * apiKeyId invariant. */ - apiKeyId?: string; - startTime: number; - /** - * Wall-clock ms of the last interaction that advanced this execution (initial - * submit, continuation accept, or sandbox iteration finish). Used by the - * stale-execution sweeper so long-running but still-active replay flows - * aren't reaped purely based on total age. - */ - lastActivity?: number; - mode: 'blocking' | 'replay'; - /** Replay-only: persisted so continuations can re-enqueue without the client re-sending code. */ - userCode?: string; - tools?: LCTool[]; - files?: t.RequestFile[]; - isPyPlot?: boolean; - timeout?: number; - callCount?: number; - /** Running total of bytes persisted into `tool_history:`. - * Tracked so continuations can reject oversized histories without scanning - * the full hash on every request. */ - historyBytes?: number; - /** Cumulative set of call_ids ever emitted by the sandbox for this - * execution, serialized as an array. Used to reject incoming - * `tool_results` entries whose `call_id` was never actually requested - * — a buggy or malicious client could otherwise pre-seed arbitrary - * ids (e.g. `call_042`) that later replay iterations would consume - * as cache hits and skip real tool calls. Populated in `runAndRespond` - * right before emitting a `tool_call_required` response. */ - emittedCallIds?: string[]; - /** Replay-only: metadata for emitted tool calls, keyed by `id` at use sites. - * Stored so continuation commits can persist the original tool name and a - * compact input signature beside the result. Bash replay uses that metadata - * to safely resolve background tool calls whose completion order can differ - * between runs. */ - emittedToolCalls?: Array<{ - id: string; - name: string; - input_hash?: string; - call_site?: string; - }>; - /** Target language for PTC. Defaults to 'python'. Bash is replay-only. */ - language?: 'python' | 'bash'; - /** Blocking-only legacy fields. New blocking results live in - * `exec_result:` (see setBlockingResult); `jobResult` is kept on the - * interface only as a deploy-time fallback so executions whose state was - * persisted by an older binary still resolve correctly while in-flight. */ - jobCompleted?: boolean; - jobResult?: t.ExecuteResult; - jobError?: string; + execution_id: string; + session_id: string; + sessionKey?: string; + userId: string; + tenantId?: string; + canonicalUserId?: string; + orgId?: string; + serviceId?: string; + externalUserId?: string; + principalSource?: string; + authContextHash?: string; + /** Optional even though current construction sites always set a string, + * because `ExecutionState` is JSON-deserialized from Redis and entries + * persisted by prior binaries (or future schema migrations) can omit + * it. The two continuation paths intentionally bypass the apiKeyId + * ownership check when the persisted value is missing — see + * `checkContinuationPreconditions` and the blocking continuation auth + * in `programmatic-router.ts`. Tighten on both paths in a follow-up + * after one `EXECUTION_STATE_TTL` window post a trusted-source + * apiKeyId invariant. */ + apiKeyId?: string; + startTime: number; + /** + * Wall-clock ms of the last interaction that advanced this execution (initial + * submit, continuation accept, or sandbox iteration finish). Used by the + * stale-execution sweeper so long-running but still-active replay flows + * aren't reaped purely based on total age. + */ + lastActivity?: number; + mode: 'blocking' | 'replay'; + /** Replay-only: persisted so continuations can re-enqueue without the client re-sending code. */ + userCode?: string; + tools?: LCTool[]; + files?: t.RequestFile[]; + isPyPlot?: boolean; + timeout?: number; + callCount?: number; + /** Running total of bytes persisted into `tool_history:`. + * Tracked so continuations can reject oversized histories without scanning + * the full hash on every request. */ + historyBytes?: number; + /** Cumulative set of call_ids ever emitted by the sandbox for this + * execution, serialized as an array. Used to reject incoming + * `tool_results` entries whose `call_id` was never actually requested + * — a buggy or malicious client could otherwise pre-seed arbitrary + * ids (e.g. `call_042`) that later replay iterations would consume + * as cache hits and skip real tool calls. Populated in `runAndRespond` + * right before emitting a `tool_call_required` response. */ + emittedCallIds?: string[]; + /** Replay-only: metadata for emitted tool calls, keyed by `id` at use sites. + * Stored so continuation commits can persist the original tool name and a + * compact input signature beside the result. Bash replay uses that metadata + * to safely resolve background tool calls whose completion order can differ + * between runs. */ + emittedToolCalls?: Array<{ + id: string; + name: string; + input_hash?: string; + call_site?: string; + }>; + /** Target language for PTC. Defaults to 'python'. Bash is replay-only. */ + language?: 'python' | 'bash'; + /** Blocking-only legacy fields. New blocking results live in + * `exec_result:` (see setBlockingResult); `jobResult` is kept on the + * interface only as a deploy-time fallback so executions whose state was + * persisted by an older binary still resolve correctly while in-flight. */ + jobCompleted?: boolean; + jobResult?: t.ExecuteResult; + jobError?: string; } export interface HistoryEntry { - result: unknown; - is_error?: boolean; - error_message?: string; - received_at: number; - tool_name?: string; - input_hash?: string; - call_site?: string; + result: unknown; + is_error?: boolean; + error_message?: string; + received_at: number; + tool_name?: string; + input_hash?: string; + call_site?: string; } /** Describes how an incoming `tool_results` batch would interact with the @@ -174,15 +188,15 @@ export interface HistoryEntry { * Redis mutation so the caller can cheaply enforce caps without risking a * half-applied update. */ export interface ToolHistoryDelta { - /** Serialized entries keyed by call_id, ready to pass to `HSET`. */ - serializedByCallId: Map; - /** Call_ids that weren't previously persisted. */ - newCallIds: string[]; - /** Signed byte delta this batch would apply to the aggregate history - * size: `sum(newEntryBytes)` for new call_ids plus - * `sum(newEntryBytes - oldEntryBytes)` for overwrites. Negative when a - * retry shrinks a previously-persisted entry. */ - bytesDelta: number; + /** Serialized entries keyed by call_id, ready to pass to `HSET`. */ + serializedByCallId: Map; + /** Call_ids that weren't previously persisted. */ + newCallIds: string[]; + /** Signed byte delta this batch would apply to the aggregate history + * size: `sum(newEntryBytes)` for new call_ids plus + * `sum(newEntryBytes - oldEntryBytes)` for overwrites. Negative when a + * retry shrinks a previously-persisted entry. */ + bytesDelta: number; } /** Thrown by `setExecutionState` / `commitToolHistoryAndState` when the @@ -193,14 +207,16 @@ export interface ToolHistoryDelta { * valid request, which is a client/input sizing problem, not a server * failure. */ export class ExecutionStateTooLargeError extends Error { - readonly bytes: number; - readonly cap: number; - constructor(execution_id: string, bytes: number, cap: number) { - super(`ExecutionState for ${execution_id} is ${bytes} bytes, exceeds cap ${cap}`); - this.name = 'ExecutionStateTooLargeError'; - this.bytes = bytes; - this.cap = cap; - } + readonly bytes: number; + readonly cap: number; + constructor(execution_id: string, bytes: number, cap: number) { + super( + `ExecutionState for ${execution_id} is ${bytes} bytes, exceeds cap ${cap}`, + ); + this.name = 'ExecutionStateTooLargeError'; + this.bytes = bytes; + this.cap = cap; + } } // --------------------------------------------------------------------------- @@ -223,52 +239,52 @@ export class ExecutionStateTooLargeError extends Error { * Lua runner, but its Lua coverage is a subset of real Redis (notably * partial `redis.call` arg parsing); the integration suite still runs * against a real Redis container so any divergence surfaces there. */ -type RedisWithScripts = Redis & { - releaseExecutionLockScript(lockKey: string, token: string): Promise; - setExecutionResultScript( - stateKey: string, - resultKey: string, - stateJson: string, - resultJson: string, - ttlSeconds: string, - ): Promise; - setExecutionErrorScript( - stateKey: string, - stateJson: string, - ttlSeconds: string, - ): Promise; +type RedisWithScripts = RedisClient & { + releaseExecutionLockScript(lockKey: string, token: string): Promise; + setExecutionResultScript( + stateKey: string, + resultKey: string, + stateJson: string, + resultJson: string, + ttlSeconds: string, + ): Promise; + setExecutionErrorScript( + stateKey: string, + stateJson: string, + ttlSeconds: string, + ): Promise; }; const SCRIPTS_REGISTERED = Symbol.for('replay-state.scriptsRegistered'); -function registerScripts(client: Redis): RedisWithScripts { - const tagged = client as Redis & { [SCRIPTS_REGISTERED]?: true }; - if (tagged[SCRIPTS_REGISTERED]) return client as RedisWithScripts; - client.defineCommand('releaseExecutionLockScript', { - numberOfKeys: 1, - lua: "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end", - }); - client.defineCommand('setExecutionResultScript', { - numberOfKeys: 2, - lua: `if redis.call('exists', KEYS[1]) == 1 then +function registerScripts(client: RedisClient): RedisWithScripts { + const tagged = client as RedisClient & { [SCRIPTS_REGISTERED]?: true }; + if (tagged[SCRIPTS_REGISTERED]) return client as RedisWithScripts; + client.defineCommand('releaseExecutionLockScript', { + numberOfKeys: 1, + lua: "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end", + }); + client.defineCommand('setExecutionResultScript', { + numberOfKeys: 2, + lua: `if redis.call('exists', KEYS[1]) == 1 then redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[3]) redis.call('set', KEYS[2], ARGV[2], 'EX', ARGV[3]) return 1 else return 0 end`, - }); - client.defineCommand('setExecutionErrorScript', { - numberOfKeys: 1, - lua: `if redis.call('exists', KEYS[1]) == 1 then + }); + client.defineCommand('setExecutionErrorScript', { + numberOfKeys: 1, + lua: `if redis.call('exists', KEYS[1]) == 1 then redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[2]) return 1 else return 0 end`, - }); - tagged[SCRIPTS_REGISTERED] = true; - return client as RedisWithScripts; + }); + tagged[SCRIPTS_REGISTERED] = true; + return client as RedisWithScripts; } /** Module-level Redis handle. Defaults to the shared queue connection so @@ -282,13 +298,13 @@ let redis: RedisWithScripts = registerScripts(connection); * `setRedisForTests`/`resetRedisForTests` lets a test suite isolate per-test * keyspaces by handing in fresh mock instances. */ export function setRedisForTests(client: Redis): void { - redis = registerScripts(client); + redis = registerScripts(client); } /** Restore the production Redis handle. Useful after a test suite finishes * so accidental reuse doesn't leak a closed mock connection. */ export function resetRedisForTests(): void { - redis = registerScripts(connection); + redis = registerScripts(connection); } // --------------------------------------------------------------------------- @@ -299,38 +315,44 @@ export function resetRedisForTests(): void { * key; normalize on read. Safe to delete one EXECUTION_STATE_TTL window * after the external_user_id rollout. */ export function normalizeExecutionState(state: ExecutionState): ExecutionState { - const legacy = (state as Record)['chcUserId']; // leak-check:allow - if (state.externalUserId == null && typeof legacy === 'string') { - state.externalUserId = legacy; - } - return state; + const legacy = (state as Record)['chcUserId']; // leak-check:allow + if (state.externalUserId == null && typeof legacy === 'string') { + state.externalUserId = legacy; + } + return state; } -export async function getExecutionState(execution_id: string): Promise { - const data = await redis.get(`exec_state:${execution_id}`); - return data != null ? normalizeExecutionState(JSON.parse(data) as ExecutionState) : null; +export async function getExecutionState( + execution_id: string, +): Promise { + const data = await redis.get(`exec_state:${hashTag(execution_id)}`); + return data != null + ? normalizeExecutionState(JSON.parse(data) as ExecutionState) + : null; } export async function setExecutionState(state: ExecutionState): Promise { - const serialized = JSON.stringify(state); - const serializedBytes = Buffer.byteLength(serialized, 'utf8'); - if (serializedBytes > MAX_EXECUTION_STATE_BYTES) { - throw new ExecutionStateTooLargeError( - state.execution_id, - serializedBytes, - MAX_EXECUTION_STATE_BYTES, + const serialized = JSON.stringify(state); + const serializedBytes = Buffer.byteLength(serialized, 'utf8'); + if (serializedBytes > MAX_EXECUTION_STATE_BYTES) { + throw new ExecutionStateTooLargeError( + state.execution_id, + serializedBytes, + MAX_EXECUTION_STATE_BYTES, + ); + } + await redis.set( + `exec_state:${hashTag(state.execution_id)}`, + serialized, + 'EX', + EXECUTION_STATE_TTL, ); - } - await redis.set( - `exec_state:${state.execution_id}`, - serialized, - 'EX', - EXECUTION_STATE_TTL, - ); } -export async function deleteExecutionState(execution_id: string): Promise { - await redis.del(`exec_state:${execution_id}`); +export async function deleteExecutionState( + execution_id: string, +): Promise { + await redis.del(`exec_state:${hashTag(execution_id)}`); } // --------------------------------------------------------------------------- @@ -342,53 +364,32 @@ export async function deleteExecutionState(execution_id: string): Promise * lock token on success (used to release only the current holder) or `null` if * another request holds the lock. Prevents TOCTOU on concurrent continuations. */ -export async function acquireExecutionLock(execution_id: string): Promise { - const token = nanoid(); - const result = await redis.set( - `exec_lock:${execution_id}`, - token, - 'PX', - REPLAY_LOCK_TTL_MS, - 'NX', - ); - return result === 'OK' ? token : null; -} - -export async function releaseExecutionLock(execution_id: string, token: string): Promise { - try { - await redis.releaseExecutionLockScript(`exec_lock:${execution_id}`, token); - } catch (err) { - logger.warn('Failed to release exec lock', { execution_id, err }); - } +export async function acquireExecutionLock( + execution_id: string, +): Promise { + const token = nanoid(); + const result = await redis.set( + `exec_lock:${hashTag(execution_id)}`, + token, + 'PX', + REPLAY_LOCK_TTL_MS, + 'NX', + ); + return result === 'OK' ? token : null; } -// --------------------------------------------------------------------------- -// Generic key scan helper -// --------------------------------------------------------------------------- - -/** Iterate matching Redis keys with SCAN instead of the blocking KEYS command. - * Stops collecting once `limit` keys have been gathered to keep the array - * bounded on degenerate datasets. */ -export async function scanKeys( - match: string, - count = 200, - limit = SCAN_KEYS_DEFAULT_LIMIT, -): Promise { - const stream = redis.scanStream({ match, count }); - const out: string[] = []; - for await (const batch of stream as AsyncIterable) { - for (const key of batch) { - out.push(key); - if (out.length >= limit) { - stream.destroy(); - logger.warn('scanKeys hit limit; remaining keys deferred to next pass', { - match, limit, - }); - return out; - } +export async function releaseExecutionLock( + execution_id: string, + token: string, +): Promise { + try { + await redis.releaseExecutionLockScript( + `exec_lock:${hashTag(execution_id)}`, + token, + ); + } catch (err) { + logger.warn('Failed to release exec lock', { execution_id, err }); } - } - return out; } // --------------------------------------------------------------------------- @@ -406,25 +407,32 @@ export async function scanKeys( * on the lighter `exec_state:` blob the replay path mutates on every * continuation. */ function blockingResultKey(execution_id: string): string { - return `exec_result:${execution_id}`; + return `exec_result:${hashTag(execution_id)}`; } -export async function setBlockingResult(execution_id: string, result: t.ExecuteResult): Promise { - await redis.set( - blockingResultKey(execution_id), - JSON.stringify(result), - 'EX', - EXECUTION_STATE_TTL, - ); +export async function setBlockingResult( + execution_id: string, + result: t.ExecuteResult, +): Promise { + await redis.set( + blockingResultKey(execution_id), + JSON.stringify(result), + 'EX', + EXECUTION_STATE_TTL, + ); } -export async function getBlockingResult(execution_id: string): Promise { - const data = await redis.get(blockingResultKey(execution_id)); - return data != null ? (JSON.parse(data) as t.ExecuteResult) : null; +export async function getBlockingResult( + execution_id: string, +): Promise { + const data = await redis.get(blockingResultKey(execution_id)); + return data != null ? (JSON.parse(data) as t.ExecuteResult) : null; } -export async function deleteBlockingResult(execution_id: string): Promise { - await redis.del(blockingResultKey(execution_id)); +export async function deleteBlockingResult( + execution_id: string, +): Promise { + await redis.del(blockingResultKey(execution_id)); } /** Persist the terminal result of a blocking execution. @@ -442,51 +450,71 @@ export async function deleteBlockingResult(execution_id: string): Promise * and, only if so, writes BOTH the updated state (with `jobCompleted=true`) * and the result blob in a single hop. If cleanup has already removed the * state, the entire update is skipped. */ -export async function setExecutionResult(execution_id: string, result: t.ExecuteResult): Promise { - const stateKey = `exec_state:${execution_id}`; - const resultKey = `exec_result:${execution_id}`; - const existing = await getExecutionState(execution_id); - if (!existing) return; - const updated: ExecutionState = { ...existing, jobCompleted: true, jobResult: undefined }; - const updatedSerialized = JSON.stringify(updated); - if (Buffer.byteLength(updatedSerialized, 'utf8') > MAX_EXECUTION_STATE_BYTES) { - throw new ExecutionStateTooLargeError( - execution_id, - Buffer.byteLength(updatedSerialized, 'utf8'), - MAX_EXECUTION_STATE_BYTES, +export async function setExecutionResult( + execution_id: string, + result: t.ExecuteResult, +): Promise { + const stateKey = `exec_state:${hashTag(execution_id)}`; + const resultKey = `exec_result:${hashTag(execution_id)}`; + const existing = await getExecutionState(execution_id); + if (!existing) return; + const updated: ExecutionState = { + ...existing, + jobCompleted: true, + jobResult: undefined, + }; + const updatedSerialized = JSON.stringify(updated); + if ( + Buffer.byteLength(updatedSerialized, 'utf8') > MAX_EXECUTION_STATE_BYTES + ) { + throw new ExecutionStateTooLargeError( + execution_id, + Buffer.byteLength(updatedSerialized, 'utf8'), + MAX_EXECUTION_STATE_BYTES, + ); + } + /** EXISTS-then-SET both keys atomically so a concurrent + * `cleanupExecution` racing between our `getExecutionState` above and + * the writes below cannot leave an orphan blob behind. The Lua body + * is registered on the client via `defineCommand`. */ + await redis.setExecutionResultScript( + stateKey, + resultKey, + updatedSerialized, + JSON.stringify(result), + String(EXECUTION_STATE_TTL), ); - } - /** EXISTS-then-SET both keys atomically so a concurrent - * `cleanupExecution` racing between our `getExecutionState` above and - * the writes below cannot leave an orphan blob behind. The Lua body - * is registered on the client via `defineCommand`. */ - await redis.setExecutionResultScript( - stateKey, - resultKey, - updatedSerialized, - JSON.stringify(result), - String(EXECUTION_STATE_TTL), - ); -} - -export async function setExecutionError(execution_id: string, error: Error): Promise { - const stateKey = `exec_state:${execution_id}`; - const existing = await getExecutionState(execution_id); - if (!existing) return; - const updated: ExecutionState = { ...existing, jobCompleted: true, jobError: error.message }; - const serialized = JSON.stringify(updated); - if (Buffer.byteLength(serialized, 'utf8') > MAX_EXECUTION_STATE_BYTES) { - throw new ExecutionStateTooLargeError( - execution_id, - Buffer.byteLength(serialized, 'utf8'), - MAX_EXECUTION_STATE_BYTES, +} + +export async function setExecutionError( + execution_id: string, + error: Error, +): Promise { + const stateKey = `exec_state:${hashTag(execution_id)}`; + const existing = await getExecutionState(execution_id); + if (!existing) return; + const updated: ExecutionState = { + ...existing, + jobCompleted: true, + jobError: error.message, + }; + const serialized = JSON.stringify(updated); + if (Buffer.byteLength(serialized, 'utf8') > MAX_EXECUTION_STATE_BYTES) { + throw new ExecutionStateTooLargeError( + execution_id, + Buffer.byteLength(serialized, 'utf8'), + MAX_EXECUTION_STATE_BYTES, + ); + } + /** Same race window as `setExecutionResult`: a `cleanupExecution` + * winning between our `getExecutionState` and the write would + * otherwise resurrect torn-down state. The conditional SET makes + * the write a no-op once the execution has been cleaned up. */ + await redis.setExecutionErrorScript( + stateKey, + serialized, + String(EXECUTION_STATE_TTL), ); - } - /** Same race window as `setExecutionResult`: a `cleanupExecution` - * winning between our `getExecutionState` and the write would - * otherwise resurrect torn-down state. The conditional SET makes - * the write a no-op once the execution has been cleaned up. */ - await redis.setExecutionErrorScript(stateKey, serialized, String(EXECUTION_STATE_TTL)); } // --------------------------------------------------------------------------- @@ -494,7 +522,7 @@ export async function setExecutionError(execution_id: string, error: Error): Pro // --------------------------------------------------------------------------- export function historyKey(execution_id: string): string { - return `tool_history:${execution_id}`; + return `tool_history:${hashTag(execution_id)}`; } /** Canonical byte-comparable form of the "semantic" fields of a history @@ -504,15 +532,15 @@ export function historyKey(execution_id: string): string { * which is sufficient for checking equality of payloads produced from * the same client-supplied object. */ function canonicalEntryBody( - result: unknown, - isError: boolean | undefined, - errorMessage: string | undefined, + result: unknown, + isError: boolean | undefined, + errorMessage: string | undefined, ): string { - return JSON.stringify({ - result, - is_error: isError ?? false, - error_message: errorMessage, - }); + return JSON.stringify({ + result, + is_error: isError ?? false, + error_message: errorMessage, + }); } /** Compute the delta between an incoming result batch and the persisted @@ -535,82 +563,93 @@ function canonicalEntryBody( * 409-style mismatch error and no Redis mutation is attempted. */ export async function computeToolHistoryDelta( - execution_id: string, - results: Array<{ - call_id: string; - result: unknown; - is_error?: boolean; - error_message?: string; - tool_name?: string; - input_hash?: string; - call_site?: string; - }>, + execution_id: string, + results: Array<{ + call_id: string; + result: unknown; + is_error?: boolean; + error_message?: string; + tool_name?: string; + input_hash?: string; + call_site?: string; + }>, ): Promise { - const serializedByCallId = new Map(); - if (results.length === 0) { - return { serializedByCallId, newCallIds: [], bytesDelta: 0 }; - } - const key = historyKey(execution_id); - const callIds = results.map(r => r.call_id); - const existingRaw = callIds.length > 0 ? await redis.hmget(key, ...callIds) : []; - const oldByCallId = new Map(); - for (let i = 0; i < callIds.length; i++) { - const raw = existingRaw[i]; - if (typeof raw !== 'string') continue; - try { - const parsed = JSON.parse(raw) as HistoryEntry; - oldByCallId.set(callIds[i], { - entry: parsed, - bytes: Buffer.byteLength(raw, 'utf8'), - }); - } catch { - logger.warn('Malformed existing tool_history entry; treating as absent', { - execution_id, - call_id: callIds[i], - }); + const serializedByCallId = new Map(); + if (results.length === 0) { + return { serializedByCallId, newCallIds: [], bytesDelta: 0 }; } - } - const now = Date.now(); - const newCallIds: string[] = []; - let bytesDelta = 0; - for (const r of results) { - const entry: HistoryEntry = { - result: r.result, - is_error: r.is_error ?? false, - error_message: r.error_message, - received_at: now, - tool_name: r.tool_name, - input_hash: r.input_hash, - call_site: r.call_site, - }; - const serialized = JSON.stringify(entry); - const newBytes = Buffer.byteLength(serialized, 'utf8'); - if (newBytes > MAX_TOOL_RESULT_BYTES) { - return { - error: `tool_results[${r.call_id}] serialized entry is ${newBytes} bytes, exceeds per-entry cap ${MAX_TOOL_RESULT_BYTES}`, - }; + const key = historyKey(execution_id); + const callIds = results.map(r => r.call_id); + const existingRaw = + callIds.length > 0 ? await redis.hmget(key, ...callIds) : []; + const oldByCallId = new Map< + string, + { entry: HistoryEntry; bytes: number } + >(); + for (let i = 0; i < callIds.length; i++) { + const raw = existingRaw[i]; + if (typeof raw !== 'string') continue; + try { + const parsed = JSON.parse(raw) as HistoryEntry; + oldByCallId.set(callIds[i], { + entry: parsed, + bytes: Buffer.byteLength(raw, 'utf8'), + }); + } catch { + logger.warn( + 'Malformed existing tool_history entry; treating as absent', + { + execution_id, + call_id: callIds[i], + }, + ); + } } - const existing = oldByCallId.get(r.call_id); - if (existing) { - const incomingBody = canonicalEntryBody(r.result, r.is_error, r.error_message); - const storedBody = canonicalEntryBody( - existing.entry.result, - existing.entry.is_error, - existing.entry.error_message, - ); - if (incomingBody !== storedBody) { - return { - status: 409, - error: `tool_results[${r.call_id}] has already been recorded with a different payload; replay history is immutable`, + const now = Date.now(); + const newCallIds: string[] = []; + let bytesDelta = 0; + for (const r of results) { + const entry: HistoryEntry = { + result: r.result, + is_error: r.is_error ?? false, + error_message: r.error_message, + received_at: now, + tool_name: r.tool_name, + input_hash: r.input_hash, + call_site: r.call_site, }; - } - continue; + const serialized = JSON.stringify(entry); + const newBytes = Buffer.byteLength(serialized, 'utf8'); + if (newBytes > MAX_TOOL_RESULT_BYTES) { + return { + error: `tool_results[${r.call_id}] serialized entry is ${newBytes} bytes, exceeds per-entry cap ${MAX_TOOL_RESULT_BYTES}`, + }; + } + const existing = oldByCallId.get(r.call_id); + if (existing) { + const incomingBody = canonicalEntryBody( + r.result, + r.is_error, + r.error_message, + ); + const storedBody = canonicalEntryBody( + existing.entry.result, + existing.entry.is_error, + existing.entry.error_message, + ); + if (incomingBody !== storedBody) { + return { + status: 409, + error: `tool_results[${r.call_id}] has already been recorded with a different payload; replay history is immutable`, + }; + } + continue; + } + serializedByCallId.set(r.call_id, serialized); + newCallIds.push(r.call_id); + bytesDelta += newBytes; } - serializedByCallId.set(r.call_id, serialized); - newCallIds.push(r.call_id); - bytesDelta += newBytes; - } - return { serializedByCallId, newCallIds, bytesDelta }; + return { serializedByCallId, newCallIds, bytesDelta }; } /** Commit a previously-computed history delta AND the updated execution @@ -621,58 +660,60 @@ export async function computeToolHistoryDelta( * blob must already be validated against `MAX_EXECUTION_STATE_BYTES` * before this is called; failure is logged and propagated. */ export async function commitToolHistoryAndState( - state: ExecutionState, - delta: ToolHistoryDelta, + state: ExecutionState, + delta: ToolHistoryDelta, ): Promise { - const serialized = JSON.stringify(state); - const serializedBytes = Buffer.byteLength(serialized, 'utf8'); - if (serializedBytes > MAX_EXECUTION_STATE_BYTES) { - /** Use the typed error so the continuation handler can distinguish - * a sizing problem (-> 413) from a generic Redis transaction failure - * (-> 503). A plain `Error` here would bubble through to the top-level - * router catch as an opaque 500. */ - throw new ExecutionStateTooLargeError( - state.execution_id, - serializedBytes, - MAX_EXECUTION_STATE_BYTES, - ); - } - const hKey = historyKey(state.execution_id); - const sKey = `exec_state:${state.execution_id}`; - const tx = redis.multi(); - if (delta.serializedByCallId.size > 0) { - const kvs: string[] = []; - for (const [callId, s] of delta.serializedByCallId) { - kvs.push(callId, s); + const serialized = JSON.stringify(state); + const serializedBytes = Buffer.byteLength(serialized, 'utf8'); + if (serializedBytes > MAX_EXECUTION_STATE_BYTES) { + /** Use the typed error so the continuation handler can distinguish + * a sizing problem (-> 413) from a generic Redis transaction failure + * (-> 503). A plain `Error` here would bubble through to the top-level + * router catch as an opaque 500. */ + throw new ExecutionStateTooLargeError( + state.execution_id, + serializedBytes, + MAX_EXECUTION_STATE_BYTES, + ); + } + const hKey = historyKey(state.execution_id); + const sKey = `exec_state:${hashTag(state.execution_id)}`; + const tx = redis.multi(); + if (delta.serializedByCallId.size > 0) { + const kvs: string[] = []; + for (const [callId, s] of delta.serializedByCallId) { + kvs.push(callId, s); + } + tx.hset(hKey, ...kvs); } - tx.hset(hKey, ...kvs); - } - /** Refresh the history-hash TTL on every commit, including idempotent - * retries where `delta.serializedByCallId` is empty. Without this, a - * retry near TTL expiry renews `exec_state` (via the `SET EX` below) - * but leaves `tool_history` on its original expiry; a slow next - * sandbox run could then span the history's original TTL and cause - * subsequent continuations to load an empty history and re-emit - * already-resolved tool calls. `EXPIRE` on a missing hash is a safe - * no-op. */ - tx.expire(hKey, EXECUTION_STATE_TTL); - tx.set(sKey, serialized, 'EX', EXECUTION_STATE_TTL); - const results = await tx.exec(); - if (!results) { - throw new Error(`Redis transaction aborted for execution ${state.execution_id}`); - } - for (const [err] of results) { - if (err) { - throw err; + /** Refresh the history-hash TTL on every commit, including idempotent + * retries where `delta.serializedByCallId` is empty. Without this, a + * retry near TTL expiry renews `exec_state` (via the `SET EX` below) + * but leaves `tool_history` on its original expiry; a slow next + * sandbox run could then span the history's original TTL and cause + * subsequent continuations to load an empty history and re-emit + * already-resolved tool calls. `EXPIRE` on a missing hash is a safe + * no-op. */ + tx.expire(hKey, EXECUTION_STATE_TTL); + tx.set(sKey, serialized, 'EX', EXECUTION_STATE_TTL); + const results = await tx.exec(); + if (!results) { + throw new Error( + `Redis transaction aborted for execution ${state.execution_id}`, + ); } - } - /** Observe the size and entry-count of the persisted history *after* - * commit succeeds so failed commits don't pollute the histogram with - * sizes that never made it to Redis. `historyBytes` and `callCount` - * have already been advanced on `state` to reflect the post-commit - * truth, so they're the right values to record. */ - ptcReplayHistorySize.observe(state.historyBytes ?? 0); - ptcReplayHistoryEntries.observe(state.callCount ?? 0); + for (const [err] of results) { + if (err) { + throw err; + } + } + /** Observe the size and entry-count of the persisted history *after* + * commit succeeds so failed commits don't pollute the histogram with + * sizes that never made it to Redis. `historyBytes` and `callCount` + * have already been advanced on `state` to reflect the post-commit + * truth, so they're the right values to record. */ + ptcReplayHistorySize.observe(state.historyBytes ?? 0); + ptcReplayHistoryEntries.observe(state.callCount ?? 0); } /** Extend the Redis TTL on both the execution-state blob and the tool-history @@ -680,27 +721,35 @@ export async function commitToolHistoryAndState( * still valid but prior tool results have already expired, which would force * the sandbox to re-emit earlier calls on replay. */ export async function refreshExecutionTtl(execution_id: string): Promise { - await Promise.all([ - redis.expire(`exec_state:${execution_id}`, EXECUTION_STATE_TTL), - redis.expire(historyKey(execution_id), EXECUTION_STATE_TTL), - ]); + await Promise.all([ + redis.expire( + `exec_state:${hashTag(execution_id)}`, + EXECUTION_STATE_TTL, + ), + redis.expire(historyKey(execution_id), EXECUTION_STATE_TTL), + ]); } -export async function loadToolHistory(execution_id: string): Promise> { - const raw = await redis.hgetall(historyKey(execution_id)); - const out: Record = {}; - for (const [k, v] of Object.entries(raw)) { - try { - out[k] = JSON.parse(v) as HistoryEntry; - } catch { - logger.warn('Dropping malformed tool_history entry', { execution_id, call_id: k }); +export async function loadToolHistory( + execution_id: string, +): Promise> { + const raw = await redis.hgetall(historyKey(execution_id)); + const out: Record = {}; + for (const [k, v] of Object.entries(raw)) { + try { + out[k] = JSON.parse(v) as HistoryEntry; + } catch { + logger.warn('Dropping malformed tool_history entry', { + execution_id, + call_id: k, + }); + } } - } - return out; + return out; } export async function deleteToolHistory(execution_id: string): Promise { - await redis.del(historyKey(execution_id)); + await redis.del(historyKey(execution_id)); } // --------------------------------------------------------------------------- @@ -714,117 +763,155 @@ export async function deleteToolHistory(execution_id: string): Promise { const CLEANUP_BATCH_SIZE = 200; export async function cleanupStaleExecutions(): Promise { - try { - const keys = await scanKeys('exec_state:*'); - if (keys.length === 0) return 0; - const now = Date.now(); - let cleaned = 0; - - /** Batch the per-key state reads via `MGET` so a sweep over many - * exec_state keys completes in O(keys / batch) Redis round-trips - * rather than O(keys) sequential `GET`s. The decision-and-delete - * step still iterates per-execution because the deletes (history, - * blocking result, tool_call_server session, exec_state key) are - * heterogeneous and each touches different Redis keyspaces. */ - for (let offset = 0; offset < keys.length; offset += CLEANUP_BATCH_SIZE) { - const batchKeys = keys.slice(offset, offset + CLEANUP_BATCH_SIZE); - const values = await redis.mget(...batchKeys); - for (let i = 0; i < batchKeys.length; i++) { - const data = values[i]; - if (data == null) continue; - - let state: ExecutionState; - try { - state = JSON.parse(data) as ExecutionState; - } catch { - /** Malformed JSON in `exec_state:` (corrupt write or partial - * legacy migration) — drop it AND its sibling `tool_history` / - * `exec_result` keys instead of breaking the whole sweep. - * Earlier we only dropped `exec_state` here, leaving siblings - * orphaned until their independent Redis TTL fired (up to - * `EXECUTION_STATE_TTL` of wasted memory per malformed key). - * The execution id is encoded in the key suffix; we can't - * recover the typed state, but we don't need to in order to - * scope the targeted deletes. - * - * Counted toward `cleaned` and logged at warn level so a burst - * of malformed keys (corrupt writes, mid-deploy migrations) is - * visible to operators via both `ptc_replay_stale_cleanups_total` - * and structured logs, instead of silently disappearing. */ - const orphanId = batchKeys[i].slice('exec_state:'.length); - logger.warn('Reaping malformed exec_state and sibling keys', { - execution_id: orphanId, - }); - await Promise.all([ - redis.del(batchKeys[i]), - deleteToolHistory(orphanId), - deleteBlockingResult(orphanId), - ]).catch(err => { - logger.warn('Sibling delete failed during malformed-key cleanup', { - execution_id: orphanId, - error: err instanceof Error ? err.message : String(err), - }); - }); - cleaned++; - continue; + try { + const keys = await scanKeys( + redis, + 'exec_state:*', + 200, + SCAN_KEYS_DEFAULT_LIMIT, + ); + if (keys.length === 0) return 0; + const now = Date.now(); + let cleaned = 0; + + /** Batch the per-key state reads via `MGET` so a sweep over many + * exec_state keys completes in O(keys / batch) Redis round-trips + * rather than O(keys) sequential `GET`s. The decision-and-delete + * step still iterates per-execution because the deletes (history, + * blocking result, tool_call_server session, exec_state key) are + * heterogeneous and each touches different Redis keyspaces. */ + for ( + let offset = 0; + offset < keys.length; + offset += CLEANUP_BATCH_SIZE + ) { + const batchKeys = keys.slice(offset, offset + CLEANUP_BATCH_SIZE); + const values = await redis.mget(...batchKeys); + for (let i = 0; i < batchKeys.length; i++) { + const data = values[i]; + if (data == null) continue; + + let state: ExecutionState; + try { + state = JSON.parse(data) as ExecutionState; + } catch { + /** Malformed JSON in `exec_state:` (corrupt write or partial + * legacy migration) — drop it AND its sibling `tool_history` / + * `exec_result` keys instead of breaking the whole sweep. + * Earlier we only dropped `exec_state` here, leaving siblings + * orphaned until their independent Redis TTL fired (up to + * `EXECUTION_STATE_TTL` of wasted memory per malformed key). + * The execution id is encoded in the key suffix; we can't + * recover the typed state, but we don't need to in order to + * scope the targeted deletes. + * + * Counted toward `cleaned` and logged at warn level so a burst + * of malformed keys (corrupt writes, mid-deploy migrations) is + * visible to operators via both `ptc_replay_stale_cleanups_total` + * and structured logs, instead of silently disappearing. */ + const orphanId = stripHashTag( + batchKeys[i].slice('exec_state:'.length), + ); + logger.warn( + 'Reaping malformed exec_state and sibling keys', + { + execution_id: orphanId, + }, + ); + await Promise.all([ + redis.del(batchKeys[i]), + deleteToolHistory(orphanId), + deleteBlockingResult(orphanId), + ]).catch(err => { + logger.warn( + 'Sibling delete failed during malformed-key cleanup', + { + execution_id: orphanId, + error: + err instanceof Error + ? err.message + : String(err), + }, + ); + }); + cleaned++; + continue; + } + const lastActivity = state.lastActivity ?? state.startTime; + const idle = now - lastActivity; + + if ( + idle > EXECUTION_STATE_TTL * 1000 && + state.jobCompleted !== true + ) { + logger.warn('Cleaning up stale execution', { + execution_id: state.execution_id, + idleSeconds: idle / 1000, + totalAgeSeconds: (now - state.startTime) / 1000, + mode: state.mode, + }); + + if (state.mode === 'blocking') { + await axios + .delete( + `${env.TOOL_CALL_SERVER_URL}/sessions/${state.execution_id}`, + { + headers: internalServiceHeaders(), + }, + ) + .catch(() => {}); + } + await Promise.all([ + deleteToolHistory(state.execution_id), + deleteBlockingResult(state.execution_id), + redis.del(batchKeys[i]), + ]); + cleaned++; + } + } } - const lastActivity = state.lastActivity ?? state.startTime; - const idle = now - lastActivity; - - if (idle > EXECUTION_STATE_TTL * 1000 && state.jobCompleted !== true) { - logger.warn('Cleaning up stale execution', { - execution_id: state.execution_id, - idleSeconds: idle / 1000, - totalAgeSeconds: (now - state.startTime) / 1000, - mode: state.mode, - }); - - if (state.mode === 'blocking') { - await axios.delete(`${env.TOOL_CALL_SERVER_URL}/sessions/${state.execution_id}`, { - headers: internalServiceHeaders(), - }) - .catch(() => {}); - } - await Promise.all([ - deleteToolHistory(state.execution_id), - deleteBlockingResult(state.execution_id), - redis.del(batchKeys[i]), - ]); - cleaned++; + + if (cleaned > 0) { + logger.info(`Cleaned up ${cleaned} stale executions`); + ptcReplayStaleCleanups.inc(cleaned); } - } + return cleaned; + } catch (error) { + logger.error('Error cleaning up stale executions:', error); + return 0; } +} - if (cleaned > 0) { - logger.info(`Cleaned up ${cleaned} stale executions`); - ptcReplayStaleCleanups.inc(cleaned); - } - return cleaned; - } catch (error) { - logger.error('Error cleaning up stale executions:', error); - return 0; - } -} - -export async function cleanupExecution(execution_id: string, mode: 'blocking' | 'replay'): Promise { - try { - const ops: Array> = [ - deleteExecutionState(execution_id), - deleteToolHistory(execution_id), - deleteBlockingResult(execution_id), - ]; - if (mode === 'blocking') { - ops.push( - axios.delete(`${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}`, { - headers: internalServiceHeaders(), - }).catch(() => {}), - ); +export async function cleanupExecution( + execution_id: string, + mode: 'blocking' | 'replay', +): Promise { + try { + const ops: Array> = [ + deleteExecutionState(execution_id), + deleteToolHistory(execution_id), + deleteBlockingResult(execution_id), + ]; + if (mode === 'blocking') { + ops.push( + axios + .delete( + `${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}`, + { + headers: internalServiceHeaders(), + }, + ) + .catch(() => {}), + ); + } + await Promise.all(ops); + logger.info('Execution cleanup completed', { execution_id, mode }); + } catch (error) { + logger.error('Error during execution cleanup:', { + execution_id, + error, + }); } - await Promise.all(ops); - logger.info('Execution cleanup completed', { execution_id, mode }); - } catch (error) { - logger.error('Error during execution cleanup:', { execution_id, error }); - } } // --------------------------------------------------------------------------- @@ -835,58 +922,72 @@ export async function cleanupExecution(execution_id: string, mode: 'blocking' | * Validate the shape of a single `tool_results` entry. Prevents garbage * (null, wrong types, bad call_ids) from being persisted into Redis history. */ -export function validateToolResult( - r: unknown, -): - | { call_id: string; result: unknown; is_error?: boolean; error_message?: string } - | { error: string } { - if (r == null || typeof r !== 'object' || Array.isArray(r)) { - return { error: 'tool_results entries must be objects' }; - } - const obj = r as Record; - const callId = obj.call_id; - if (typeof callId !== 'string' || !CALL_ID_RE.test(callId)) { - return { error: `tool_results[].call_id must match /^call_\\d{3,6}$/, got: ${JSON.stringify(callId)}` }; - } - if (!('result' in obj)) { - return { error: `tool_results[${callId}].result is required` }; - } - if (obj.is_error !== undefined && typeof obj.is_error !== 'boolean') { - return { error: `tool_results[${callId}].is_error must be a boolean if present` }; - } - if (obj.error_message !== undefined && typeof obj.error_message !== 'string') { - return { error: `tool_results[${callId}].error_message must be a string if present` }; - } - let resultSerialized: string | undefined; - try { - resultSerialized = JSON.stringify(obj.result); - } catch (err) { - return { error: `tool_results[${callId}].result is not JSON-serializable` }; - } - /** `JSON.stringify` only throws on circular references; for `undefined`, - * functions, and `Symbol` it returns the JS value `undefined` rather - * than the string `'undefined'`. The earlier `?? 'null'` fallback - * silently coerced these to a 4-byte placeholder and accepted them - * as valid history entries — but the entry would then deserialize - * to `null` on replay, drifting from what the client originally - * sent. Reject explicitly so callers get a clear error. */ - if (resultSerialized === undefined) { - return { - error: `tool_results[${callId}].result is not JSON-serializable (got undefined / function / symbol)`, - }; - } - const resultBytes = Buffer.byteLength(resultSerialized, 'utf8'); - if (resultBytes > MAX_TOOL_RESULT_BYTES) { +export function validateToolResult(r: unknown): + | { + call_id: string; + result: unknown; + is_error?: boolean; + error_message?: string; + } + | { error: string } { + if (r == null || typeof r !== 'object' || Array.isArray(r)) { + return { error: 'tool_results entries must be objects' }; + } + const obj = r as Record; + const callId = obj.call_id; + if (typeof callId !== 'string' || !CALL_ID_RE.test(callId)) { + return { + error: `tool_results[].call_id must match /^call_\\d{3,6}$/, got: ${JSON.stringify(callId)}`, + }; + } + if (!('result' in obj)) { + return { error: `tool_results[${callId}].result is required` }; + } + if (obj.is_error !== undefined && typeof obj.is_error !== 'boolean') { + return { + error: `tool_results[${callId}].is_error must be a boolean if present`, + }; + } + if ( + obj.error_message !== undefined && + typeof obj.error_message !== 'string' + ) { + return { + error: `tool_results[${callId}].error_message must be a string if present`, + }; + } + let resultSerialized: string | undefined; + try { + resultSerialized = JSON.stringify(obj.result); + } catch (err) { + return { + error: `tool_results[${callId}].result is not JSON-serializable`, + }; + } + /** `JSON.stringify` only throws on circular references; for `undefined`, + * functions, and `Symbol` it returns the JS value `undefined` rather + * than the string `'undefined'`. The earlier `?? 'null'` fallback + * silently coerced these to a 4-byte placeholder and accepted them + * as valid history entries — but the entry would then deserialize + * to `null` on replay, drifting from what the client originally + * sent. Reject explicitly so callers get a clear error. */ + if (resultSerialized === undefined) { + return { + error: `tool_results[${callId}].result is not JSON-serializable (got undefined / function / symbol)`, + }; + } + const resultBytes = Buffer.byteLength(resultSerialized, 'utf8'); + if (resultBytes > MAX_TOOL_RESULT_BYTES) { + return { + error: `tool_results[${callId}].result is ${resultBytes} bytes, exceeds per-result cap ${MAX_TOOL_RESULT_BYTES}`, + }; + } return { - error: `tool_results[${callId}].result is ${resultBytes} bytes, exceeds per-result cap ${MAX_TOOL_RESULT_BYTES}`, + call_id: callId, + result: obj.result, + is_error: obj.is_error as boolean | undefined, + error_message: obj.error_message as string | undefined, }; - } - return { - call_id: callId, - result: obj.result, - is_error: obj.is_error as boolean | undefined, - error_message: obj.error_message as string | undefined, - }; } // --------------------------------------------------------------------------- @@ -894,13 +995,13 @@ export function validateToolResult( // --------------------------------------------------------------------------- export interface ValidatedContinuationResult { - call_id: string; - result: unknown; - is_error?: boolean; - error_message?: string; - tool_name?: string; - input_hash?: string; - call_site?: string; + call_id: string; + result: unknown; + is_error?: boolean; + error_message?: string; + tool_name?: string; + input_hash?: string; + call_site?: string; } /** Validate the raw `tool_results` array a continuation request submitted: @@ -908,39 +1009,41 @@ export interface ValidatedContinuationResult { * uniqueness within the batch. Pure; no Redis. Extracted from the router * so the branch coverage is unit-testable. */ -export function validateContinuationBatch(tool_results: unknown[]): - | { ok: true; results: ValidatedContinuationResult[] } - | { ok: false; status: number; error: string } { - if (tool_results.length > MAX_REPLAY_CALLS) { - return { - ok: false, - status: 400, - error: `tool_results has ${tool_results.length} entries; the per-batch cap is ${MAX_REPLAY_CALLS}`, - }; - } - const results: ValidatedContinuationResult[] = []; - const seen = new Set(); - for (const raw of tool_results) { - const v = validateToolResult(raw); - if ('error' in v) { - return { ok: false, status: 400, error: v.error }; +export function validateContinuationBatch( + tool_results: unknown[], +): + | { ok: true; results: ValidatedContinuationResult[] } + | { ok: false; status: number; error: string } { + if (tool_results.length > MAX_REPLAY_CALLS) { + return { + ok: false, + status: 400, + error: `tool_results has ${tool_results.length} entries; the per-batch cap is ${MAX_REPLAY_CALLS}`, + }; } - if (seen.has(v.call_id)) { - return { - ok: false, - status: 400, - error: `Duplicate call_id in tool_results: ${v.call_id}`, - }; + const results: ValidatedContinuationResult[] = []; + const seen = new Set(); + for (const raw of tool_results) { + const v = validateToolResult(raw); + if ('error' in v) { + return { ok: false, status: 400, error: v.error }; + } + if (seen.has(v.call_id)) { + return { + ok: false, + status: 400, + error: `Duplicate call_id in tool_results: ${v.call_id}`, + }; + } + seen.add(v.call_id); + results.push({ + call_id: v.call_id, + result: v.result, + is_error: v.is_error, + error_message: v.error_message, + }); } - seen.add(v.call_id); - results.push({ - call_id: v.call_id, - result: v.result, - is_error: v.is_error, - error_message: v.error_message, - }); - } - return { ok: true, results }; + return { ok: true, results }; } /** Apply the post-state-load pre-checks to a continuation request: @@ -952,57 +1055,66 @@ export function validateContinuationBatch(tool_results: unknown[]): * Redis client. */ export function checkContinuationPreconditions(params: { - state: ExecutionState; - results: ValidatedContinuationResult[]; - userId: string; - apiKeyId?: string; - tenantId?: string; - authContextHash?: string; - delta: ToolHistoryDelta; + state: ExecutionState; + results: ValidatedContinuationResult[]; + userId: string; + apiKeyId?: string; + tenantId?: string; + authContextHash?: string; + delta: ToolHistoryDelta; }): - | { ok: true } - | { ok: false; status: number; error: string; cleanupOnReject?: boolean } { - const { state, results, userId, apiKeyId, tenantId, authContextHash, delta } = params; - if (state.mode !== 'replay') { - return { - ok: false, - status: 400, - error: 'Execution was started in blocking mode; continuation mode mismatch', - }; - } - if ( - state.userId !== userId || - (state.apiKeyId != null && state.apiKeyId !== apiKeyId) || - (state.tenantId != null && state.tenantId !== tenantId) || - (state.authContextHash != null && state.authContextHash !== authContextHash) - ) { - return { ok: false, status: 403, error: 'Forbidden' }; - } - const issued = new Set(state.emittedCallIds ?? []); - for (const r of results) { - if (!issued.has(r.call_id)) { - return { - ok: false, - status: 400, - error: `tool_results[${r.call_id}] was not issued by the sandbox for this execution`, - }; + | { ok: true } + | { ok: false; status: number; error: string; cleanupOnReject?: boolean } { + const { + state, + results, + userId, + apiKeyId, + tenantId, + authContextHash, + delta, + } = params; + if (state.mode !== 'replay') { + return { + ok: false, + status: 400, + error: 'Execution was started in blocking mode; continuation mode mismatch', + }; } - } - if ((state.callCount ?? 0) + delta.newCallIds.length > MAX_REPLAY_CALLS) { - return { - ok: false, - status: 400, - error: `Execution exceeded maximum tool calls (${MAX_REPLAY_CALLS})`, - cleanupOnReject: true, - }; - } - const projectedHistoryBytes = (state.historyBytes ?? 0) + delta.bytesDelta; - if (projectedHistoryBytes > MAX_TOOL_HISTORY_TOTAL_BYTES) { - return { - ok: false, - status: 400, - error: `Aggregate tool_history for this execution would be ${projectedHistoryBytes} bytes, exceeds cap ${MAX_TOOL_HISTORY_TOTAL_BYTES}`, - }; - } - return { ok: true }; + if ( + state.userId !== userId || + (state.apiKeyId != null && state.apiKeyId !== apiKeyId) || + (state.tenantId != null && state.tenantId !== tenantId) || + (state.authContextHash != null && + state.authContextHash !== authContextHash) + ) { + return { ok: false, status: 403, error: 'Forbidden' }; + } + const issued = new Set(state.emittedCallIds ?? []); + for (const r of results) { + if (!issued.has(r.call_id)) { + return { + ok: false, + status: 400, + error: `tool_results[${r.call_id}] was not issued by the sandbox for this execution`, + }; + } + } + if ((state.callCount ?? 0) + delta.newCallIds.length > MAX_REPLAY_CALLS) { + return { + ok: false, + status: 400, + error: `Execution exceeded maximum tool calls (${MAX_REPLAY_CALLS})`, + cleanupOnReject: true, + }; + } + const projectedHistoryBytes = (state.historyBytes ?? 0) + delta.bytesDelta; + if (projectedHistoryBytes > MAX_TOOL_HISTORY_TOTAL_BYTES) { + return { + ok: false, + status: 400, + error: `Aggregate tool_history for this execution would be ${projectedHistoryBytes} bytes, exceeds cap ${MAX_TOOL_HISTORY_TOTAL_BYTES}`, + }; + } + return { ok: true }; } diff --git a/service/src/tool-call-server.ts b/service/src/tool-call-server.ts index 866de62..d6e72c2 100644 --- a/service/src/tool-call-server.ts +++ b/service/src/tool-call-server.ts @@ -1,20 +1,26 @@ -import IORedis from 'ioredis'; import { nanoid } from 'nanoid'; -import type * as tls from 'tls'; import { - httpLatencyElapsedSeconds, - httpLatencyStartMs, - metricsResponse, - recordHttpRequest, - toolCalls, - toolCallTimeouts, - toolCallActiveSessions, + httpLatencyElapsedSeconds, + httpLatencyStartMs, + metricsResponse, + recordHttpRequest, + toolCalls, + toolCallTimeouts, + toolCallActiveSessions, } from './metrics'; -import { internalServiceAuthEnabled, isAuthorizedInternalServiceRequest } from './internal-service-auth'; +import { + internalServiceAuthEnabled, + isAuthorizedInternalServiceRequest, +} from './internal-service-auth'; import { isRegisteredToolName } from './tool-scope'; -import { normalizeTracePath, shutdownTelemetry, withSpan, withTraceContext } from './telemetry'; +import { + normalizeTracePath, + shutdownTelemetry, + withSpan, + withTraceContext, +} from './telemetry'; import logger from './toolCallServerLogger'; -import { redisKeepAliveOptions } from './redis-options'; +import { createRedisConnection, hashTag, scanKeys } from './redis-connection'; const INSTANCE_ID = process.env.INSTANCE_ID ?? nanoid(); const PORT = Number(process.env.TOOL_CALL_SERVER_PORT) || 3033; @@ -22,604 +28,712 @@ const REQUEST_TIMEOUT = Number(process.env.TOOL_CALL_REQUEST_TIMEOUT) || 300000; const SESSION_EXPIRY = Number(process.env.TOOL_CALL_SESSION_EXPIRY) || 600; // 10 minutes in seconds if (!internalServiceAuthEnabled()) { - logger.warn('CODEAPI_INTERNAL_SERVICE_TOKEN is not set; tool-call session management routes are unauthenticated'); + logger.warn( + 'CODEAPI_INTERNAL_SERVICE_TOKEN is not set; tool-call session management routes are unauthenticated', + ); } // Redis connection -const useAltDnsLookup = process.env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP === 'true'; - -const redis = new IORedis({ - host: process.env.REDIS_HOST ?? 'redis', - port: Number(process.env.REDIS_PORT) || 6379, - password: process.env.REDIS_PASSWORD, - enableReadyCheck: false, - tls: process.env.REDIS_TLS === 'true' ? { - rejectUnauthorized: false - } as tls.ConnectionOptions : undefined, - connectTimeout: 10000, - ...redisKeepAliveOptions(), - maxRetriesPerRequest: 3, - retryStrategy(times: number): number { - const delay = Math.min(times * 500, 2000); - return delay; - }, - reconnectOnError(err: Error): boolean { - const targetError = 'READONLY'; - if (err.message.includes(targetError)) { - return true; - } - return false; - }, - // Alternative DNS lookup for AWS ElastiCache TLS connections - ...(useAltDnsLookup - ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } - : {}) +const redis = createRedisConnection({ + enableReadyCheck: false, + maxRetriesPerRequest: 3, + retryStrategy(times: number): number { + return Math.min(times * 500, 2000); + }, + reconnectOnError(err: Error): boolean { + return err.message.includes('READONLY'); + }, }); -redis.on('error', (err) => { - logger.error('Redis Client Error', { error: err }); +redis.on('error', err => { + logger.error('Redis Client Error', { error: err }); }); redis.on('connect', () => { - logger.info('Redis Client Connected', { - host: process.env.REDIS_HOST, - port: process.env.REDIS_PORT - }); + logger.info('Redis Client Connected', { + host: process.env.REDIS_HOST, + port: process.env.REDIS_PORT, + }); }); redis.on('ready', () => { - logger.info('Redis Client Ready'); + logger.info('Redis Client Ready'); }); // Types interface ToolCallSession { - execution_id: string; - session_id: string; - callback_token: string; - status: 'running' | 'waiting' | 'completed' | 'error'; - tools: Array<{ name: string; description?: string; parameters?: Record }>; - created_at: number; - updated_at: number; - timeout: number; + execution_id: string; + session_id: string; + callback_token: string; + status: 'running' | 'waiting' | 'completed' | 'error'; + tools: Array<{ + name: string; + description?: string; + parameters?: Record; + }>; + created_at: number; + updated_at: number; + timeout: number; } interface ToolCallRequest { - call_id: string; - tool_name: string; - tool_input: Record; - timestamp: number; + call_id: string; + tool_name: string; + tool_input: Record; + timestamp: number; } interface ToolCallResult { - call_id: string; - result: unknown; - is_error: boolean; - error_message?: string; - received_at: number; + call_id: string; + result: unknown; + is_error: boolean; + error_message?: string; + received_at: number; } // Helper functions function jsonResponse(data: unknown, status = 200): Response { - return new Response(JSON.stringify(data), { - status, - headers: { 'Content-Type': 'application/json' } - }); + return new Response(JSON.stringify(data), { + status, + headers: { 'Content-Type': 'application/json' }, + }); } function errorResponse(error: string, status = 400): Response { - return jsonResponse({ error }, status); + return jsonResponse({ error }, status); } -async function getSession(executionId: string): Promise { - const data = await redis.get(`tool_call:session:${executionId}`); - return data != null && data !== '' ? JSON.parse(data) : null; +/** All `tool_call:*` keys for a session are hash-tagged with the execution + * id (see `hashTag`) so they land on the same Redis Cluster slot, letting + * `handleDeleteSession` clean them up with a single multi-key `DEL`. */ +function sessionKey(executionId: string): string { + return `tool_call:session:${hashTag(executionId)}`; } -async function setSession(session: ToolCallSession): Promise { - await redis.set( - `tool_call:session:${session.execution_id}`, - JSON.stringify(session), - 'EX', - SESSION_EXPIRY - ); +function pendingKey(executionId: string): string { + return `tool_call:pending:${hashTag(executionId)}`; } -// Route handlers -async function handleCreateSession(req: Request): Promise { - try { - const body = await req.json() as { - execution_id: string; - session_id: string; - timeout?: number; - tools?: ToolCallSession['tools']; - }; - - const { execution_id, session_id, timeout = REQUEST_TIMEOUT, tools = [] } = body; - - if (!execution_id || !session_id) { - return errorResponse('Missing required fields: execution_id, session_id', 400); - } - - const callback_token = nanoid(32); - - const session: ToolCallSession = { - execution_id, - session_id, - callback_token, - status: 'running', - tools, - created_at: Date.now(), - updated_at: Date.now(), - timeout - }; - - await setSession(session); - toolCallActiveSessions.inc(); - - logger.info(`[${INSTANCE_ID}] Session created: ${execution_id}`); - - return jsonResponse({ - success: true, - execution_id, - callback_url: `http://localhost:${PORT}`, - callback_token - }); - } catch (error) { - logger.error('Error creating session:', { error }); - return errorResponse('Internal server error', 500); - } +function resultKey(executionId: string, callId: string): string { + return `tool_call:result:${hashTag(executionId)}:${callId}`; } -async function handleGetPending(executionId: string): Promise { - try { - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); - } +function completeKey(executionId: string): string { + return `tool_call:complete:${hashTag(executionId)}`; +} - // Get pending calls from Redis list - const pendingData = await redis.lrange(`tool_call:pending:${executionId}`, 0, -1); - const pendingCalls: ToolCallRequest[] = pendingData.map(d => JSON.parse(d)); +function errorKey(executionId: string): string { + return `tool_call:error:${hashTag(executionId)}`; +} - return jsonResponse({ - status: session.status, - pending_calls: pendingCalls, - partial_stdout: '', - partial_stderr: '' - }); - } catch (error) { - logger.error('Error getting pending calls:', { error }); - return errorResponse('Internal server error', 500); - } +async function getSession( + executionId: string, +): Promise { + const data = await redis.get(sessionKey(executionId)); + return data != null && data !== '' ? JSON.parse(data) : null; } -async function handleSubmitResults(executionId: string, req: Request): Promise { - try { - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); - } +async function setSession(session: ToolCallSession): Promise { + await redis.set( + sessionKey(session.execution_id), + JSON.stringify(session), + 'EX', + SESSION_EXPIRY, + ); +} - const body = await req.json() as { results: ToolCallResult[] }; - const { results } = body; +// Route handlers +async function handleCreateSession(req: Request): Promise { + try { + const body = (await req.json()) as { + execution_id: string; + session_id: string; + timeout?: number; + tools?: ToolCallSession['tools']; + }; + + const { + execution_id, + session_id, + timeout = REQUEST_TIMEOUT, + tools = [], + } = body; + + if (!execution_id || !session_id) { + return errorResponse( + 'Missing required fields: execution_id, session_id', + 400, + ); + } - if (!Array.isArray(results)) { - return errorResponse('Invalid results format', 400); + const callback_token = nanoid(32); + + const session: ToolCallSession = { + execution_id, + session_id, + callback_token, + status: 'running', + tools, + created_at: Date.now(), + updated_at: Date.now(), + timeout, + }; + + await setSession(session); + toolCallActiveSessions.inc(); + + logger.info(`[${INSTANCE_ID}] Session created: ${execution_id}`); + + return jsonResponse({ + success: true, + execution_id, + callback_url: `http://localhost:${PORT}`, + callback_token, + }); + } catch (error) { + logger.error('Error creating session:', { error }); + return errorResponse('Internal server error', 500); } +} - let processed = 0; +async function handleGetPending(executionId: string): Promise { + try { + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - // Get all pending calls once - const pendingData = await redis.lrange(`tool_call:pending:${executionId}`, 0, -1); + // Get pending calls from Redis list + const pendingData = await redis.lrange(pendingKey(executionId), 0, -1); + const pendingCalls: ToolCallRequest[] = pendingData.map(d => + JSON.parse(d), + ); + + return jsonResponse({ + status: session.status, + pending_calls: pendingCalls, + partial_stdout: '', + partial_stderr: '', + }); + } catch (error) { + logger.error('Error getting pending calls:', { error }); + return errorResponse('Internal server error', 500); + } +} - for (const result of results) { - const resultData: ToolCallResult = { - call_id: result.call_id, - result: result.result, - is_error: result.is_error || false, - error_message: result.error_message, - received_at: Date.now() - }; +async function handleSubmitResults( + executionId: string, + req: Request, +): Promise { + try { + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - // Store result - await redis.set( - `tool_call:result:${executionId}:${result.call_id}`, - JSON.stringify(resultData), - 'EX', - SESSION_EXPIRY - ); + const body = (await req.json()) as { results: ToolCallResult[] }; + const { results } = body; - // Publish for waiting subscribers - await redis.publish( - `tool_call:result:${executionId}:${result.call_id}`, - JSON.stringify(resultData) - ); + if (!Array.isArray(results)) { + return errorResponse('Invalid results format', 400); + } - // Remove from pending list - find the exact string that was stored - const pendingToRemove = pendingData.find(p => { - try { - return JSON.parse(p).call_id === result.call_id; - } catch { - return false; + let processed = 0; + + // Get all pending calls once + const pendingData = await redis.lrange(pendingKey(executionId), 0, -1); + + for (const result of results) { + const resultData: ToolCallResult = { + call_id: result.call_id, + result: result.result, + is_error: result.is_error || false, + error_message: result.error_message, + received_at: Date.now(), + }; + + // Store result + await redis.set( + resultKey(executionId, result.call_id), + JSON.stringify(resultData), + 'EX', + SESSION_EXPIRY, + ); + + // Publish for waiting subscribers + await redis.publish( + resultKey(executionId, result.call_id), + JSON.stringify(resultData), + ); + + // Remove from pending list - find the exact string that was stored + const pendingToRemove = pendingData.find(p => { + try { + return JSON.parse(p).call_id === result.call_id; + } catch { + return false; + } + }); + + if (pendingToRemove != null && pendingToRemove !== '') { + await redis.lrem(pendingKey(executionId), 1, pendingToRemove); + logger.info( + `[${INSTANCE_ID}] Removed pending call ${result.call_id} from ${executionId}`, + ); + } + + processed++; } - }); - if (pendingToRemove != null && pendingToRemove !== '') { - await redis.lrem(`tool_call:pending:${executionId}`, 1, pendingToRemove); - logger.info(`[${INSTANCE_ID}] Removed pending call ${result.call_id} from ${executionId}`); - } + // Update session status + const remainingPending = await redis.llen(pendingKey(executionId)); + if (remainingPending === 0) { + session.status = 'running'; + session.updated_at = Date.now(); + await setSession(session); + } - processed++; - } + logger.info( + `[${INSTANCE_ID}] Results submitted for ${executionId}: ${processed} processed`, + ); - // Update session status - const remainingPending = await redis.llen(`tool_call:pending:${executionId}`); - if (remainingPending === 0) { - session.status = 'running'; - session.updated_at = Date.now(); - await setSession(session); + return jsonResponse({ success: true, processed }); + } catch (error) { + logger.error('Error submitting results:', { error }); + return errorResponse('Internal server error', 500); } - - logger.info(`[${INSTANCE_ID}] Results submitted for ${executionId}: ${processed} processed`); - - return jsonResponse({ success: true, processed }); - } catch (error) { - logger.error('Error submitting results:', { error }); - return errorResponse('Internal server error', 500); - } } async function handleGetStatus(executionId: string): Promise { - try { - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); - } + try { + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - return jsonResponse({ - status: session.status, - execution_id: session.execution_id, - session_id: session.session_id, - created_at: session.created_at, - updated_at: session.updated_at - }); - } catch (error) { - logger.error('Error getting status:', { error }); - return errorResponse('Internal server error', 500); - } + return jsonResponse({ + status: session.status, + execution_id: session.execution_id, + session_id: session.session_id, + created_at: session.created_at, + updated_at: session.updated_at, + }); + } catch (error) { + logger.error('Error getting status:', { error }); + return errorResponse('Internal server error', 500); + } } -async function handleComplete(executionId: string, req: Request): Promise { - try { - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); - } +async function handleComplete( + executionId: string, + req: Request, +): Promise { + try { + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - const body = await req.json() as { - stdout?: string; - stderr?: string; - exit_code?: number; - }; - - const wasActive = session.status !== 'completed' && session.status !== 'error'; - session.status = 'completed'; - session.updated_at = Date.now(); - await setSession(session); - if (wasActive) { - toolCallActiveSessions.dec(); - } + const body = (await req.json()) as { + stdout?: string; + stderr?: string; + exit_code?: number; + }; + + const wasActive = + session.status !== 'completed' && session.status !== 'error'; + session.status = 'completed'; + session.updated_at = Date.now(); + await setSession(session); + if (wasActive) { + toolCallActiveSessions.dec(); + } - // Store completion data - await redis.set( - `tool_call:complete:${executionId}`, - JSON.stringify({ - ...body, - completed_at: Date.now() - }), - 'EX', - SESSION_EXPIRY - ); + // Store completion data + await redis.set( + completeKey(executionId), + JSON.stringify({ + ...body, + completed_at: Date.now(), + }), + 'EX', + SESSION_EXPIRY, + ); - logger.info(`[${INSTANCE_ID}] Execution completed: ${executionId}`); + logger.info(`[${INSTANCE_ID}] Execution completed: ${executionId}`); - return jsonResponse({ success: true }); - } catch (error) { - logger.error('Error marking complete:', { error }); - return errorResponse('Internal server error', 500); - } + return jsonResponse({ success: true }); + } catch (error) { + logger.error('Error marking complete:', { error }); + return errorResponse('Internal server error', 500); + } } -async function handleError(executionId: string, req: Request): Promise { - try { - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); - } +async function handleError( + executionId: string, + req: Request, +): Promise { + try { + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - const body = await req.json() as { - error: string; - error_type?: string; - stderr?: string; - }; - - const wasActive = session.status !== 'completed' && session.status !== 'error'; - session.status = 'error'; - session.updated_at = Date.now(); - await setSession(session); - if (wasActive) { - toolCallActiveSessions.dec(); - } + const body = (await req.json()) as { + error: string; + error_type?: string; + stderr?: string; + }; + + const wasActive = + session.status !== 'completed' && session.status !== 'error'; + session.status = 'error'; + session.updated_at = Date.now(); + await setSession(session); + if (wasActive) { + toolCallActiveSessions.dec(); + } - // Store error data - await redis.set( - `tool_call:error:${executionId}`, - JSON.stringify({ - ...body, - error_at: Date.now() - }), - 'EX', - SESSION_EXPIRY - ); + // Store error data + await redis.set( + errorKey(executionId), + JSON.stringify({ + ...body, + error_at: Date.now(), + }), + 'EX', + SESSION_EXPIRY, + ); - logger.info(`[${INSTANCE_ID}] Execution error: ${executionId}`); + logger.info(`[${INSTANCE_ID}] Execution error: ${executionId}`); - return jsonResponse({ success: true }); - } catch (error) { - logger.error('Error marking error:', { error }); - return errorResponse('Internal server error', 500); - } + return jsonResponse({ success: true }); + } catch (error) { + logger.error('Error marking error:', { error }); + return errorResponse('Internal server error', 500); + } } async function handleDeleteSession(executionId: string): Promise { - try { - // Check session status before deleting — only decrement if still active - const session = await getSession(executionId); - const wasActive = session != null && session.status !== 'completed' && session.status !== 'error'; - - const keys = await redis.keys(`tool_call:*:${executionId}*`); - if (keys.length > 0) { - await redis.del(...keys); - } + try { + // Check session status before deleting — only decrement if still active + const session = await getSession(executionId); + const wasActive = + session != null && + session.status !== 'completed' && + session.status !== 'error'; + + const keys = await scanKeys( + redis, + `tool_call:*${hashTag(executionId)}*`, + ); + if (keys.length > 0) { + await redis.del(...keys); + } - if (wasActive) { - toolCallActiveSessions.dec(); - } - logger.info(`[${INSTANCE_ID}] Session deleted: ${executionId}, cleaned ${keys.length} keys`); + if (wasActive) { + toolCallActiveSessions.dec(); + } + logger.info( + `[${INSTANCE_ID}] Session deleted: ${executionId}, cleaned ${keys.length} keys`, + ); - return jsonResponse({ success: true, cleaned_keys: keys.length }); - } catch (error) { - logger.error('Error deleting session:', { error }); - return errorResponse('Internal server error', 500); - } + return jsonResponse({ success: true, cleaned_keys: keys.length }); + } catch (error) { + logger.error('Error deleting session:', { error }); + return errorResponse('Internal server error', 500); + } } async function handleToolCall(req: Request): Promise { - try { - const executionId = req.headers.get('X-Execution-ID') ?? ''; - const callbackToken = req.headers.get('X-Callback-Token') ?? ''; - const callId = req.headers.get('X-Tool-Call-ID') ?? ''; + try { + const executionId = req.headers.get('X-Execution-ID') ?? ''; + const callbackToken = req.headers.get('X-Callback-Token') ?? ''; + const callId = req.headers.get('X-Tool-Call-ID') ?? ''; - if (!executionId || !callbackToken || !callId) { - return errorResponse('Missing required headers', 400); - } + if (!executionId || !callbackToken || !callId) { + return errorResponse('Missing required headers', 400); + } - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); - } + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - if (session.callback_token !== callbackToken) { - return errorResponse('Invalid callback token', 401); - } + if (session.callback_token !== callbackToken) { + return errorResponse('Invalid callback token', 401); + } - const body = await req.json() as { - tool_name: string; - input: Record; - }; + const body = (await req.json()) as { + tool_name: string; + input: Record; + }; - const { tool_name, input } = body; - if (typeof tool_name !== 'string' || tool_name === '') { - return errorResponse('Invalid tool name', 400); - } - if (!isRegisteredToolName(tool_name, session.tools)) { - logger.warn(`[${INSTANCE_ID}] Rejected unregistered tool call: ${executionId}/${callId} - ${tool_name}`); - return errorResponse('Tool is not registered for this execution', 403); - } + const { tool_name, input } = body; + if (typeof tool_name !== 'string' || tool_name === '') { + return errorResponse('Invalid tool name', 400); + } + if (!isRegisteredToolName(tool_name, session.tools)) { + logger.warn( + `[${INSTANCE_ID}] Rejected unregistered tool call: ${executionId}/${callId} - ${tool_name}`, + ); + return errorResponse( + 'Tool is not registered for this execution', + 403, + ); + } - // Create pending call record - const toolCall: ToolCallRequest = { - call_id: callId, - tool_name, - tool_input: input, - timestamp: Date.now() - }; - - // Add to pending list - await redis.rpush(`tool_call:pending:${executionId}`, JSON.stringify(toolCall)); - - // Update session status - session.status = 'waiting'; - session.updated_at = Date.now(); - await setSession(session); - - toolCalls.inc(); - logger.info(`[${INSTANCE_ID}] Tool call received: ${executionId}/${callId} - ${tool_name}`); - - // Wait for result (blocking) - const result = await waitForResult(executionId, callId, session.timeout); - - if (result === null) { - toolCallTimeouts.inc(); - return jsonResponse({ - success: false, - error: 'timeout', - message: 'Tool call timed out waiting for result' - }, 408); - } + // Create pending call record + const toolCall: ToolCallRequest = { + call_id: callId, + tool_name, + tool_input: input, + timestamp: Date.now(), + }; + + // Add to pending list + await redis.rpush(pendingKey(executionId), JSON.stringify(toolCall)); + + // Update session status + session.status = 'waiting'; + session.updated_at = Date.now(); + await setSession(session); + + toolCalls.inc(); + logger.info( + `[${INSTANCE_ID}] Tool call received: ${executionId}/${callId} - ${tool_name}`, + ); + + // Wait for result (blocking) + const result = await waitForResult( + executionId, + callId, + session.timeout, + ); + + if (result === null) { + toolCallTimeouts.inc(); + return jsonResponse( + { + success: false, + error: 'timeout', + message: 'Tool call timed out waiting for result', + }, + 408, + ); + } - return jsonResponse({ - success: true, - result: result.result, - is_error: result.is_error, - error_message: result.error_message - }); - } catch (error) { - logger.error('Error handling tool call:', { error }); - return errorResponse('Internal server error', 500); - } + return jsonResponse({ + success: true, + result: result.result, + is_error: result.is_error, + error_message: result.error_message, + }); + } catch (error) { + logger.error('Error handling tool call:', { error }); + return errorResponse('Internal server error', 500); + } } async function waitForResult( - executionId: string, - callId: string, - timeout: number + executionId: string, + callId: string, + timeout: number, ): Promise { - const resultKey = `tool_call:result:${executionId}:${callId}`; - const startTime = Date.now(); - const pollInterval = 100; // 100ms - - while (Date.now() - startTime < timeout) { - const result = await redis.get(resultKey); - if (result != null && result !== '') { - await redis.del(resultKey); // Consume the result - return JSON.parse(result); + const resultKeyName = resultKey(executionId, callId); + const startTime = Date.now(); + const pollInterval = 100; // 100ms + + while (Date.now() - startTime < timeout) { + const result = await redis.get(resultKeyName); + if (result != null && result !== '') { + await redis.del(resultKeyName); // Consume the result + return JSON.parse(result); + } + await new Promise(resolve => setTimeout(resolve, pollInterval)); } - await new Promise(resolve => setTimeout(resolve, pollInterval)); - } - return null; + return null; } // Health check async function handleHealth(): Promise { - try { - await redis.ping(); - return jsonResponse({ - status: 'healthy', - redis: true, - uptime: process.uptime(), - instance_id: INSTANCE_ID - }); - } catch (error) { - return jsonResponse({ - status: 'unhealthy', - redis: false, - error: (error as Error).message - }, 503); - } + try { + await redis.ping(); + return jsonResponse({ + status: 'healthy', + redis: true, + uptime: process.uptime(), + instance_id: INSTANCE_ID, + }); + } catch (error) { + return jsonResponse( + { + status: 'unhealthy', + redis: false, + error: (error as Error).message, + }, + 503, + ); + } } // Router -async function routeToolCallRequest(req: Request): Promise<{ response: Response; route: string }> { - const url = new URL(req.url); - const path = url.pathname; - const method = req.method; - - // Health check - if (path === '/health' && method === 'GET') { - return { response: await handleHealth(), route: '/health' }; - } - - // Prometheus metrics - if (path === '/metrics' && method === 'GET') { - const { body, contentType } = await metricsResponse(); - return { response: new Response(body, { status: 200, headers: { 'Content-Type': contentType } }), route: '/metrics' }; - } - - if ((path === '/sessions' || path.startsWith('/sessions/')) && !isAuthorizedInternalServiceRequest(req.headers)) { - return { response: errorResponse('Unauthorized', 401), route: '/sessions/*' }; - } - - // POST /sessions - Create session - if (path === '/sessions' && method === 'POST') { - return { response: await handleCreateSession(req), route: '/sessions' }; - } - - // POST /tool-call - Tool call (blocking) - if (path === '/tool-call' && method === 'POST') { - return { response: await handleToolCall(req), route: '/tool-call' }; - } - - // Session-specific routes - const sessionMatch = path.match(/^\/sessions\/([^/]+)(\/(.+))?$/); - if (sessionMatch) { - const executionId = sessionMatch[1]; - const subPath = sessionMatch[3]; - - // GET /sessions/:id - if (!subPath && method === 'GET') { - return { response: await handleGetStatus(executionId), route: '/sessions/:executionId' }; +async function routeToolCallRequest( + req: Request, +): Promise<{ response: Response; route: string }> { + const url = new URL(req.url); + const path = url.pathname; + const method = req.method; + + // Health check + if (path === '/health' && method === 'GET') { + return { response: await handleHealth(), route: '/health' }; } - // DELETE /sessions/:id - if (!subPath && method === 'DELETE') { - return { response: await handleDeleteSession(executionId), route: '/sessions/:executionId' }; + // Prometheus metrics + if (path === '/metrics' && method === 'GET') { + const { body, contentType } = await metricsResponse(); + return { + response: new Response(body, { + status: 200, + headers: { 'Content-Type': contentType }, + }), + route: '/metrics', + }; } - if (subPath === 'pending' && method === 'GET') { - return { response: await handleGetPending(executionId), route: '/sessions/:executionId/pending' }; + if ( + (path === '/sessions' || path.startsWith('/sessions/')) && + !isAuthorizedInternalServiceRequest(req.headers) + ) { + return { + response: errorResponse('Unauthorized', 401), + route: '/sessions/*', + }; } - // POST /sessions/:id/results - if (subPath === 'results' && method === 'POST') { - return { response: await handleSubmitResults(executionId, req), route: '/sessions/:executionId/results' }; + // POST /sessions - Create session + if (path === '/sessions' && method === 'POST') { + return { response: await handleCreateSession(req), route: '/sessions' }; } - // GET /sessions/:id/status - if (subPath === 'status' && method === 'GET') { - return { response: await handleGetStatus(executionId), route: '/sessions/:executionId/status' }; + // POST /tool-call - Tool call (blocking) + if (path === '/tool-call' && method === 'POST') { + return { response: await handleToolCall(req), route: '/tool-call' }; } - // POST /sessions/:id/complete - if (subPath === 'complete' && method === 'POST') { - return { response: await handleComplete(executionId, req), route: '/sessions/:executionId/complete' }; - } + // Session-specific routes + const sessionMatch = path.match(/^\/sessions\/([^/]+)(\/(.+))?$/); + if (sessionMatch) { + const executionId = sessionMatch[1]; + const subPath = sessionMatch[3]; + + // GET /sessions/:id + if (!subPath && method === 'GET') { + return { + response: await handleGetStatus(executionId), + route: '/sessions/:executionId', + }; + } + + // DELETE /sessions/:id + if (!subPath && method === 'DELETE') { + return { + response: await handleDeleteSession(executionId), + route: '/sessions/:executionId', + }; + } + + if (subPath === 'pending' && method === 'GET') { + return { + response: await handleGetPending(executionId), + route: '/sessions/:executionId/pending', + }; + } + + // POST /sessions/:id/results + if (subPath === 'results' && method === 'POST') { + return { + response: await handleSubmitResults(executionId, req), + route: '/sessions/:executionId/results', + }; + } + + // GET /sessions/:id/status + if (subPath === 'status' && method === 'GET') { + return { + response: await handleGetStatus(executionId), + route: '/sessions/:executionId/status', + }; + } + + // POST /sessions/:id/complete + if (subPath === 'complete' && method === 'POST') { + return { + response: await handleComplete(executionId, req), + route: '/sessions/:executionId/complete', + }; + } - // POST /sessions/:id/error - if (subPath === 'error' && method === 'POST') { - return { response: await handleError(executionId, req), route: '/sessions/:executionId/error' }; + // POST /sessions/:id/error + if (subPath === 'error' && method === 'POST') { + return { + response: await handleError(executionId, req), + route: '/sessions/:executionId/error', + }; + } } - } - return { response: errorResponse('Not found', 404), route: 'unmatched' }; + return { response: errorResponse('Not found', 404), route: 'unmatched' }; } async function handleRequest(req: Request): Promise { - return withTraceContext(Object.fromEntries(req.headers.entries()), () => withSpan('codeapi.tool_call_server.request', { - 'http.request.method': req.method, - 'url.path': normalizeTracePath(new URL(req.url).pathname), - }, async (span) => { - const start = httpLatencyStartMs(); - let route = 'unmatched'; - try { - const { response, route: matchedRoute } = await routeToolCallRequest(req); - route = matchedRoute; - span.setAttribute('http.response.status_code', response.status); - span.setAttribute('http.route', route); - recordHttpRequest({ - method: req.method, - route, - statusCode: response.status, - durationSeconds: httpLatencyElapsedSeconds(start), - }); - return response; - } catch (error) { - recordHttpRequest({ - method: req.method, - route, - statusCode: 500, - durationSeconds: httpLatencyElapsedSeconds(start), - }); - throw error; - } - }, 'SERVER')); + return withTraceContext(Object.fromEntries(req.headers.entries()), () => + withSpan( + 'codeapi.tool_call_server.request', + { + 'http.request.method': req.method, + 'url.path': normalizeTracePath(new URL(req.url).pathname), + }, + async span => { + const start = httpLatencyStartMs(); + let route = 'unmatched'; + try { + const { response, route: matchedRoute } = + await routeToolCallRequest(req); + route = matchedRoute; + span.setAttribute( + 'http.response.status_code', + response.status, + ); + span.setAttribute('http.route', route); + recordHttpRequest({ + method: req.method, + route, + statusCode: response.status, + durationSeconds: httpLatencyElapsedSeconds(start), + }); + return response; + } catch (error) { + recordHttpRequest({ + method: req.method, + route, + statusCode: 500, + durationSeconds: httpLatencyElapsedSeconds(start), + }); + throw error; + } + }, + 'SERVER', + ), + ); } // Start server const server = Bun.serve({ - port: PORT, - fetch: handleRequest, + port: PORT, + fetch: handleRequest, }); logger.info(`[${INSTANCE_ID}] Tool Call Server running on port ${PORT}`); @@ -628,38 +742,42 @@ logger.info(`[${INSTANCE_ID}] Tool Call Server running on port ${PORT}`); let shuttingDown = false; async function shutdown(): Promise { - if (shuttingDown) return; - shuttingDown = true; - logger.info(`[${INSTANCE_ID}] Shutting down...`); - try { - server.stop(); - await redis.quit(); + if (shuttingDown) return; + shuttingDown = true; + logger.info(`[${INSTANCE_ID}] Shutting down...`); try { - await shutdownTelemetry(); - } catch (telemetryError) { - logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { error: telemetryError }); - } - process.exit(0); - } catch (error) { - logger.error(`[${INSTANCE_ID}] Shutdown failed`, { error }); - try { - await shutdownTelemetry(); - } catch (telemetryError) { - logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { error: telemetryError }); + server.stop(); + await redis.quit(); + try { + await shutdownTelemetry(); + } catch (telemetryError) { + logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { + error: telemetryError, + }); + } + process.exit(0); + } catch (error) { + logger.error(`[${INSTANCE_ID}] Shutdown failed`, { error }); + try { + await shutdownTelemetry(); + } catch (telemetryError) { + logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { + error: telemetryError, + }); + } + process.exit(1); } - process.exit(1); - } } process.on('SIGTERM', () => void shutdown()); process.on('SIGINT', () => void shutdown()); -process.on('uncaughtException', (error) => { - logger.error('Uncaught Exception', { error }); +process.on('uncaughtException', error => { + logger.error('Uncaught Exception', { error }); }); process.on('unhandledRejection', (reason, promise) => { - logger.error('Unhandled Rejection', { reason, promise }); + logger.error('Unhandled Rejection', { reason, promise }); }); export { server }; diff --git a/service/src/workers.ts b/service/src/workers.ts index e9c3b2b..b118f71 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -1,302 +1,360 @@ import axios from 'axios'; import { Worker } from 'bullmq'; import type * as t from './types'; -import { filterSystemLogs, applySystemReplacements, getAxiosErrorDetails, sandboxErrorMessageFromAxios } from './utils'; -import { jobProcessingDuration, jobsCompleted, jobsFailed, activeJobs, workerRunning } from './metrics'; +import { + filterSystemLogs, + applySystemReplacements, + getAxiosErrorDetails, + sandboxErrorMessageFromAxios, +} from './utils'; +import { + jobProcessingDuration, + jobsCompleted, + jobsFailed, + activeJobs, + workerRunning, +} from './metrics'; import { Queues } from './enum'; import { connection } from './queue'; import { env, jobDeadlineAtMs } from './config'; import { summarizeSandboxResponse, summarizeText } from './execution-log'; -import { createGatewayEgressGrant, restoreGatewaySandboxResult, revokeGatewayEgressGrant } from './egress-gateway-client'; +import { + createGatewayEgressGrant, + restoreGatewaySandboxResult, + revokeGatewayEgressGrant, +} from './egress-gateway-client'; import { refreshEgressGrantClaims } from './sandbox-egress'; import { buildSandboxExecuteRequest } from './sandbox-dispatch'; import { prepareInputDelivery } from './runtime-session/input-delivery'; import { SessionFilesError } from './runtime-session/files'; import { resolveRuntimeSessionForJob } from './runtime-session/job-policy'; -import { getSandboxBackend, SandboxBackendError, type SandboxRawResponse } from './sandbox-backend'; +import { + getSandboxBackend, + SandboxBackendError, + type SandboxRawResponse, +} from './sandbox-backend'; import { isSyntheticPrincipalSource } from './auth/synthetic'; import { withSpan, withTraceContext } from './telemetry'; import { workerDeadlineFailure } from './worker-error'; import logger from './logger'; +import { bullmqPrefix } from './redis-connection'; const { INSTANCE_ID } = env; const WORKER_ID = `${INSTANCE_ID}-${process.pid}`; function isAbortError(error: unknown): boolean { - return axios.isAxiosError(error) && (error.name === 'AbortError' || error.code === 'ERR_CANCELED'); + return ( + axios.isAxiosError(error) && + (error.name === 'AbortError' || error.code === 'ERR_CANCELED') + ); } async function processJob(job: t.ExecuteJob): Promise { - return withTraceContext(job.data._otel, () => withSpan('codeapi.job.process', { - 'messaging.system': 'bullmq', - 'messaging.operation.name': 'process', - 'messaging.message.id': typeof job.id === 'string' ? job.id : String(job.id ?? ''), - 'codeapi.language': job.data.payload?.language ?? 'unknown', - }, () => processJobInner(job), 'CONSUMER')); + return withTraceContext(job.data._otel, () => + withSpan( + 'codeapi.job.process', + { + 'messaging.system': 'bullmq', + 'messaging.operation.name': 'process', + 'messaging.message.id': + typeof job.id === 'string' ? job.id : String(job.id ?? ''), + 'codeapi.language': job.data.payload?.language ?? 'unknown', + }, + () => processJobInner(job), + 'CONSUMER', + ), + ); } async function processJobInner(job: t.ExecuteJob): Promise { - const { code, payload, isPyPlot } = job.data; - const isSyntheticJob = job.data.isSynthetic === true || isSyntheticPrincipalSource(job.data.principalSource); - const language = payload?.language ?? 'unknown'; - const endTimer = jobProcessingDuration.startTimer({ language }); - activeJobs.inc({ language }); + const { code, payload, isPyPlot } = job.data; + const isSyntheticJob = + job.data.isSynthetic === true || + isSyntheticPrincipalSource(job.data.principalSource); + const language = payload?.language ?? 'unknown'; + const endTimer = jobProcessingDuration.startTimer({ language }); + activeJobs.inc({ language }); - const controller = new AbortController(); - const deadlineAtMs = jobDeadlineAtMs(job.timestamp, env.JOB_TIMEOUT); - const remainingBudgetMs = Math.max(0, deadlineAtMs - Date.now()); - const timer = remainingBudgetMs > 0 - ? setTimeout(() => controller.abort(), remainingBudgetMs) - : undefined; - if (remainingBudgetMs === 0) controller.abort(); - let egressGrantId: string | undefined; - let egressGrantTokenForRestore: string | undefined; - let revokeReason = 'completed'; + const controller = new AbortController(); + const deadlineAtMs = jobDeadlineAtMs(job.timestamp, env.JOB_TIMEOUT); + const remainingBudgetMs = Math.max(0, deadlineAtMs - Date.now()); + const timer = + remainingBudgetMs > 0 + ? setTimeout(() => controller.abort(), remainingBudgetMs) + : undefined; + if (remainingBudgetMs === 0) controller.abort(); + let egressGrantId: string | undefined; + let egressGrantTokenForRestore: string | undefined; + let revokeReason = 'completed'; - try { - if (controller.signal.aborted) { - throw new Error(`Job timed out after ${env.JOB_TIMEOUT}ms`); - } - let sandboxPayload = payload; - let executionManifestClaims = job.data.executionManifestClaims; - let egressGrantToken = job.data.egressGrantToken; + try { + if (controller.signal.aborted) { + throw new Error(`Job timed out after ${env.JOB_TIMEOUT}ms`); + } + let sandboxPayload = payload; + let executionManifestClaims = job.data.executionManifestClaims; + let egressGrantToken = job.data.egressGrantToken; - if (job.data.egressGrantClaims) { - const nowSeconds = Math.floor(Date.now() / 1000); - const prepared = await createGatewayEgressGrant({ - payload, - claims: refreshEgressGrantClaims(job.data.egressGrantClaims, nowSeconds), - isSynthetic: isSyntheticJob, - signal: controller.signal, - }); - egressGrantId = prepared.grant_id; - sandboxPayload = prepared.payload; - egressGrantToken = prepared.egressGrantToken; - egressGrantTokenForRestore = prepared.egressGrantToken; - executionManifestClaims = (env.EXECUTION_MANIFEST_PRIVATE_KEY || env.EXECUTION_MANIFEST_SECRET) - ? prepared.executionManifestClaims - : undefined; - } + if (job.data.egressGrantClaims) { + const nowSeconds = Math.floor(Date.now() / 1000); + const prepared = await createGatewayEgressGrant({ + payload, + claims: refreshEgressGrantClaims( + job.data.egressGrantClaims, + nowSeconds, + ), + isSynthetic: isSyntheticJob, + signal: controller.signal, + }); + egressGrantId = prepared.grant_id; + sandboxPayload = prepared.payload; + egressGrantToken = prepared.egressGrantToken; + egressGrantTokenForRestore = prepared.egressGrantToken; + executionManifestClaims = + env.EXECUTION_MANIFEST_PRIVATE_KEY || + env.EXECUTION_MANIFEST_SECRET + ? prepared.executionManifestClaims + : undefined; + } - const delivery = prepareInputDelivery(payload, sandboxPayload); - const sandboxRequest = buildSandboxExecuteRequest({ - payload: delivery.payload, - egressGrantToken, - executionManifestClaims, - executionManifestPrivateKey: env.EXECUTION_MANIFEST_PRIVATE_KEY, - executionManifestSecret: env.EXECUTION_MANIFEST_SECRET, - executionManifestTtlSeconds: env.EXECUTION_MANIFEST_TTL_SECONDS, - }); - egressGrantTokenForRestore = egressGrantToken; + const delivery = prepareInputDelivery(payload, sandboxPayload); + const sandboxRequest = buildSandboxExecuteRequest({ + payload: delivery.payload, + egressGrantToken, + executionManifestClaims, + executionManifestPrivateKey: env.EXECUTION_MANIFEST_PRIVATE_KEY, + executionManifestSecret: env.EXECUTION_MANIFEST_SECRET, + executionManifestTtlSeconds: env.EXECUTION_MANIFEST_TTL_SECONDS, + }); + egressGrantTokenForRestore = egressGrantToken; - const runtimeSession = resolveRuntimeSessionForJob({ - workerMode: env.RUNTIME_SESSION_MODE, - workerBackend: env.SANDBOX_BACKEND, - runtimeSessionMode: job.data.runtimeSessionMode, - runtimeSessionId: job.data.runtimeSessionId, - runtimeSessionExemption: job.data.runtimeSessionExemption, - isSynthetic: isSyntheticJob, - }); + const runtimeSession = resolveRuntimeSessionForJob({ + workerMode: env.RUNTIME_SESSION_MODE, + workerBackend: env.SANDBOX_BACKEND, + runtimeSessionMode: job.data.runtimeSessionMode, + runtimeSessionId: job.data.runtimeSessionId, + runtimeSessionExemption: job.data.runtimeSessionExemption, + isSynthetic: isSyntheticJob, + }); - /* Stateful Lambda runs this inside its session commit barrier. The worker - * still invokes it unconditionally as the HTTP/stateless fallback; marking - * the transformed object makes that second call an idempotent no-op. */ - const resultRestoreToken = egressGrantTokenForRestore; - const finalizedSandboxResults = new WeakSet(); - const finalizeSandboxResult = async (result: SandboxRawResponse): Promise => { - if ( - resultRestoreToken === undefined || - resultRestoreToken.length === 0 || - finalizedSandboxResults.has(result) - ) { - return result; - } - const restored = await restoreGatewaySandboxResult({ - grantId: egressGrantId, - egressGrantToken: resultRestoreToken, - result, - isSynthetic: isSyntheticJob, - signal: controller.signal, - }); - finalizedSandboxResults.add(restored); - return restored; - }; + /* Stateful Lambda runs this inside its session commit barrier. The worker + * still invokes it unconditionally as the HTTP/stateless fallback; marking + * the transformed object makes that second call an idempotent no-op. */ + const resultRestoreToken = egressGrantTokenForRestore; + const finalizedSandboxResults = new WeakSet(); + const finalizeSandboxResult = async ( + result: SandboxRawResponse, + ): Promise => { + if ( + resultRestoreToken === undefined || + resultRestoreToken.length === 0 || + finalizedSandboxResults.has(result) + ) { + return result; + } + const restored = await restoreGatewaySandboxResult({ + grantId: egressGrantId, + egressGrantToken: resultRestoreToken, + result, + isSynthetic: isSyntheticJob, + signal: controller.signal, + }); + finalizedSandboxResults.add(restored); + return restored; + }; - const responseRaw = await getSandboxBackend().execute( - { - body: sandboxRequest.body, - headers: sandboxRequest.headers, - inputDelivery: delivery.refs, - }, - { - executionId: job.data.executionId ?? '', - language, - isSynthetic: isSyntheticJob, - signal: controller.signal, - deadlineAtMs, - tenantId: job.data.tenantId, - canonicalUserId: job.data.canonicalUserId, - runtimeSessionId: runtimeSession.runtimeSessionId, - runtimeSessionMode: runtimeSession.runtimeSessionMode, - /* Stateful backends run this as a commit barrier after user code but - * before checkpointing/reusing the mutated workspace. Stateless/HTTP - * paths retain the worker-owned fallback immediately below. */ - sessionResultFinalizer: resultRestoreToken !== undefined && resultRestoreToken.length > 0 - ? finalizeSandboxResult - : undefined, - }, - ); + const responseRaw = await getSandboxBackend().execute( + { + body: sandboxRequest.body, + headers: sandboxRequest.headers, + inputDelivery: delivery.refs, + }, + { + executionId: job.data.executionId ?? '', + language, + isSynthetic: isSyntheticJob, + signal: controller.signal, + deadlineAtMs, + tenantId: job.data.tenantId, + canonicalUserId: job.data.canonicalUserId, + runtimeSessionId: runtimeSession.runtimeSessionId, + runtimeSessionMode: runtimeSession.runtimeSessionMode, + /* Stateful backends run this as a commit barrier after user code but + * before checkpointing/reusing the mutated workspace. Stateless/HTTP + * paths retain the worker-owned fallback immediately below. */ + sessionResultFinalizer: + resultRestoreToken !== undefined && + resultRestoreToken.length > 0 + ? finalizeSandboxResult + : undefined, + }, + ); - const responseData = await finalizeSandboxResult(responseRaw); + const responseData = await finalizeSandboxResult(responseRaw); - if (!isSyntheticJob) { - logger.info('Sandbox response', summarizeSandboxResponse(responseData)); - } + if (!isSyntheticJob) { + logger.info( + 'Sandbox response', + summarizeSandboxResponse(responseData), + ); + } - const { files } = responseData; - const run = responseData.run; - const stdout = applySystemReplacements(run?.stdout ?? ''); - const stderr = filterSystemLogs(run?.stderr ?? '', isPyPlot); + const { files } = responseData; + const run = responseData.run; + const stdout = applySystemReplacements(run?.stdout ?? ''); + const stderr = filterSystemLogs(run?.stderr ?? '', isPyPlot); - const result: t.ExecuteResult = { - session_id: responseData.session_id, - /* `files` is optional on the sandbox response (e.g. dry-run - * execute with no outputs); the public `ExecuteResult.files` is - * required and downstream callers always iterate it. Default to - * `[]` so the strictened response type from Phase B doesn't - * surface a regression that wasn't there before. */ - files: files ?? [], - stdout, - stderr, - }; + const result: t.ExecuteResult = { + session_id: responseData.session_id, + /* `files` is optional on the sandbox response (e.g. dry-run + * execute with no outputs); the public `ExecuteResult.files` is + * required and downstream callers always iterate it. Default to + * `[]` so the strictened response type from Phase B doesn't + * surface a regression that wasn't there before. */ + files: files ?? [], + stdout, + stderr, + }; - if (run) { - result.code = run.code ?? null; - result.signal = run.signal != null ? String(run.signal) : null; - result.message = run.message ?? null; - result.status = run.status ?? null; - result.wall_time = (run as Record).wall_time as number | null ?? null; - } + if (run) { + result.code = run.code ?? null; + result.signal = run.signal != null ? String(run.signal) : null; + result.message = run.message ?? null; + result.status = run.status ?? null; + result.wall_time = + ((run as Record).wall_time as number | null) ?? + null; + } - if (result.message || result.signal) { - logger.warn('Sandbox execution error metadata', { - session_id: responseData.session_id, - code: result.code, - signal: result.signal, - message: summarizeText(result.message), - status: result.status, - wall_time: result.wall_time, - }); - } + if (result.message || result.signal) { + logger.warn('Sandbox execution error metadata', { + session_id: responseData.session_id, + code: result.code, + signal: result.signal, + message: summarizeText(result.message), + status: result.status, + wall_time: result.wall_time, + }); + } - return result; - } catch (error) { - revokeReason = controller.signal.aborted || isAbortError(error) ? 'timeout' : 'failed'; - const errorDetails = getAxiosErrorDetails(error); - logger.error('Error processing job', errorDetails); + return result; + } catch (error) { + revokeReason = + controller.signal.aborted || isAbortError(error) + ? 'timeout' + : 'failed'; + const errorDetails = getAxiosErrorDetails(error); + logger.error('Error processing job', errorDetails); - const deadlineFailure = workerDeadlineFailure( - error, - controller.signal.aborted, - env.JOB_TIMEOUT, - ); - if (deadlineFailure) { - throw deadlineFailure; - } else if (error instanceof SandboxBackendError) { - throw new Error(`${error.code}: ${error.message}`); - } else if (error instanceof SessionFilesError) { - /* BullMQ serializes Error rather than preserving custom prototypes. - * Carry the stable code in the message so the public router can map the - * input failure without confusing it with MicroVM health. */ - throw new Error(`${error.code}: ${error.message}`); - } else if (isAbortError(error)) { - throw new Error(`Job timed out after ${env.JOB_TIMEOUT}ms`); - } else if (axios.isAxiosError(error)) { - /** Preserve error message from sandbox */ - const sandboxError = sandboxErrorMessageFromAxios(error); - throw new Error(`Error from sandbox: ${sandboxError}`); + const deadlineFailure = workerDeadlineFailure( + error, + controller.signal.aborted, + env.JOB_TIMEOUT, + ); + if (deadlineFailure) { + throw deadlineFailure; + } else if (error instanceof SandboxBackendError) { + throw new Error(`${error.code}: ${error.message}`); + } else if (error instanceof SessionFilesError) { + /* BullMQ serializes Error rather than preserving custom prototypes. + * Carry the stable code in the message so the public router can map the + * input failure without confusing it with MicroVM health. */ + throw new Error(`${error.code}: ${error.message}`); + } else if (isAbortError(error)) { + throw new Error(`Job timed out after ${env.JOB_TIMEOUT}ms`); + } else if (axios.isAxiosError(error)) { + /** Preserve error message from sandbox */ + const sandboxError = sandboxErrorMessageFromAxios(error); + throw new Error(`Error from sandbox: ${sandboxError}`); + } + throw error; + } finally { + if (egressGrantId || egressGrantTokenForRestore) { + await revokeGatewayEgressGrant({ + grantId: egressGrantId, + egressGrantToken: egressGrantId + ? undefined + : egressGrantTokenForRestore, + isSynthetic: isSyntheticJob, + reason: revokeReason, + timeoutMs: env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS, + }).catch(error => { + logger.error('Failed to revoke egress grant', { + grantId: egressGrantId, + error: getAxiosErrorDetails(error), + }); + }); + } + if (timer) clearTimeout(timer); + endTimer(); + activeJobs.dec({ language }); } - throw error; - } finally { - if (egressGrantId || egressGrantTokenForRestore) { - await revokeGatewayEgressGrant({ - grantId: egressGrantId, - egressGrantToken: egressGrantId ? undefined : egressGrantTokenForRestore, - isSynthetic: isSyntheticJob, - reason: revokeReason, - timeoutMs: env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS, - }).catch(error => { - logger.error('Failed to revoke egress grant', { grantId: egressGrantId, error: getAxiosErrorDetails(error) }); - }); - } - if (timer) clearTimeout(timer); - endTimer(); - activeJobs.dec({ language }); - } } // Global workers - no INSTANCE_ID prefix // This enables horizontal scaling where any worker can process any job from the shared queue // Each worker respects its own concurrency limit based on its co-located sandbox capacity export const pyWorker = new Worker(Queues.python, processJob, { - connection, - concurrency: env.PYTHON_CONCURRENCY, - limiter: { - max: env.PYTHON_CONCURRENCY, - duration: env.JOB_WINDOW, - }, + connection, + prefix: bullmqPrefix(), + concurrency: env.PYTHON_CONCURRENCY, + limiter: { + max: env.PYTHON_CONCURRENCY, + duration: env.JOB_WINDOW, + }, }); export const otherWorker = new Worker(Queues.other, processJob, { - connection, - concurrency: env.OTHER_CONCURRENCY, - limiter: { - max: env.OTHER_CONCURRENCY, - duration: env.JOB_WINDOW, - }, + connection, + prefix: bullmqPrefix(), + concurrency: env.OTHER_CONCURRENCY, + limiter: { + max: env.OTHER_CONCURRENCY, + duration: env.JOB_WINDOW, + }, }); workerRunning.set({ worker_type: 'python' }, 1); workerRunning.set({ worker_type: 'other' }, 1); pyWorker.on('completed', job => { - if (job.data.isSynthetic !== true) { - logger.info(`[${WORKER_ID}] Python job completed ${job.id}`); - } - jobsCompleted.inc({ language: 'python' }); + if (job.data.isSynthetic !== true) { + logger.info(`[${WORKER_ID}] Python job completed ${job.id}`); + } + jobsCompleted.inc({ language: 'python' }); }); otherWorker.on('completed', job => { - if (job.data.isSynthetic !== true) { - logger.info(`[${WORKER_ID}] Other job completed ${job.id}`); - } - jobsCompleted.inc({ language: 'other' }); + if (job.data.isSynthetic !== true) { + logger.info(`[${WORKER_ID}] Other job completed ${job.id}`); + } + jobsCompleted.inc({ language: 'other' }); }); pyWorker.on('failed', (job, err) => { - logger.error(`[${WORKER_ID}] Python job ${job?.id} failed`, err); - jobsFailed.inc({ language: 'python' }); + logger.error(`[${WORKER_ID}] Python job ${job?.id} failed`, err); + jobsFailed.inc({ language: 'python' }); }); otherWorker.on('failed', (job, err) => { - logger.error(`[${WORKER_ID}] Other job ${job?.id} failed`, err); - jobsFailed.inc({ language: 'other' }); + logger.error(`[${WORKER_ID}] Other job ${job?.id} failed`, err); + jobsFailed.inc({ language: 'other' }); }); -pyWorker.on('error', (err) => { - logger.error(`[${WORKER_ID}] Python worker error`, err); - workerRunning.set({ worker_type: 'python' }, 0); +pyWorker.on('error', err => { + logger.error(`[${WORKER_ID}] Python worker error`, err); + workerRunning.set({ worker_type: 'python' }, 0); }); -otherWorker.on('error', (err) => { - logger.error(`[${WORKER_ID}] Other worker error`, err); - workerRunning.set({ worker_type: 'other' }, 0); +otherWorker.on('error', err => { + logger.error(`[${WORKER_ID}] Other worker error`, err); + workerRunning.set({ worker_type: 'other' }, 0); }); pyWorker.on('closed', () => { - workerRunning.set({ worker_type: 'python' }, 0); + workerRunning.set({ worker_type: 'python' }, 0); }); otherWorker.on('closed', () => { - workerRunning.set({ worker_type: 'other' }, 0); + workerRunning.set({ worker_type: 'other' }, 0); });