sourcegraph-executor/k8s: opt-in CronJob to clean up orphaned executor job resources

charts/sourcegraph-executor/k8s/README.md

@@ -57,6 +57,10 @@ In addition to the documented values, the `executor` and `private-docker-registr
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | executor.affinity | object | `{}` | Affinity, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) |
+| executor.cleanup.enabled | bool | `false` | If true, deploy a CronJob that deletes leftover executor job resources (Jobs, Secrets, and PersistentVolumeClaims named `sg-executor-job-*`). The executor deletes these itself after each job, but leaves them behind when it is killed mid-job (e.g. OOM kill, node scale-down, rollout). |
+| executor.cleanup.image | string | `"index.docker.io/alpine/kubectl:1.33.4"` | A kubectl image that includes a shell, used by the cleanup CronJob. |
+| executor.cleanup.minimumAgeSeconds | string | `"3600"` | Only delete resources older than this many seconds. Must be greater than `executor.kubernetesJob.deadline`, so resources of running jobs are never deleted. |
+| executor.cleanup.schedule | string | `"*/30 * * * *"` | Cron schedule of the cleanup CronJob. |
 | executor.configureRbac | bool | `true` | Whether to configure the necessary RBAC resources. Required only once for all executor deployments. |
 | executor.containerSecurityContext | object | `{"privileged":false}` | Security context for the container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) |
 | executor.debug.keepJobs | string | `"false"` | If true, Kubernetes jobs will not be deleted after they complete. Not recommended for production use as it can hit cluster limits. |

charts/sourcegraph-executor/k8s/templates/executor.CleanupCronJob.yaml

@@ -0,0 +1,72 @@
+{{- if .Values.executor.cleanup.enabled }}
+{{- if lt (int .Values.executor.cleanup.minimumAgeSeconds) (int .Values.executor.kubernetesJob.deadline) }}
+{{- fail "executor.cleanup.minimumAgeSeconds must be greater than executor.kubernetesJob.deadline, so resources of running jobs are never deleted" }}
+{{- end }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "executor.name" . }}-cleanup
+  annotations:
+    description: Deletes leftover executor job resources, which the executor could not tear down because it was killed mid-job
+  labels:
+    {{- include "executor.labels" . | nindent 4 }}
+spec:
+  schedule: {{ .Values.executor.cleanup.schedule | quote }}
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 600
+      template:
+        metadata:
+          labels:
+            {{- include "executor.labels" . | nindent 12 }}
+        spec:
+          serviceAccountName: sg-executor
+          restartPolicy: OnFailure
+          containers:
+            - name: cleanup
+              image: {{ .Values.executor.cleanup.image }}
+              imagePullPolicy: {{ .Values.sourcegraph.image.pullPolicy }}
+              env:
+                - name: NAMESPACE
+                  value: {{ .Values.executor.namespace | quote }}
+                - name: MINIMUM_AGE_SECONDS
+                  value: {{ .Values.executor.cleanup.minimumAgeSeconds | quote }}
+              command:
+                - /bin/sh
+                - -eu
+                - -c
+                - |
+                  now_seconds=$(date -u +%s)
+                  sweep() {
+                    kind=$1
+                    kubectl get "$kind" --namespace "$NAMESPACE" \
+                      --output 'jsonpath={range .items[*]}{.metadata.name} {.metadata.creationTimestamp}{"\n"}{end}' \
+                    | while read -r name created_at; do
+                        case "$name" in sg-executor-job-*) ;; *) continue ;; esac
+                        created_seconds=$(date -u -D '%Y-%m-%dT%H:%M:%SZ' -d "$created_at" +%s)
+                        if [ $((now_seconds - created_seconds)) -ge "$MINIMUM_AGE_SECONDS" ]; then
+                          echo "deleting $kind/$name, created at $created_at"
+                          kubectl delete "$kind" "$name" --namespace "$NAMESPACE" --ignore-not-found
+                        fi
+                      done
+                  }
+                  sweep jobs
+                  sweep secrets
+                  sweep persistentvolumeclaims
+              {{- if not .Values.sourcegraph.localDevMode }}
+              resources:
+                requests:
+                  cpu: 10m
+                  memory: 32Mi
+                limits:
+                  cpu: 100m
+                  memory: 128Mi
+              {{- end }}
+          {{- with .Values.sourcegraph.imagePullSecrets }}
+          imagePullSecrets:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+{{- end }}

charts/sourcegraph-executor/k8s/templates/executor.Role.yaml

@@ -14,6 +14,8 @@ rules:
     resources:
       - jobs
     verbs:
+      - get
+      - list
       - create
       - delete
   - apiGroups:

charts/sourcegraph-executor/k8s/tests/cleanup_test.yaml

@@ -0,0 +1,40 @@
+suite: executor cleanup
+templates:
+  - executor.CleanupCronJob.yaml
+tests:
+  - it: should not render the CronJob by default
+    set:
+      executor:
+        queueName: "test"
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should render the CronJob when cleanup is enabled
+    set:
+      executor:
+        queueName: "test"
+        cleanup:
+          enabled: true
+    asserts:
+      - containsDocument:
+          kind: CronJob
+          apiVersion: batch/v1
+          name: executor-test-cleanup
+      - equal:
+          path: spec.schedule
+          value: "*/30 * * * *"
+      - equal:
+          path: spec.jobTemplate.spec.template.spec.serviceAccountName
+          value: sg-executor
+
+  - it: should fail when minimumAgeSeconds does not exceed the job deadline
+    set:
+      executor:
+        queueName: "test"
+        cleanup:
+          enabled: true
+          minimumAgeSeconds: "600"
+    asserts:
+      - failedTemplate:
+          errorMessage: "executor.cleanup.minimumAgeSeconds must be greater than executor.kubernetesJob.deadline, so resources of running jobs are never deleted"

charts/sourcegraph-executor/k8s/values.yaml

@@ -135,6 +135,17 @@ executor:
     keepJobs: "false"
     keepWorkspaces: "false"
 
+  cleanup:
+    # -- If true, deploy a CronJob that deletes leftover executor job resources (Jobs, Secrets, and PersistentVolumeClaims named `sg-executor-job-*`).
+    # The executor deletes these itself after each job, but leaves them behind when it is killed mid-job (e.g. OOM kill, node scale-down, rollout).
+    enabled: false
+    # -- Cron schedule of the cleanup CronJob.
+    schedule: "*/30 * * * *"
+    # -- Only delete resources older than this many seconds. Must be greater than `executor.kubernetesJob.deadline`, so resources of running jobs are never deleted.
+    minimumAgeSeconds: "3600"
+    # -- A kubectl image that includes a shell, used by the cleanup CronJob.
+    image: "index.docker.io/alpine/kubectl:1.33.4"
+
   # -- Affinity,
   # learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity)
   affinity: { }