orchestratord: use leader election and run multiple operator replicas

Cargo.toml

@@ -406,7 +406,7 @@ itertools = "0.14.0"
 jemalloc_pprof = "0.8.2"
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 junit-report = "0.8.3"
-k8s-controller = "0.11.0"
+k8s-controller = "0.12.0"
 k8s-openapi = { version = "0.27.0", features = ["schemars", "v1_32"] }
 kube = { version = "3.1.0", default-features = false, features = ["client", "derive", "openssl-tls", "runtime", "ws"] }
 launchdarkly-server-sdk = { version = "3.1.1", default-features = false, features = ["hyper-rustls-native-roots", "crypto-aws-lc-rs"] }

ci/nightly/pipeline.template.yml

@@ -2644,6 +2644,20 @@ steps:
         agents:
           queue: hetzner-aarch64-16cpu-32gb
 
+      - id: orchestratord-leader-failover
+        topics: [self-managed]
+        label: Orchestratord leader election failover
+        artifact_paths: ["mz_debug_*.zip"]
+        depends_on: devel-docker-tags
+        timeout_in_minutes: 60
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: orchestratord
+              run: leader-failover
+              ci-builder: stable
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+
       - id: orchestratord-upgrade-operator
         topics: [self-managed, upgrade]
         label: Orchestratord operator upgrade

doc/user/data/metrics.yml

@@ -67,7 +67,7 @@ metrics:
   source: src/environmentd/src/http/metrics.rs
   visibility: internal
 - name: environmentd_needs_update
-  help: Count of organizations in this cluster which are running outdated pod templates
+  help: Count of organizations in this cluster which are running outdated pod templates. Only the operator replica holding the leadership lease reconciles, so the others report zero.
   source: src/orchestratord/src/metrics.rs
   visibility: internal
 - name: jemalloc_active
@@ -4279,6 +4279,10 @@ metrics:
   - reason
   source: src/adapter/src/metrics.rs
   visibility: internal
+- name: orchestratord_is_leader
+  help: Whether this operator replica holds the controller leadership lease, and is therefore the replica reconciling. Summed across the replicas this should be 1. A sustained 0 means no replica can take the lease, for instance because the service account lacks permission on leases, and the operator is reconciling nothing.
+  source: src/orchestratord/src/metrics.rs
+  visibility: internal
 - name: outer_join_lowering_cases
   help: How many times the different outer join lowering cases happened.
   labels:

misc/helm-charts/operator/README.md

@@ -176,6 +176,9 @@ The following table lists the configurable parameters of the Materialize operato
 | `operator.image.repository` | The Docker repository for the operator image | ``"materialize/orchestratord"`` |
 | `operator.image.tag` | The tag/version of the operator image to be used | ``"v26.34.1"`` |
 | `operator.nodeSelector` | Node selector to use for the operator pod | ``{}`` |
+| `operator.podDisruptionBudget.enabled` | Whether to create a PodDisruptionBudget for the operator. Only created when `replicas` is greater than 1, since a budget over a single replica either blocks node drains or protects nothing. | ``true`` |
+| `operator.podDisruptionBudget.maxUnavailable` | Maximum number of operator pods that may be unavailable at once during voluntary disruptions. Expressed as a maximum rather than a minimum so that node drains are never blocked outright, they are only serialized. | ``1`` |
+| `operator.replicas` | Number of operator replicas. The operator uses leader election so that only one replica reconciles at a time. Running more than one replica avoids downtime of the CRD conversion webhook during rollouts and node drains. | ``2`` |
 | `operator.resources.limits` | Resource limits for the operator's CPU and memory | ``{"memory":"512Mi"}`` |
 | `operator.resources.requests` | Resources requested by the operator for CPU and memory | ``{"cpu":"100m","memory":"512Mi"}`` |
 | `operator.secretsController` | Which secrets controller to use for storing secrets. Valid values are 'kubernetes' and 'aws-secrets-manager'. Setting 'aws-secrets-manager' requires a configured AWS cloud provider and IAM role for the environment with Secrets Manager permissions. | ``"kubernetes"`` |

misc/helm-charts/operator/templates/clusterrole.yaml

@@ -37,6 +37,13 @@ rules:
   - get
   - list
   - watch
+- apiGroups: ["coordination.k8s.io"]
+  resources:
+  - leases
+  verbs:
+  - get
+  - create
+  - update
 - apiGroups: ["networking.k8s.io"]
   resources:
   - networkpolicies

misc/helm-charts/operator/templates/deployment.yaml

@@ -16,7 +16,7 @@ metadata:
   labels:
     {{- include "materialize-operator.labels" . | nindent 4 }}
 spec:
-  replicas: 1
+  replicas: {{ .Values.operator.replicas }}
   selector:
     matchLabels:
       {{- include "materialize-operator.selectorLabels" . | nindent 6 }}
@@ -32,13 +32,35 @@ spec:
         runAsUser: 65532
         runAsNonRoot: true
       serviceAccountName: {{ include "materialize-operator.serviceAccountName" . }}
+      # On SIGTERM the operator releases its leadership lease, then keeps
+      # serving the conversion webhook for a few seconds while Kubernetes
+      # removes it from the webhook service's endpoints, then drains the
+      # in-flight conversion requests. Those steps run in sequence and are
+      # individually bounded in the binary, adding up to 15 seconds. This is
+      # set explicitly, rather than left to the Kubernetes default, so that the
+      # budget the binary assumes cannot drift away from what it is given.
+      terminationGracePeriodSeconds: 30
       {{- if .Values.operator.nodeSelector }}
       nodeSelector:
         {{- toYaml .Values.operator.nodeSelector | nindent 8 }}
       {{- end }}
       {{- if .Values.operator.affinity }}
       affinity:
         {{- toYaml .Values.operator.affinity | nindent 8 }}
+      {{- else if gt (int .Values.operator.replicas) 1 }}
+      # Spread the replicas across nodes by default, so that losing a single
+      # node can't take down every replica of the conversion webhook. This is
+      # a preference rather than a requirement, so that the operator still
+      # schedules on single-node clusters.
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              topologyKey: kubernetes.io/hostname
+              labelSelector:
+                matchLabels:
+                  {{- include "materialize-operator.selectorLabels" . | nindent 18 }}
       {{- end }}
       {{- if .Values.operator.tolerations }}
       tolerations:
@@ -271,6 +293,13 @@ spec:
         - "--webhook-service-namespace"
         - {{ .Release.Namespace }}
         {{- end }}
+        env:
+        # Sets the operator's --leader-election-identity, which must be unique
+        # per replica.
+        - name: ORCHESTRATORD_LEADER_ELECTION_IDENTITY
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
         ports:
         - containerPort: 3100
           name: metrics

misc/helm-charts/operator/templates/poddisruptionbudget.yaml

@@ -0,0 +1,23 @@
+{{- if and .Values.operator.podDisruptionBudget.enabled (gt (int .Values.operator.replicas) 1) -}}
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "materialize-operator.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "materialize-operator.labels" . | nindent 4 }}
+spec:
+  maxUnavailable: {{ .Values.operator.podDisruptionBudget.maxUnavailable }}
+  selector:
+    matchLabels:
+      {{- include "materialize-operator.selectorLabels" . | nindent 6 }}
+{{- end }}

misc/helm-charts/operator/tests/deployment_test.yaml

@@ -244,3 +244,53 @@ tests:
   - notContains:
       path: spec.template.spec.containers[0].args
       content: "--disable-license-key-checks"
+
+- it: should pass the pod name for the leader election identity
+  asserts:
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: ORCHESTRATORD_LEADER_ELECTION_IDENTITY
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.name
+
+- it: should spread replicas across nodes by default
+  asserts:
+  - equal:
+      path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey
+      value: kubernetes.io/hostname
+  - equal:
+      path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchLabels["app.kubernetes.io/instance"]
+      value: RELEASE-NAME
+
+- it: should not spread a single replica
+  set:
+    operator.replicas: 1
+  asserts:
+  - notExists:
+      path: spec.template.spec.affinity
+
+- it: should let a configured affinity win over the default
+  set:
+    operator.affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+          - matchExpressions:
+            - key: kubernetes.io/os
+              operator: In
+              values: [linux]
+  asserts:
+  - notExists:
+      path: spec.template.spec.affinity.podAntiAffinity
+  - exists:
+      path: spec.template.spec.affinity.nodeAffinity
+
+# The operator's shutdown sequence is bounded to 15 seconds, so this must not
+# drop below that. See the template for the breakdown.
+- it: should give the operator time to shut down gracefully
+  asserts:
+  - equal:
+      path: spec.template.spec.terminationGracePeriodSeconds
+      value: 30

misc/helm-charts/operator/tests/poddisruptionbudget_test.yaml

@@ -0,0 +1,34 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+suite: test poddisruptionbudget
+templates:
+- poddisruptionbudget.yaml
+tests:
+- it: should create a pod disruption budget by default
+  asserts:
+  - isKind:
+      of: PodDisruptionBudget
+  - equal:
+      path: spec.maxUnavailable
+      value: 1
+
+- it: should not create a pod disruption budget with a single replica
+  set:
+    operator.replicas: 1
+  asserts:
+  - hasDocuments:
+      count: 0
+
+- it: should not create a pod disruption budget when disabled
+  set:
+    operator.podDisruptionBudget.enabled: false
+  asserts:
+  - hasDocuments:
+      count: 0

misc/helm-charts/operator/values.yaml

@@ -17,6 +17,21 @@ operator:
     # -- Policy for pulling the image: "IfNotPresent" avoids unnecessary re-pulling of images
     pullPolicy: IfNotPresent
 
+  # -- Number of operator replicas. The operator uses leader election so that
+  # only one replica reconciles at a time. Running more than one replica
+  # avoids downtime of the CRD conversion webhook during rollouts and node
+  # drains.
+  replicas: 2
+
+  podDisruptionBudget:
+    # -- Whether to create a PodDisruptionBudget for the operator. Only created
+    # when `replicas` is greater than 1, since a budget over a single replica
+    # either blocks node drains or protects nothing.
+    enabled: true
+    # -- Maximum number of operator pods that may be unavailable at once during
+    # voluntary disruptions. Expressed as a maximum rather than a minimum so
+    # that node drains are never blocked outright, they are only serialized.
+    maxUnavailable: 1
 
   args:
     # -- Log filtering settings for startup logs

src/orchestratord/Cargo.toml

@@ -14,7 +14,7 @@ anyhow.workspace = true
 async-trait.workspace = true
 axum.workspace = true
 axum-server.workspace = true
-clap.workspace = true
+clap = { workspace = true, features = ["env"] }
 futures.workspace = true
 gcp_auth.workspace = true
 http.workspace = true