diff --git a/config/samples/kraft/simplekafkacluster_kraft_4broker.yaml b/config/samples/kraft/simplekafkacluster_kraft_4broker.yaml new file mode 100644 index 000000000..766e329ce --- /dev/null +++ b/config/samples/kraft/simplekafkacluster_kraft_4broker.yaml @@ -0,0 +1,307 @@ +apiVersion: kafka.banzaicloud.io/v1beta1 +kind: KafkaCluster +metadata: + labels: + controller-tools.k8s.io: "1.0" + name: kafka +spec: + kRaft: true + monitoringConfig: + jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0" + headlessServiceEnabled: true + propagateLabels: false + oneBrokerPerNode: false + clusterImage: "ghcr.io/adobe/koperator/kafka:2.13-3.9.1" + readOnlyConfig: | + auto.create.topics.enable=false + cruise.control.metrics.topic.auto.create=true + cruise.control.metrics.topic.num.partitions=1 + cruise.control.metrics.topic.replication.factor=2 + # e2e: publish CruiseControl metrics every 15s (default 60s) so CC warms up quickly; not production defaults + cruise.control.metrics.reporter.metrics.reporting.interval.ms=15000 + brokerConfigGroups: + default: + storageConfigs: + - mountPath: "/kafka-logs" + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + broker: + processRoles: + - broker + storageConfigs: + - mountPath: "/kafka-logs-broker" + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + brokerAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9020" + brokers: + # Four broker-only nodes (0,1,2,6) + three controllers. The KRaft scaling e2e upscales the 3-broker + # simplekafkacluster_kraft.yaml to this manifest (adds broker 6) and then downscales back. >=3 brokers + # remain at all times so RF=3 / min.insync.replicas=3 topics stay satisfiable during the reassignment. + - id: 0 + brokerConfigGroup: "broker" + - id: 1 + brokerConfigGroup: "broker" + - id: 2 + brokerConfigGroup: "broker" + - id: 6 + brokerConfigGroup: "broker" + - id: 3 + brokerConfigGroup: "default" + brokerConfig: + processRoles: + - controller + # - broker + - id: 4 + brokerConfigGroup: "default" + brokerConfig: + processRoles: + - controller + - id: 5 + brokerConfigGroup: "default" + brokerConfig: + processRoles: + - controller + rollingUpgradeConfig: + failureThreshold: 1 + listenersConfig: + internalListeners: + - type: "plaintext" + name: "internal" + containerPort: 29092 + usedForInnerBrokerCommunication: true + - type: "plaintext" + name: "controller" + containerPort: 29093 + usedForInnerBrokerCommunication: false + usedForControllerCommunication: true + cruiseControlConfig: + # podSecurityContext: + # runAsNonRoot: false + # securityContext: + # privileged: true + cruiseControlTaskSpec: + RetryDurationMinutes: 5 + topicConfig: + # e2e: fewer partitions on the __CruiseControlMetrics topic => faster CC monitoring coverage + partitions: 3 + replicationFactor: 3 +# resourceRequirements: +# requests: +# cpu: 500m +# memory: 1Gi +# limits: +# cpu: 500m +# memory: 1Gi +# image: "adobe/cruise-control:3.0.3-adbe-20250804" + config: | + # Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. + # + # This is an example property file for Kafka Cruise Control. See KafkaCruiseControlConfig for more details. + # Configuration for the metadata client. + # ======================================= + # The maximum interval in milliseconds between two metadata refreshes. + # e2e: must be <= metric.sampling.interval.ms (CC sanityCheckSamplingPeriod); low for the fast 15s sampling + metadata.max.age.ms=10000 + # Client id for the Cruise Control. It is used for the metadata client. + #client.id=kafka-cruise-control + # The size of TCP send buffer bytes for the metadata client. + #send.buffer.bytes=131072 + # The size of TCP receive buffer size for the metadata client. + #receive.buffer.bytes=131072 + # The time to wait before disconnect an idle TCP connection. + #connections.max.idle.ms=540000 + # The time to wait before reconnect to a given host. + #reconnect.backoff.ms=50 + # The time to wait for a response from a host after sending a request. + #request.timeout.ms=30000 + # Configurations for the load monitor + # ======================================= + # The number of metric fetcher thread to fetch metrics for the Kafka cluster + num.metric.fetchers=1 + # The metric sampler class + metric.sampler.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.CruiseControlMetricsReporterSampler + # Configurations for CruiseControlMetricsReporterSampler + metric.reporter.topic.pattern=__CruiseControlMetrics + # The sample store class name + sample.store.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.KafkaSampleStore + # The config for the Kafka sample store to save the partition metric samples + partition.metric.sample.store.topic=__KafkaCruiseControlPartitionMetricSamples + # The config for the Kafka sample store to save the model training samples + broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples + # The replication factor of Kafka metric sample store topic + sample.store.topic.replication.factor=2 + # e2e: shrink CC's sample-store topics from their 32-partitions-each default so broker/disk removal + # reshuffles only a few partitions (not 64) and the CC execution completes quickly; not production defaults. + # NOTE: CC only ever increases these partition counts (never shrinks), so this only takes effect on a fresh cluster. + partition.sample.store.topic.partition.count=3 + broker.sample.store.topic.partition.count=3 + # The config for the number of Kafka sample store consumer threads + num.sample.loading.threads=8 + # The partition assignor class for the metric samplers + metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor + # The metric sampling interval in milliseconds + # e2e: sample every 15s (default 120s) so CC accrues valid windows fast; not production defaults + metric.sampling.interval.ms=15000 + # e2e: CC reads the reporter interval for a sanity check and requires it <= the sampling interval + cruise.control.metrics.reporter.metrics.reporting.interval.ms=15000 + # e2e: must be >= metric.sampling.interval.ms (CC sanityCheckSamplingPeriod) + metric.anomaly.detection.interval.ms=180000 + # The partition metrics window size in milliseconds + # e2e: shrink window to the 15s sampling interval so CC accrues a valid window fast; not production defaults + partition.metrics.window.ms=15000 + # The number of partition metric windows to keep in memory + num.partition.metrics.windows=1 + # The minimum partition metric samples required for a partition in each window + min.samples.per.partition.metrics.window=1 + # The broker metrics window size in milliseconds + # e2e: shrink window to the 15s sampling interval to speed up CC warmup; not production defaults + broker.metrics.window.ms=15000 + # The number of broker metric windows to keep in memory + # e2e: only 2 broker windows needed for warmup on a tiny static cluster (was 20); not production defaults + num.broker.metrics.windows=2 + # The minimum broker metric samples required for a partition in each window + min.samples.per.broker.metrics.window=1 + # The configuration for the BrokerCapacityConfigFileResolver (supports JBOD and non-JBOD broker capacities) + capacity.config.file=config/capacity.json + #capacity.config.file=config/capacityJBOD.json + # Configurations for the analyzer + # ======================================= + # The list of goals to optimize the Kafka cluster for with pre-computed proposals + default.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal + # The list of supported goals + goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.kafkaassigner.KafkaAssignerDiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PreferredLeaderElectionGoal + # The list of supported hard goals + hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The minimum percentage of well monitored partitions out of all the partitions + # e2e: lowered from CC's 0.995 default so the load model becomes valid quickly on a tiny cluster + # with few partitions/windows; not production defaults + min.valid.partition.ratio=0.5 + # The balance threshold for CPU + cpu.balance.threshold=1.1 + # The balance threshold for disk + disk.balance.threshold=1.1 + # The balance threshold for network inbound utilization + network.inbound.balance.threshold=1.1 + # The balance threshold for network outbound utilization + network.outbound.balance.threshold=1.1 + # The balance threshold for the replica count + replica.count.balance.threshold=1.1 + # The capacity threshold for CPU in percentage + cpu.capacity.threshold=0.8 + # The capacity threshold for disk in percentage + disk.capacity.threshold=0.8 + # The capacity threshold for network inbound utilization in percentage + network.inbound.capacity.threshold=0.8 + # The capacity threshold for network outbound utilization in percentage + network.outbound.capacity.threshold=0.8 + # The threshold to define the cluster to be in a low CPU utilization state + cpu.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + disk.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low network inbound utilization state + network.inbound.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + network.outbound.low.utilization.threshold=0.0 + # The metric anomaly percentile upper threshold + metric.anomaly.percentile.upper.threshold=90.0 + # The metric anomaly percentile lower threshold + metric.anomaly.percentile.lower.threshold=10.0 + # How often should the cached proposal be expired and recalculated if necessary + proposal.expiration.ms=60000 + # The maximum number of replicas that can reside on a broker at any given time. + max.replicas.per.broker=10000 + # The number of threads to use for proposal candidate precomputing. + num.proposal.precompute.threads=1 + # the topics that should be excluded from the partition movement. + #topics.excluded.from.partition.movement + # Configurations for the executor + # ======================================= + # The max number of partitions to move in/out on a given broker at a given time. + num.concurrent.partition.movements.per.broker=10 + # The interval between two execution progress checks. + # e2e: 5s is the floor (must be >= min.execution.progress.check.interval.ms, default 5000); was 10s + execution.progress.check.interval.ms=5000 + # Configurations for anomaly detector + # ======================================= + # The goal violation notifier class + anomaly.notifier.class=com.linkedin.kafka.cruisecontrol.detector.notifier.SelfHealingNotifier + # The metric anomaly finder class + metric.anomaly.finder.class=com.linkedin.kafka.cruisecontrol.detector.KafkaMetricAnomalyFinder + # The anomaly detection interval + anomaly.detection.interval.ms=10000 + # The goal violation to detect. + anomaly.detection.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The interested metrics for metric anomaly analyzer. + metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_MAX,BROKER_PRODUCE_LOCAL_TIME_MS_MEAN,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MAX,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MAX,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_LOG_FLUSH_TIME_MS_MAX,BROKER_LOG_FLUSH_TIME_MS_MEAN + ## Adjust accordingly if your metrics reporter is an older version and does not produce these metrics. + #metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_50TH,BROKER_PRODUCE_LOCAL_TIME_MS_999TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_50TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_999TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_50TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_999TH,BROKER_LOG_FLUSH_TIME_MS_50TH,BROKER_LOG_FLUSH_TIME_MS_999TH + # The cluster configurations for the KafkaTopicConfigProvider + cluster.configs.file=config/clusterConfigs.json + # The maximum time in milliseconds to store the response and access details of a completed user task. + completed.user.task.retention.time.ms=21600000 + # The maximum time in milliseconds to retain the demotion history of brokers. + demotion.history.retention.time.ms=86400000 + # The maximum number of completed user tasks for which the response and access details will be cached. + max.cached.completed.user.tasks=500 + # The maximum number of user tasks for concurrently running in async endpoints across all users. + max.active.user.tasks=25 + # Enable self healing for all anomaly detectors, unless the particular anomaly detector is explicitly disabled + self.healing.enabled=true + # Enable self healing for broker failure detector + #self.healing.broker.failure.enabled=true + # Enable self healing for goal violation detector + #self.healing.goal.violation.enabled=true + # Enable self healing for metric anomaly detector + #self.healing.metric.anomaly.enabled=true + # configurations for the webserver + # ================================ + # HTTP listen port + webserver.http.port=9090 + # HTTP listen address + webserver.http.address=0.0.0.0 + # Whether CORS support is enabled for API or not + webserver.http.cors.enabled=false + # Value for Access-Control-Allow-Origin + webserver.http.cors.origin=http://localhost:8080/ + # Value for Access-Control-Request-Method + webserver.http.cors.allowmethods=OPTIONS,GET,POST + # Headers that should be exposed to the Browser (Webapp) + # This is a special header that is used by the + # User Tasks subsystem and should be explicitly + # Enabled when CORS mode is used as part of the + # Admin Interface + webserver.http.cors.exposeheaders=User-Task-ID + # REST API default prefix + # (dont forget the ending *) + webserver.api.urlprefix=/kafkacruisecontrol/* + # Location where the Cruise Control frontend is deployed + webserver.ui.diskpath=./cruise-control-ui/dist/ + # URL path prefix for UI + # (dont forget the ending *) + webserver.ui.urlprefix=/* + # Time After which request is converted to Async + webserver.request.maxBlockTimeMs=10000 + # Default Session Expiry Period + webserver.session.maxExpiryTimeMs=60000 + # Session cookie path + webserver.session.path=/ + # Server Access Logs + webserver.accesslog.enabled=true + # Location of HTTP Request Logs + webserver.accesslog.path=access.log + # HTTP Request Log retention days + webserver.accesslog.retention.days=14 + clusterConfig: | + { + "min.insync.replicas": 3 + } diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index be7d7011e..fa4512dd8 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -25,6 +25,7 @@ import ( "emperror.dev/errors" "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -224,6 +225,11 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques return requeueAfter(defaultRequeueIntervalInSeconds) } + // Do not execute a broker add/remove while the Cruise Control Deployment is mid-rollout (see #301). + if result, handled, err := r.requeueIfCCDeploymentNotRolledOut(ctx, log, kafkaCluster, ccOperationExecution.CurrentTaskOperation()); handled { + return result, err + } + log.Info("executing Cruise Control task", "operation", ccOperationExecution.CurrentTaskOperation(), "parameters", ccOperationExecution.CurrentTaskParameters()) // Executing operation cruseControlTaskResult, err := r.executeOperation(ctx, ccOperationExecution) @@ -270,6 +276,71 @@ func (r *CruiseControlOperationReconciler) addFinalizer(ctx context.Context, cur return nil } +// requeueIfCCDeploymentNotRolledOut defers execution of a broker add_broker/remove_broker operation while +// the Cruise Control Deployment is mid-rollout. A broker add/remove regenerates capacity.json, which (being +// hashed into the CC pod template) rolls the CC Deployment; during that RollingUpdate two CC pods briefly +// run behind one Service, so submitting/tracking the operation against a rolling CC makes the fresh pod lose +// the in-memory task and reset its metric-sampling window, stalling the operation (see #301). Waiting for a +// settled, single-Ready-replica Deployment also guarantees an add runs against a CC that has already loaded +// the new broker's exact capacity from capacity.json (no dependency on capacity estimation). Stop-execution +// is never gated. The returned bool reports whether the caller should return the (result, error) as-is. +func (r *CruiseControlOperationReconciler) requeueIfCCDeploymentNotRolledOut(ctx context.Context, log logr.Logger, + kafkaCluster *banzaiv1beta1.KafkaCluster, op banzaiv1alpha1.CruiseControlTaskOperation) (ctrl.Result, bool, error) { + if op != banzaiv1alpha1.OperationAddBroker && op != banzaiv1alpha1.OperationRemoveBroker { + return ctrl.Result{}, false, nil + } + + deployment := &appsv1.Deployment{} + key := client.ObjectKey{ + Name: fmt.Sprintf("%s-cruisecontrol", kafkaCluster.Name), + Namespace: kafkaCluster.Namespace, + } + if err := r.Get(ctx, key, deployment); err != nil { + if apiErrors.IsNotFound(err) { + // No Cruise Control Deployment: no rollout in progress to race with, so do not gate. + return ctrl.Result{}, false, nil + } + result, wErr := requeueWithError(log, "could not determine Cruise Control Deployment rollout state", err) + return result, true, wErr + } + + if isDeploymentRolling(deployment) { + log.Info("requeue: Cruise Control Deployment is mid-rollout; deferring broker operation to avoid racing a CC restart", "operation", op) + result, _ := requeueAfter(defaultRequeueIntervalInSeconds) + return result, true, nil + } + return ctrl.Result{}, false, nil +} + +// isDeploymentRolling reports whether a Deployment is actively in the middle of a rollout: a new pod +// template has been applied but not yet observed by the Deployment controller, its pods have surged above +// the desired count, or not all running replicas are the latest revision yet. It deliberately keys off +// positive evidence of an in-progress rollout rather than "fully settled" so that a Deployment whose status +// has never been populated (observedGeneration == 0, e.g. under envtest where no Deployment controller +// runs) reads as NOT rolling and the check does not block. Initial CruiseControl availability is enforced +// separately by CruiseControlStatus.IsReady; this gate only guards against submitting a broker operation +// while an already-running CC is being re-rolled (e.g. by a capacity.json change). +func isDeploymentRolling(deployment *appsv1.Deployment) bool { + specReplicas := int32(1) + if deployment.Spec.Replicas != nil { + specReplicas = *deployment.Spec.Replicas + } + s := deployment.Status + switch { + case s.ObservedGeneration != 0 && deployment.Generation > s.ObservedGeneration: + // A new pod template was applied but the Deployment controller has not observed it yet. + return true + case s.Replicas > specReplicas: + // RollingUpdate surge: an old-revision pod is still running alongside the new one. + return true + case s.UpdatedReplicas < s.Replicas: + // Not all running pods are the latest revision yet. + return true + default: + return false + } +} + func (r *CruiseControlOperationReconciler) executeOperation(ctx context.Context, ccOperationExecution *banzaiv1alpha1.CruiseControlOperation) (*scale.Result, error) { var cruseControlTaskResult *scale.Result var err error diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index 2cbb84ddf..f3c2964a5 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -23,6 +23,7 @@ import ( "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -54,6 +55,48 @@ func createCCRetryExecutionOperation(createTime time.Time, id string, operation } } +func TestIsDeploymentRolling(t *testing.T) { + i32 := func(v int32) *int32 { return &v } + dep := func(generation, observedGeneration int64, specReplicas, replicas, updated int32) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: v1.ObjectMeta{Generation: generation}, + Spec: appsv1.DeploymentSpec{Replicas: i32(specReplicas)}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: observedGeneration, + Replicas: replicas, + UpdatedReplicas: updated, + }, + } + } + + tests := []struct { + name string + d *appsv1.Deployment + want bool + }{ + {"settled single replica is not rolling", dep(3, 3, 1, 1, 1), false}, + {"new pod template not yet observed is rolling", dep(4, 3, 1, 1, 1), true}, + {"surge: an old-revision pod still present is rolling", dep(3, 3, 1, 2, 1), true}, + {"not all running replicas updated yet is rolling", dep(3, 3, 1, 1, 0), true}, + // envtest / no Deployment controller: status never populated (observedGeneration == 0). Must read + // as NOT rolling so the gate does not block where nothing rolls the Deployment. + {"unpopulated status (observedGeneration 0) is not rolling", dep(1, 0, 1, 0, 0), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isDeploymentRolling(tt.d)) + }) + } + + t.Run("nil spec.replicas defaults to 1; settled is not rolling", func(t *testing.T) { + d := &appsv1.Deployment{ + ObjectMeta: v1.ObjectMeta{Generation: 1}, + Status: appsv1.DeploymentStatus{ObservedGeneration: 1, Replicas: 1, UpdatedReplicas: 1}, + } + assert.False(t, isDeploymentRolling(d)) + }) +} + func TestSortOperations(t *testing.T) { timeNow := time.Now() testCases := []struct { diff --git a/pkg/resources/cruisecontrol/cruisecontrol.go b/pkg/resources/cruisecontrol/cruisecontrol.go index a361e98c8..7f489d8b4 100644 --- a/pkg/resources/cruisecontrol/cruisecontrol.go +++ b/pkg/resources/cruisecontrol/cruisecontrol.go @@ -18,6 +18,7 @@ package cruisecontrol import ( "context" "fmt" + "strconv" "emperror.dev/errors" "github.com/go-logr/logr" @@ -112,7 +113,7 @@ func (r *Reconciler) Reconcile(log logr.Logger) error { } var config *corev1.ConfigMap - if isBrokerDeletionInProgress(r.KafkaCluster.Status.BrokersState) { + if isBrokerDeletionInProgress(r.KafkaCluster.Status.BrokersState) || isBrokerRemovalPending(r.KafkaCluster) { key := types.NamespacedName{ Name: fmt.Sprintf(configAndVolumeNameTemplate, r.KafkaCluster.Name), Namespace: r.KafkaCluster.Namespace, @@ -201,3 +202,27 @@ func isBrokerDeletionInProgress(brokerState map[string]v1beta1.BrokerState) bool } return false } + +// isBrokerRemovalPending reports whether a broker that is still present in the status has already been +// dropped from the spec - i.e. a removal that has not yet been marked as a Cruise Control downscale. +// +// During this window the operator must keep reusing the already-deployed capacity.json instead of +// regenerating a fallback entry for the departing broker. Regenerating it changes capacity.json, which +// (because capacity.json is hashed into the Cruise Control pod template) rolls the Cruise Control +// Deployment and resets CC's metric-sampling window - keeping CC un-ready exactly when +// reconcileKafkaPodDelete needs CC ready (via BrokersWithState) to mark the downscale. That chicken-and-egg +// otherwise prevents the remove_broker operation from ever being created and stalls the removal (see #301). +// Once the pod is gone and the broker is dropped from the status too, capacity.json shrinks with a single +// harmless roll and no operation in flight. +func isBrokerRemovalPending(kafkaCluster *v1beta1.KafkaCluster) bool { + specBrokerIDs := make(map[string]struct{}, len(kafkaCluster.Spec.Brokers)) + for i := range kafkaCluster.Spec.Brokers { + specBrokerIDs[strconv.Itoa(int(kafkaCluster.Spec.Brokers[i].Id))] = struct{}{} + } + for brokerID := range kafkaCluster.Status.BrokersState { + if _, ok := specBrokerIDs[brokerID]; !ok { + return true + } + } + return false +} diff --git a/pkg/resources/cruisecontrol/cruisecontrol_test.go b/pkg/resources/cruisecontrol/cruisecontrol_test.go new file mode 100644 index 000000000..39916de43 --- /dev/null +++ b/pkg/resources/cruisecontrol/cruisecontrol_test.go @@ -0,0 +1,70 @@ +// Copyright 2025 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cruisecontrol + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/banzaicloud/koperator/api/v1beta1" +) + +func TestIsBrokerRemovalPending(t *testing.T) { + cluster := func(specIDs []int32, statusIDs []string) *v1beta1.KafkaCluster { + kc := &v1beta1.KafkaCluster{} + for _, id := range specIDs { + kc.Spec.Brokers = append(kc.Spec.Brokers, v1beta1.Broker{Id: id}) + } + kc.Status.BrokersState = map[string]v1beta1.BrokerState{} + for _, id := range statusIDs { + kc.Status.BrokersState[id] = v1beta1.BrokerState{} + } + return kc + } + + tests := []struct { + testName string + cluster *v1beta1.KafkaCluster + expected bool + }{ + { + testName: "steady state: every status broker is in the spec", + cluster: cluster([]int32{0, 1, 2}, []string{"0", "1", "2"}), + expected: false, + }, + { + testName: "removal pending: a status broker was dropped from the spec", + cluster: cluster([]int32{0, 1, 2}, []string{"0", "1", "2", "6"}), + expected: true, + }, + { + testName: "upscale in progress: a new spec broker not yet in status is NOT a removal", + cluster: cluster([]int32{0, 1, 2, 6}, []string{"0", "1", "2"}), + expected: false, + }, + { + testName: "empty status", + cluster: cluster([]int32{0, 1, 2}, nil), + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.testName, func(t *testing.T) { + require.Equal(t, test.expected, isBrokerRemovalPending(test.cluster)) + }) + } +} diff --git a/tests/e2e/koperator_suite_test.go b/tests/e2e/koperator_suite_test.go index 189f41305..41247b85a 100644 --- a/tests/e2e/koperator_suite_test.go +++ b/tests/e2e/koperator_suite_test.go @@ -77,6 +77,7 @@ var _ = ginkgo.When("Testing e2e test altogether", ginkgo.Ordered, func() { testInstallKafkaCluster("../../config/samples/kraft/simplekafkacluster_kraft.yaml") testProduceConsumeInternal() testJmxExporter() + testKRaftBrokerScaling() testUninstallKafkaCluster() testUninstall() snapshotClusterAndCompare(snapshottedInfo) diff --git a/tests/e2e/test_kraft_broker_scaling.go b/tests/e2e/test_kraft_broker_scaling.go new file mode 100644 index 000000000..0ed62c31f --- /dev/null +++ b/tests/e2e/test_kraft_broker_scaling.go @@ -0,0 +1,170 @@ +// Copyright 2025 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e + +package e2e + +import ( + "context" + + "github.com/gruntwork-io/terratest/modules/k8s" + ginkgo "github.com/onsi/ginkgo/v2" + gomega "github.com/onsi/gomega" + + "github.com/banzaicloud/koperator/api/v1alpha1" + "github.com/banzaicloud/koperator/api/v1beta1" +) + +// testKRaftBrokerScaling upscales a KRaft cluster from 3 broker-only nodes to 4 (add broker 6) and then +// downscales it back to 3, asserting Cruise Control drives each direction with exactly one +// add_broker / remove_broker operation, the broker-only pod count tracks the spec, and the 3 +// controller-only pods are never touched (controller-only nodes are not CC brokers). +// +// This is the regression test for #301, and it deliberately exercises BOTH directions: +// - UPSCALE is the case a "stop hashing capacity.json" fix would have silently broken: the capacity roll +// is what loads the new broker's exact capacity into CC, so the fix must keep rolling on add and +// sequence the add_broker op after the roll settles. +// - DOWNSCALE is the case that stalled on master: the pre-downscale capacity roll kept CC un-ready so the +// remove_broker op was never created. +func testKRaftBrokerScaling() bool { + return ginkgo.When("KRaft broker scaling: upscale then downscale, controllers untouched", func() { + var kubectlOptions k8s.KubectlOptions + var err error + + ginkgo.It("Acquiring K8s config and context", func() { + kubectlOptions, err = kubectlOptionsForCurrentContext() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + kubectlOptions.Namespace = koperatorLocalHelmDescriptor.Namespace + }) + + ginkgo.It("Waiting for Cruise Control to be ready and settled before upscale", func() { + ginkgo.By("Ensuring the KafkaCluster is running") + gomega.Expect(waitForKafkaClusterWithPodStatusCheck(kubectlOptions, kafkaClusterName, kafkaClusterResourceReadinessTimeout)).NotTo(gomega.HaveOccurred()) + ginkgo.By("Waiting until no Cruise Control operation is in flight (initial rebalance finished)") + gomega.Eventually(context.Background(), func() (bool, error) { + return hasNoInFlightCruiseControlOperation(kubectlOptions) + }, batchedBrokerRemovalTimeout, batchedBrokerRemovalPollInterval).Should(gomega.BeTrue()) + ginkgo.By("Waiting until the Cruise Control Deployment is fully rolled out (single Ready replica)") + gomega.Eventually(context.Background(), func() (bool, error) { + return isCruiseControlDeploymentRolledOut(kubectlOptions) + }, batchedBrokerRemovalTimeout, batchedBrokerRemovalPollInterval).Should(gomega.BeTrue()) + }) + + ginkgo.It("Asserting the cluster starts with 3 broker-only pods and 3 controller-only pods", func() { + ok, err := hasExactlyNBrokerPods(kubectlOptions, 3) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(ok).To(gomega.BeTrue(), "expected 3 broker-only pods before scaling") + ok, err = hasExactlyNControllerPods(kubectlOptions, 3) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(ok).To(gomega.BeTrue(), "expected 3 controller-only pods before scaling") + }) + + // --- Upscale: add broker 6 --- + + ginkgo.It("Applying the 4-broker KRaft manifest to add broker 6", func() { + applyK8sResourceManifest(kubectlOptions, "../../config/samples/kraft/simplekafkacluster_kraft_4broker.yaml") + }) + + ginkgo.It("Waiting for exactly one add_broker CruiseControlOperation", func() { + gomega.Eventually(context.Background(), func() (bool, error) { + return hasExactlyNBrokerOperations(kubectlOptions, v1alpha1.OperationAddBroker, 1) + }, batchedBrokerRemovalTimeout, batchedBrokerRemovalPollInterval).Should(gomega.BeTrue()) + }) + + ginkgo.It("Waiting for broker 6 to join (4 broker-only pods) and the cluster to be healthy", func() { + gomega.Eventually(context.Background(), func() (bool, error) { + return hasExactlyNBrokerPods(kubectlOptions, 4) + }, batchedBrokerRemovalTimeout, batchedBrokerRemovalPollInterval).Should(gomega.BeTrue()) + gomega.Expect(waitForKafkaClusterWithPodStatusCheck(kubectlOptions, kafkaClusterName, kafkaClusterResourceReadinessTimeout)).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("Asserting controllers are untouched after upscale (still 3 controller-only pods)", func() { + ok, err := hasExactlyNControllerPods(kubectlOptions, 3) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(ok).To(gomega.BeTrue(), "controller-only pods must not be affected by an upscale") + }) + + // --- Downscale: remove broker 6 --- + + ginkgo.It("Waiting for Cruise Control to be settled again before downscale", func() { + gomega.Eventually(context.Background(), func() (bool, error) { + return hasNoInFlightCruiseControlOperation(kubectlOptions) + }, batchedBrokerRemovalTimeout, batchedBrokerRemovalPollInterval).Should(gomega.BeTrue()) + gomega.Eventually(context.Background(), func() (bool, error) { + return isCruiseControlDeploymentRolledOut(kubectlOptions) + }, batchedBrokerRemovalTimeout, batchedBrokerRemovalPollInterval).Should(gomega.BeTrue()) + }) + + ginkgo.It("Applying the 3-broker KRaft manifest to remove broker 6", func() { + applyK8sResourceManifest(kubectlOptions, "../../config/samples/kraft/simplekafkacluster_kraft.yaml") + }) + + ginkgo.It("Waiting for exactly one remove_broker CruiseControlOperation", func() { + gomega.Eventually(context.Background(), func() (bool, error) { + return hasExactlyNBrokerOperations(kubectlOptions, v1alpha1.OperationRemoveBroker, 1) + }, batchedBrokerRemovalTimeout, batchedBrokerRemovalPollInterval).Should(gomega.BeTrue()) + }) + + ginkgo.It("Waiting for broker 6 to be removed (3 broker-only pods) and the cluster to be healthy", func() { + gomega.Eventually(context.Background(), func() (bool, error) { + return hasExactlyNBrokerPods(kubectlOptions, 3) + }, batchedBrokerRemovalTimeout, batchedBrokerRemovalPollInterval).Should(gomega.BeTrue()) + gomega.Expect(waitForKafkaClusterWithPodStatusCheck(kubectlOptions, kafkaClusterName, kafkaClusterResourceReadinessTimeout)).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("Asserting controllers are untouched after downscale (still 3 controller-only pods)", func() { + ok, err := hasExactlyNControllerPods(kubectlOptions, 3) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(ok).To(gomega.BeTrue(), "controller-only pods must not be affected by a downscale") + }) + }) +} + +// hasExactlyNBrokerOperations returns true when exactly n CruiseControlOperations whose current task is the +// given operation type exist in the namespace. +func hasExactlyNBrokerOperations(kubectlOptions k8s.KubectlOptions, operation v1alpha1.CruiseControlTaskOperation, n int) (bool, error) { + ops, err := getK8sResources(kubectlOptions, + []string{"cruisecontroloperation"}, + "", + "", + "-o", "jsonpath={range .items[*]}{.status.currentTask.operation}{'\\n'}{end}", + ) + if err != nil { + return false, err + } + count := 0 + for _, op := range ops { + if op == string(operation) { + count++ + } + } + return count == n, nil +} + +// hasExactlyNControllerPods returns true when exactly n controller pods (isControllerNode=true) exist and +// are Running in the namespace. +func hasExactlyNControllerPods(kubectlOptions k8s.KubectlOptions, n int) (bool, error) { + pods, err := getK8sResources(kubectlOptions, + []string{"pod"}, + v1beta1.KafkaCRLabelKey+"="+kafkaClusterName+","+kafkaLabelSelectorControllers, + "", + "--field-selector=status.phase=Running", + "-o", "name", + ) + if err != nil { + return false, err + } + return len(pods) == n, nil +}