diff --git a/cmd/operator/app/thread_keeper.go b/cmd/operator/app/thread_keeper.go index 053cd6cd4..51dc7788d 100644 --- a/cmd/operator/app/thread_keeper.go +++ b/cmd/operator/app/thread_keeper.go @@ -17,7 +17,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/predicate" - // ctrl "sigs.k8s.io/controller-runtime/pkg/controller" + ctrlController "sigs.k8s.io/controller-runtime/pkg/controller" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/chop" @@ -84,6 +84,9 @@ func initKeeper(ctx context.Context) error { builder.WithPredicates(keeperPredicate()), ). Owns(&apps.StatefulSet{}). + WithOptions(ctrlController.Options{ + MaxConcurrentReconciles: chop.Config().Reconcile.Runtime.ReconcileCHKsThreadsNumber, + }). Complete( &controller.Controller{ Client: manager.GetClient(), diff --git a/config/config-dev.yaml b/config/config-dev.yaml index 072a9b9ca..4edb872d2 100644 --- a/config/config-dev.yaml +++ b/config/config-dev.yaml @@ -316,6 +316,8 @@ reconcile: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 1 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently diff --git a/config/config.yaml b/config/config.yaml index b29a88d42..23d35a095 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -419,6 +419,8 @@ reconcile: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently diff --git a/deploy/builder/templates-config/config.yaml b/deploy/builder/templates-config/config.yaml index a9eb7797e..9b3c37677 100644 --- a/deploy/builder/templates-config/config.yaml +++ b/deploy/builder/templates-config/config.yaml @@ -413,6 +413,8 @@ reconcile: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml index a30a06953..6be3c6eb4 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml @@ -339,6 +339,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml index d5c4c3135..fe4a1208a 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -339,6 +339,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 diff --git a/deploy/helm/clickhouse-operator/values.schema.json b/deploy/helm/clickhouse-operator/values.schema.json index 64e907cf3..f51eda897 100644 --- a/deploy/helm/clickhouse-operator/values.schema.json +++ b/deploy/helm/clickhouse-operator/values.schema.json @@ -480,6 +480,9 @@ "reconcileCHIsThreadsNumber": { "type": "integer" }, + "reconcileCHKsThreadsNumber": { + "type": "integer" + }, "reconcileShardsMaxConcurrencyPercent": { "type": "integer" }, diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml index 9091a4da9..6eb50dbab 100644 --- a/deploy/helm/clickhouse-operator/values.yaml +++ b/deploy/helm/clickhouse-operator/values.yaml @@ -699,6 +699,8 @@ configs: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently # can not be greater than 'reconcileShardsThreadsNumber'. diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml index a0c3954d6..718a53687 100644 --- a/deploy/operator/clickhouse-operator-install-ansible.yaml +++ b/deploy/operator/clickhouse-operator-install-ansible.yaml @@ -4052,6 +4052,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -6105,7 +6110,9 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 - + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 + # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently # can not be greater than 'reconcileShardsThreadsNumber'. diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml index 0716b6803..6419ab335 100644 --- a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml @@ -4019,6 +4019,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -6304,6 +6309,8 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml index 62a497785..825cb0fff 100644 --- a/deploy/operator/clickhouse-operator-install-bundle.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle.yaml @@ -4045,6 +4045,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -6364,7 +6369,9 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 - + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 + # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently # can not be greater than 'reconcileShardsThreadsNumber'. diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml index 13ce1cecf..962b9f349 100644 --- a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml @@ -4019,6 +4019,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -6051,6 +6056,8 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml index 70d9f948f..fc83e599a 100644 --- a/deploy/operator/clickhouse-operator-install-template.yaml +++ b/deploy/operator/clickhouse-operator-install-template.yaml @@ -4045,6 +4045,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -6098,7 +6103,9 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 - + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 + # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently # can not be greater than 'reconcileShardsThreadsNumber'. diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml index 5bb2526e0..cc50ce437 100644 --- a/deploy/operator/clickhouse-operator-install-tf.yaml +++ b/deploy/operator/clickhouse-operator-install-tf.yaml @@ -4052,6 +4052,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -6105,7 +6110,9 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 - + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 + # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently # can not be greater than 'reconcileShardsThreadsNumber'. diff --git a/deploy/operator/parts/crd.yaml b/deploy/operator/parts/crd.yaml index 0be3adbfb..9acb4118e 100644 --- a/deploy/operator/parts/crd.yaml +++ b/deploy/operator/parts/crd.yaml @@ -8589,6 +8589,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go index f9610f1ab..b9cd5d7b0 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go @@ -115,6 +115,10 @@ const ( // Used in case no other specified in config defaultReconcileCHIsThreadsNumber = 1 + // defaultReconcileCHKsThreadsNumber specifies default number of CHK controller threads running concurrently. + // Used in case no other specified in config + defaultReconcileCHKsThreadsNumber = 1 + // defaultReconcileShardsThreadsNumber specifies the default number of threads usable for concurrent shard reconciliation // within a single cluster reconciliation. Defaults to 1, which means strictly sequential shard reconciliation. defaultReconcileShardsThreadsNumber = 1 @@ -663,6 +667,7 @@ type OperatorConfigReconcileRecoveryCompletedScope struct { type OperatorConfigReconcileRuntime struct { ReconcileCHIsThreadsNumber int `json:"reconcileCHIsThreadsNumber" yaml:"reconcileCHIsThreadsNumber"` + ReconcileCHKsThreadsNumber int `json:"reconcileCHKsThreadsNumber" yaml:"reconcileCHKsThreadsNumber"` ReconcileShardsThreadsNumber int `json:"reconcileShardsThreadsNumber" yaml:"reconcileShardsThreadsNumber"` ReconcileShardsMaxConcurrencyPercent int `json:"reconcileShardsMaxConcurrencyPercent" yaml:"reconcileShardsMaxConcurrencyPercent"` @@ -1399,6 +1404,9 @@ func (c *OperatorConfig) normalizeSectionReconcileRuntime() { if c.Reconcile.Runtime.ReconcileCHIsThreadsNumber == 0 { c.Reconcile.Runtime.ReconcileCHIsThreadsNumber = defaultReconcileCHIsThreadsNumber } + if c.Reconcile.Runtime.ReconcileCHKsThreadsNumber == 0 { + c.Reconcile.Runtime.ReconcileCHKsThreadsNumber = defaultReconcileCHKsThreadsNumber + } if c.Reconcile.Runtime.ReconcileShardsThreadsNumber == 0 { c.Reconcile.Runtime.ReconcileShardsThreadsNumber = defaultReconcileShardsThreadsNumber }