From 69e789b8f4fe63946c4b8360092fc83a1b44c86b Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Fri, 10 Jul 2026 20:43:53 +0200 Subject: [PATCH 1/6] Add Envoy Gateway (Gateway API) ingress support for Kafka external listeners Adds a new "envoygateway" ingress controller for exposing Kafka external listeners, built on the Kubernetes Gateway API (https://gateway.envoyproxy.io/). Enable it per external listener with `spec.ingressController: envoygateway` and configure it via the new `envoyGatewayConfig` field. The `ingressController` enum is now `envoy;contour;envoygateway`. The previous istio-based ingress relied on banzaicloud/istio-operator, which is EOL/unmaintained (already removed on master); Envoy Gateway is a CNCF-backed, actively maintained Gateway API implementation. Changes: - New envoygateway resource reconciler (pkg/resources/envoygateway): generates Gateway + TCPRoute, with optional TLS. - New EnvoyGatewayIngressConfig API type + `ingressController: envoygateway`. - Refactor external listener status handling into a per-listener dispatcher (createListenerStatuses) so ingress types plug in cleanly. - Sample manifests for ZooKeeper and KRaft modes. - Additional unit tests (envoy resource, k8sutil, nodeport external access) and e2e coverage (install/produce/consume/uninstall with Envoy Gateway ingress in both ZooKeeper and KRaft modes, Envoy Gateway Helm chart v1.5.4). Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile | 13 +- api/v1beta1/kafkacluster_types.go | 61 +- api/v1beta1/zz_generated.deepcopy.go | 28 + charts/kafka-operator/crds/kafkaclusters.yaml | 53 + .../templates/operator-rbac.yaml | 14 + .../kafka.banzaicloud.io_kafkaclusters.yaml | 53 + config/base/rbac/role.yaml | 14 + .../banzaicloud_v1beta1_kafkacluster.yaml | 40 +- .../kraft/simplekafkacluster_kraft.yaml | 1 + .../simplekafkacluster_kraft_with_envoy.yaml | 278 ++ ...ekafkacluster_kraft_with_envoygateway.yaml | 311 ++ .../simplekafkacluster_with_envoy.yaml | 258 ++ .../simplekafkacluster_with_envoygateway.yaml | 288 ++ ...eway.networking.k8s.io_gatewayclasses.yaml | 515 +++ .../gateway.networking.k8s.io_gateways.yaml | 3283 +++++++++++++++++ .../gateway.networking.k8s.io_tcproutes.yaml | 756 ++++ controllers/kafkacluster_controller.go | 5 + ...fkacluster_controller_envoygateway_test.go | 173 + controllers/tests/suite_test.go | 5 + go.mod | 2 +- main.go | 5 + pkg/k8sutil/resource_test.go | 277 ++ pkg/resources/envoy/deployment.go | 2 +- pkg/resources/envoy/envoy.go | 9 +- pkg/resources/envoy/envoy_test.go | 302 ++ pkg/resources/envoygateway/envoygateway.go | 170 + .../envoygateway/envoygateway_test.go | 237 ++ pkg/resources/envoygateway/gateway.go | 132 + pkg/resources/envoygateway/tcproute.go | 135 + pkg/resources/kafka/kafka.go | 164 +- .../nodeportExternalAccess_test.go | 233 ++ pkg/util/envoygateway/common.go | 40 + pkg/util/util.go | 29 + pkg/webhooks/kafkacluster_validator.go | 6 + pkg/webhooks/kafkacluster_validator_test.go | 17 + run-e2e.sh | 15 +- tests/e2e/const.go | 1 + tests/e2e/global.go | 19 + tests/e2e/kcat.go | 9 +- tests/e2e/koperator_suite_test.go | 32 +- tests/e2e/test_install.go | 29 +- tests/e2e/test_install_cluster.go | 52 - tests/e2e/test_install_kafka_cluster.go | 82 + tests/e2e/test_snapshot.go | 76 +- tests/e2e/test_uninstall.go | 5 + tests/e2e/test_uninstall_cluster.go | 28 + tests/e2e/types.go | 16 +- tests/e2e/uninstall.go | 66 +- tests/e2e/uninstall_cluster.go | 2 +- tests/e2e/versions.go | 3 + 50 files changed, 8140 insertions(+), 204 deletions(-) create mode 100644 config/samples/kraft/simplekafkacluster_kraft_with_envoy.yaml create mode 100644 config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml create mode 100644 config/samples/simplekafkacluster_with_envoy.yaml create mode 100644 config/samples/simplekafkacluster_with_envoygateway.yaml create mode 100644 config/test/crd/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml create mode 100644 config/test/crd/gateway-api/gateway.networking.k8s.io_gateways.yaml create mode 100644 config/test/crd/gateway-api/gateway.networking.k8s.io_tcproutes.yaml create mode 100644 controllers/tests/kafkacluster_controller_envoygateway_test.go create mode 100644 pkg/k8sutil/resource_test.go create mode 100644 pkg/resources/envoy/envoy_test.go create mode 100644 pkg/resources/envoygateway/envoygateway.go create mode 100644 pkg/resources/envoygateway/envoygateway_test.go create mode 100644 pkg/resources/envoygateway/gateway.go create mode 100644 pkg/resources/envoygateway/tcproute.go create mode 100644 pkg/resources/nodeportexternalaccess/nodeportExternalAccess_test.go create mode 100644 pkg/util/envoygateway/common.go delete mode 100644 tests/e2e/test_install_cluster.go create mode 100644 tests/e2e/test_install_kafka_cluster.go diff --git a/Makefile b/Makefile index 3d9ba57ed..1539dfe43 100644 --- a/Makefile +++ b/Makefile @@ -216,7 +216,7 @@ deploy: install-kustomize install ## Deploy controller into the configured Kuber bin/kustomize build $(KUSTOMIZE_BASE) | kubectl apply -f - # Generate manifests e.g. CRD, RBAC etc. -manifests: bin/controller-gen ## Generate (Kubebuilder) manifests e.g. CRD, RBAC etc. +manifests: bin/controller-gen crds-gatewayapi ## Generate (Kubebuilder) manifests e.g. CRD, RBAC etc. cd api && $(CONTROLLER_GEN) $(CRD_OPTIONS) webhook paths="./..." output:crd:artifacts:config=../config/base/crds output:webhook:artifacts:config=../config/base/webhook $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./controllers/..." output:rbac:artifacts:config=./config/base/rbac ## Regenerate CRDs and RBAC for the helm chart @@ -229,6 +229,17 @@ manifests: bin/controller-gen ## Generate (Kubebuilder) manifests e.g. CRD, RBAC @sed -n '/# RBAC_RULES_END/,$$p' charts/kafka-operator/templates/operator-rbac.yaml >> charts/kafka-operator/templates/operator-rbac.yaml.tmp @mv charts/kafka-operator/templates/operator-rbac.yaml.tmp charts/kafka-operator/templates/operator-rbac.yaml +GATEWAY_API_TEST_CRD_DIR = config/test/crd/gateway-api + +.PHONY: crds-gatewayapi +crds-gatewayapi: ## Regenerate config/test/crd/gateway-api from the sigs.k8s.io/gateway-api version pinned in go.mod. + go mod download sigs.k8s.io/gateway-api + @GATEWAY_API_MOD_DIR=$$(go list -m -f '{{.Dir}}' sigs.k8s.io/gateway-api); \ + rm -f $(GATEWAY_API_TEST_CRD_DIR)/*.yaml; \ + cp "$$GATEWAY_API_MOD_DIR/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml" $(GATEWAY_API_TEST_CRD_DIR)/; \ + cp "$$GATEWAY_API_MOD_DIR/config/crd/standard/gateway.networking.k8s.io_gateways.yaml" $(GATEWAY_API_TEST_CRD_DIR)/; \ + cp "$$GATEWAY_API_MOD_DIR/config/crd/experimental/gateway.networking.k8s.io_tcproutes.yaml" $(GATEWAY_API_TEST_CRD_DIR)/ + fmt: ## Run go fmt against code. go fmt ./... cd api && go fmt ./... diff --git a/api/v1beta1/kafkacluster_types.go b/api/v1beta1/kafkacluster_types.go index 4bc0fe516..e18a0c1eb 100644 --- a/api/v1beta1/kafkacluster_types.go +++ b/api/v1beta1/kafkacluster_types.go @@ -180,7 +180,7 @@ type KafkaClusterSpec struct { RollingUpgradeConfig RollingUpgradeConfig `json:"rollingUpgradeConfig"` // Selector for broker pods that need to be recycled/reconciled TaintedBrokersSelector *metav1.LabelSelector `json:"taintedBrokersSelector,omitempty"` - // +kubebuilder:validation:Enum=envoy;contour + // +kubebuilder:validation:Enum=envoy;contour;envoygateway // IngressController specifies the type of the ingress controller to be used for external listeners. IngressController string `json:"ingressController,omitempty"` // If true OneBrokerPerNode ensures that each kafka broker will be placed on a different node unless a custom @@ -190,13 +190,14 @@ type KafkaClusterSpec struct { // when false, they will be kept so the Kafka cluster remains available for those Kafka clients which are still using the previous ingress setting. // +kubebuilder:default=false // +optional - RemoveUnusedIngressResources bool `json:"removeUnusedIngressResources,omitempty"` - PropagateLabels bool `json:"propagateLabels,omitempty"` - CruiseControlConfig CruiseControlConfig `json:"cruiseControlConfig"` - EnvoyConfig EnvoyConfig `json:"envoyConfig,omitempty"` - ContourIngressConfig ContourIngressConfig `json:"contourIngressConfig,omitempty"` - MonitoringConfig MonitoringConfig `json:"monitoringConfig,omitempty"` - AlertManagerConfig *AlertManagerConfig `json:"alertManagerConfig,omitempty"` + RemoveUnusedIngressResources bool `json:"removeUnusedIngressResources,omitempty"` + PropagateLabels bool `json:"propagateLabels,omitempty"` + CruiseControlConfig CruiseControlConfig `json:"cruiseControlConfig"` + EnvoyConfig EnvoyConfig `json:"envoyConfig,omitempty"` + ContourIngressConfig ContourIngressConfig `json:"contourIngressConfig,omitempty"` + EnvoyGatewayConfig EnvoyGatewayIngressConfig `json:"envoyGatewayConfig,omitempty"` + MonitoringConfig MonitoringConfig `json:"monitoringConfig,omitempty"` + AlertManagerConfig *AlertManagerConfig `json:"alertManagerConfig,omitempty"` // Envs defines environment variables for Kafka broker Pods. // Adding the "+" prefix to the name prepends the value to that environment variable instead of overwriting it. // Add the "+" suffix to append. @@ -599,7 +600,25 @@ func (c EnvoyConfig) GetBrokerHostname(brokerId int32) string { return strings.Replace(c.BrokerHostnameTemplate, "%id", strconv.Itoa(int(brokerId)), 1) } -// We use -1 for ExternalStartingPort value to enable TLS on envoy +// GetBrokerHostname returns the broker hostname for the given broker ID +func (c EnvoyGatewayIngressConfig) GetBrokerHostname(brokerId int32) string { + return strings.Replace(c.BrokerHostnameTemplate, "%id", strconv.Itoa(int(brokerId)), 1) +} + +// GetGatewayClassName returns the GatewayClassName or default value +func (c EnvoyGatewayIngressConfig) GetGatewayClassName() string { + if c.GatewayClassName == "" { + return "eg" + } + return c.GatewayClassName +} + +// GetAnnotations returns the annotations for the Gateway resource +func (c EnvoyGatewayIngressConfig) GetAnnotations() map[string]string { + return util.CloneMap(c.Annotations) +} + +// TLSEnabled We use -1 for ExternalStartingPort value to enable TLS on envoy func (c ExternalListenerConfig) TLSEnabled() bool { return c.ExternalStartingPort == -1 } @@ -697,8 +716,9 @@ type Config struct { type IngressConfig struct { IngressServiceSettings `json:",inline"` - EnvoyConfig *EnvoyConfig `json:"envoyConfig,omitempty"` - ContourIngressConfig *ContourIngressConfig `json:"contourIngressConfig,omitempty"` + EnvoyConfig *EnvoyConfig `json:"envoyConfig,omitempty"` + ContourIngressConfig *ContourIngressConfig `json:"contourIngressConfig,omitempty"` + EnvoyGatewayConfig *EnvoyGatewayIngressConfig `json:"envoyGatewayConfig,omitempty"` } type ContourIngressConfig struct { @@ -708,6 +728,25 @@ type ContourIngressConfig struct { BrokerFQDNTemplate string `json:"brokerFQDNTemplate"` } +type EnvoyGatewayIngressConfig struct { + // GatewayClassName is the name of the GatewayClass resource to use + // +optional + GatewayClassName string `json:"gatewayClassName,omitempty"` + // GatewayName is the name of the Gateway resource to create + // +optional + GatewayName string `json:"gatewayName,omitempty"` + // TLSSecretName is the name of the secret containing TLS certificates for TLS termination + // +optional + TLSSecretName string `json:"tlsSecretName,omitempty"` + // BrokerHostnameTemplate is the template for generating broker hostnames (e.g., "kafka-%id.example.com") + // The %id placeholder will be replaced with the broker ID + // +optional + BrokerHostnameTemplate string `json:"brokerHostnameTemplate,omitempty"` + // Annotations to add to the Gateway resource + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} + // InternalListenerConfig defines the internal listener config for Kafka type InternalListenerConfig struct { CommonListenerSpec `json:",inline"` diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index d5cfaf00d..7b0e0e055 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -540,6 +540,28 @@ func (in *EnvoyConfig) DeepCopy() *EnvoyConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvoyGatewayIngressConfig) DeepCopyInto(out *EnvoyGatewayIngressConfig) { + *out = *in + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyGatewayIngressConfig. +func (in *EnvoyGatewayIngressConfig) DeepCopy() *EnvoyGatewayIngressConfig { + if in == nil { + return nil + } + out := new(EnvoyGatewayIngressConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExternalListenerConfig) DeepCopyInto(out *ExternalListenerConfig) { *out = *in @@ -632,6 +654,11 @@ func (in *IngressConfig) DeepCopyInto(out *IngressConfig) { *out = new(ContourIngressConfig) **out = **in } + if in.EnvoyGatewayConfig != nil { + in, out := &in.EnvoyGatewayConfig, &out.EnvoyGatewayConfig + *out = new(EnvoyGatewayIngressConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressConfig. @@ -784,6 +811,7 @@ func (in *KafkaClusterSpec) DeepCopyInto(out *KafkaClusterSpec) { in.CruiseControlConfig.DeepCopyInto(&out.CruiseControlConfig) in.EnvoyConfig.DeepCopyInto(&out.EnvoyConfig) out.ContourIngressConfig = in.ContourIngressConfig + in.EnvoyGatewayConfig.DeepCopyInto(&out.EnvoyGatewayConfig) out.MonitoringConfig = in.MonitoringConfig if in.AlertManagerConfig != nil { in, out := &in.AlertManagerConfig, &out.AlertManagerConfig diff --git a/charts/kafka-operator/crds/kafkaclusters.yaml b/charts/kafka-operator/crds/kafkaclusters.yaml index a7266064a..37772e2e6 100644 --- a/charts/kafka-operator/crds/kafkaclusters.yaml +++ b/charts/kafka-operator/crds/kafkaclusters.yaml @@ -20837,6 +20837,31 @@ spec: type: object type: array type: object + envoyGatewayConfig: + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the Gateway resource + type: object + brokerHostnameTemplate: + description: |- + BrokerHostnameTemplate is the template for generating broker hostnames (e.g., "kafka-%id.example.com") + The %id placeholder will be replaced with the broker ID + type: string + gatewayClassName: + description: GatewayClassName is the name of the GatewayClass + resource to use + type: string + gatewayName: + description: GatewayName is the name of the Gateway resource to + create + type: string + tlsSecretName: + description: TLSSecretName is the name of the secret containing + TLS certificates for TLS termination + type: string + type: object envs: description: |- Envs defines environment variables for Kafka broker Pods. @@ -21005,6 +21030,7 @@ spec: enum: - envoy - contour + - envoygateway type: string kRaft: default: false @@ -22668,6 +22694,33 @@ spec: type: object type: array type: object + envoyGatewayConfig: + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the Gateway + resource + type: object + brokerHostnameTemplate: + description: |- + BrokerHostnameTemplate is the template for generating broker hostnames (e.g., "kafka-%id.example.com") + The %id placeholder will be replaced with the broker ID + type: string + gatewayClassName: + description: GatewayClassName is the name + of the GatewayClass resource to use + type: string + gatewayName: + description: GatewayName is the name of the + Gateway resource to create + type: string + tlsSecretName: + description: TLSSecretName is the name of + the secret containing TLS certificates for + TLS termination + type: string + type: object externalTrafficPolicy: description: |- externalTrafficPolicy denotes if this Service desires to route external diff --git a/charts/kafka-operator/templates/operator-rbac.yaml b/charts/kafka-operator/templates/operator-rbac.yaml index 2fcadd3b4..67aaa9e67 100644 --- a/charts/kafka-operator/templates/operator-rbac.yaml +++ b/charts/kafka-operator/templates/operator-rbac.yaml @@ -139,6 +139,20 @@ rules: - patch - update - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways + - tcproutes + - tlsroutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - kafka.banzaicloud.io resources: diff --git a/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml b/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml index a7266064a..37772e2e6 100644 --- a/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml +++ b/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml @@ -20837,6 +20837,31 @@ spec: type: object type: array type: object + envoyGatewayConfig: + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the Gateway resource + type: object + brokerHostnameTemplate: + description: |- + BrokerHostnameTemplate is the template for generating broker hostnames (e.g., "kafka-%id.example.com") + The %id placeholder will be replaced with the broker ID + type: string + gatewayClassName: + description: GatewayClassName is the name of the GatewayClass + resource to use + type: string + gatewayName: + description: GatewayName is the name of the Gateway resource to + create + type: string + tlsSecretName: + description: TLSSecretName is the name of the secret containing + TLS certificates for TLS termination + type: string + type: object envs: description: |- Envs defines environment variables for Kafka broker Pods. @@ -21005,6 +21030,7 @@ spec: enum: - envoy - contour + - envoygateway type: string kRaft: default: false @@ -22668,6 +22694,33 @@ spec: type: object type: array type: object + envoyGatewayConfig: + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the Gateway + resource + type: object + brokerHostnameTemplate: + description: |- + BrokerHostnameTemplate is the template for generating broker hostnames (e.g., "kafka-%id.example.com") + The %id placeholder will be replaced with the broker ID + type: string + gatewayClassName: + description: GatewayClassName is the name + of the GatewayClass resource to use + type: string + gatewayName: + description: GatewayName is the name of the + Gateway resource to create + type: string + tlsSecretName: + description: TLSSecretName is the name of + the secret containing TLS certificates for + TLS termination + type: string + type: object externalTrafficPolicy: description: |- externalTrafficPolicy denotes if this Service desires to route external diff --git a/config/base/rbac/role.yaml b/config/base/rbac/role.yaml index 524ff09ac..82aa21c64 100644 --- a/config/base/rbac/role.yaml +++ b/config/base/rbac/role.yaml @@ -116,6 +116,20 @@ rules: - patch - update - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways + - tcproutes + - tlsroutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - kafka.banzaicloud.io resources: diff --git a/config/samples/banzaicloud_v1beta1_kafkacluster.yaml b/config/samples/banzaicloud_v1beta1_kafkacluster.yaml index 1550ad04c..2ad301f4d 100644 --- a/config/samples/banzaicloud_v1beta1_kafkacluster.yaml +++ b/config/samples/banzaicloud_v1beta1_kafkacluster.yaml @@ -15,7 +15,7 @@ spec: # - name: "remote-debug" # containerPort: 5005 # protocol: "TCP" - # Specify the usable ingress controller, only envoy and istioingress supported can be left blank + # Specify the usable ingress controller, only envoy and contour supported can be left blank ingressController: "envoy" # Specify the zookeeper addresses where the Kafka should store it's metadata # This configuration has no impact if the KafkaCluster is under KRaft mode @@ -306,7 +306,7 @@ spec: # defaultIngressConfig describes which ingress configuration to use # when non set on the brokerIngressMapping field inside BrokerConfig defaultIngressConfig: "az2" - # ingressConfig bundles the two available ingress configuration envoy and istio ingress + # ingressConfig bundles the available ingress configurations (envoy and contour) ingressConfig: # Ingress config name should be unique per external listener ingress-az1: @@ -369,42 +369,6 @@ spec: # cloud-provider does not support the feature." # More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/ # loadBalancerSourceRanges: - ingress-az1-istio: - istioIngressConfig: - # annotations can be used to place annotations on the istio ingress controller deployment - annotations: istio-az1 - # resourceRequirements works exactly like Container resources, the user can specify the limit and the requests - # through this property - # resourceRequirements: - # limits: - # memory: "300Mi" - # cpu: "200m" - # requests: - # memory: "300Mi" - # cpu: "200m" - # replicas describes how many pods will be used for the created envoy proxy - # replicas: 1 - - # nodeSelector can be specified, which set the pod to fit on a node - # nodeSelector: - - # tolerations can be specified, which set the pod's tolerations - # tolerations: - - # allows to set the created gateway configuration - # gatewayConfig: - - # annotations will be placed on the created virtual service - # virtualServiceAnnotations: - - # annotations defines the annotations placed on the envoy ingress controller deployment - # annotations: - - # If specified and supported by the platform, this will restrict traffic through the cloud-provider - # load-balancer will be restricted to the specified client IPs. This field will be ignored if the - # cloud-provider does not support the feature." - # More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/ - # loadBalancerSourceRanges: # internalListeners specifies settings required to access kafka externally internalListeners: # type defines the used security type ssl, plaintext, sasl_plaintext, sasl_ssl diff --git a/config/samples/kraft/simplekafkacluster_kraft.yaml b/config/samples/kraft/simplekafkacluster_kraft.yaml index 1e993bb49..d21475d47 100644 --- a/config/samples/kraft/simplekafkacluster_kraft.yaml +++ b/config/samples/kraft/simplekafkacluster_kraft.yaml @@ -4,6 +4,7 @@ metadata: labels: controller-tools.k8s.io: "1.0" name: kafka + namespace: kafka spec: kRaft: true monitoringConfig: diff --git a/config/samples/kraft/simplekafkacluster_kraft_with_envoy.yaml b/config/samples/kraft/simplekafkacluster_kraft_with_envoy.yaml new file mode 100644 index 000000000..ed8835b94 --- /dev/null +++ b/config/samples/kraft/simplekafkacluster_kraft_with_envoy.yaml @@ -0,0 +1,278 @@ +apiVersion: kafka.banzaicloud.io/v1beta1 +kind: KafkaCluster +metadata: + labels: + controller-tools.k8s.io: "1.0" + name: kafka + namespace: kafka +spec: + kRaft: true + monitoringConfig: + jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0" + headlessServiceEnabled: true + propagateLabels: false + oneBrokerPerNode: false + clusterImage: "ghcr.io/adobe/koperator/kafka:2.13-3.9.1" + ingressController: "envoy" + readOnlyConfig: | + auto.create.topics.enable=false + cruise.control.metrics.topic.auto.create=true + cruise.control.metrics.topic.num.partitions=1 + cruise.control.metrics.topic.replication.factor=2 + brokerConfigGroups: + default: + storageConfigs: + - mountPath: "/kafka-logs" + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + broker: + processRoles: + - broker + storageConfigs: + - mountPath: "/kafka-logs-broker" + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + brokerAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9020" + brokers: + - id: 0 + brokerConfigGroup: "broker" + - id: 1 + brokerConfigGroup: "broker" + - id: 2 + brokerConfigGroup: "broker" + - id: 3 + brokerConfigGroup: "default" + brokerConfig: + processRoles: + - controller + - id: 4 + brokerConfigGroup: "default" + brokerConfig: + processRoles: + - controller + - id: 5 + brokerConfigGroup: "default" + brokerConfig: + processRoles: + - controller + rollingUpgradeConfig: + failureThreshold: 1 + cruiseControlConfig: + cruiseControlTaskSpec: + RetryDurationMinutes: 5 + topicConfig: + partitions: 12 + replicationFactor: 3 + config: | + # Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. + # + # This is an example property file for Kafka Cruise Control. See KafkaCruiseControlConfig for more details. + # Configuration for the metadata client. + # ======================================= + # The maximum interval in milliseconds between two metadata refreshes. + #metadata.max.age.ms=300000 + # Client id for the Cruise Control. It is used for the metadata client. + #client.id=kafka-cruise-control + # The size of TCP send buffer bytes for the metadata client. + #send.buffer.bytes=131072 + # The size of TCP receive buffer size for the metadata client. + #receive.buffer.bytes=131072 + # The time to wait before disconnect an idle TCP connection. + #connections.max.idle.ms=540000 + # The time to wait before reconnect to a given host. + #reconnect.backoff.ms=50 + # The time to wait for a response from a host after sending a request. + #request.timeout.ms=30000 + # Configurations for the load monitor + # ======================================= + # The number of metric fetcher thread to fetch metrics for the Kafka cluster + num.metric.fetchers=1 + # The metric sampler class + metric.sampler.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.CruiseControlMetricsReporterSampler + # Configurations for CruiseControlMetricsReporterSampler + metric.reporter.topic.pattern=__CruiseControlMetrics + # The sample store class name + sample.store.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.KafkaSampleStore + # The config for the Kafka sample store to save the partition metric samples + partition.metric.sample.store.topic=__KafkaCruiseControlPartitionMetricSamples + # The config for the Kafka sample store to save the model training samples + broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples + # The replication factor of Kafka metric sample store topic + sample.store.topic.replication.factor=2 + # The config for the number of Kafka sample store consumer threads + num.sample.loading.threads=8 + # The partition assignor class for the metric samplers + metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor + # The metric sampling interval in milliseconds + metric.sampling.interval.ms=120000 + metric.anomaly.detection.interval.ms=180000 + # The partition metrics window size in milliseconds + partition.metrics.window.ms=300000 + # The number of partition metric windows to keep in memory + num.partition.metrics.windows=1 + # The minimum partition metric samples required for a partition in each window + min.samples.per.partition.metrics.window=1 + # The broker metrics window size in milliseconds + broker.metrics.window.ms=300000 + # The number of broker metric windows to keep in memory + num.broker.metrics.windows=20 + # The minimum broker metric samples required for a partition in each window + min.samples.per.broker.metrics.window=1 + # The configuration for the BrokerCapacityConfigFileResolver (supports JBOD and non-JBOD broker capacities) + capacity.config.file=config/capacity.json + #capacity.config.file=config/capacityJBOD.json + # Configurations for the analyzer + # ======================================= + # The list of goals to optimize the Kafka cluster for with pre-computed proposals + default.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal + # The list of supported goals + goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.kafkaassigner.KafkaAssignerDiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PreferredLeaderElectionGoal + # The list of supported hard goals + hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The minimum percentage of well monitored partitions out of all the partitions + min.monitored.partition.percentage=0.95 + # The balance threshold for CPU + cpu.balance.threshold=1.1 + # The balance threshold for disk + disk.balance.threshold=1.1 + # The balance threshold for network inbound utilization + network.inbound.balance.threshold=1.1 + # The balance threshold for network outbound utilization + network.outbound.balance.threshold=1.1 + # The balance threshold for the replica count + replica.count.balance.threshold=1.1 + # The capacity threshold for CPU in percentage + cpu.capacity.threshold=0.8 + # The capacity threshold for disk in percentage + disk.capacity.threshold=0.8 + # The capacity threshold for network inbound utilization in percentage + network.inbound.capacity.threshold=0.8 + # The capacity threshold for network outbound utilization in percentage + network.outbound.capacity.threshold=0.8 + # The threshold to define the cluster to be in a low CPU utilization state + cpu.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + disk.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low network inbound utilization state + network.inbound.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + network.outbound.low.utilization.threshold=0.0 + # The metric anomaly percentile upper threshold + metric.anomaly.percentile.upper.threshold=90.0 + # The metric anomaly percentile lower threshold + metric.anomaly.percentile.lower.threshold=10.0 + # How often should the cached proposal be expired and recalculated if necessary + proposal.expiration.ms=60000 + # The maximum number of replicas that can reside on a broker at any given time. + max.replicas.per.broker=10000 + # The number of threads to use for proposal candidate precomputing. + num.proposal.precompute.threads=1 + # the topics that should be excluded from the partition movement. + #topics.excluded.from.partition.movement + # Configurations for the executor + # ======================================= + # The max number of partitions to move in/out on a given broker at a given time. + num.concurrent.partition.movements.per.broker=10 + # The interval between two execution progress checks. + execution.progress.check.interval.ms=10000 + # Configurations for anomaly detector + # ======================================= + # The goal violation notifier class + anomaly.notifier.class=com.linkedin.kafka.cruisecontrol.detector.notifier.SelfHealingNotifier + # The metric anomaly finder class + metric.anomaly.finder.class=com.linkedin.kafka.cruisecontrol.detector.KafkaMetricAnomalyFinder + # The anomaly detection interval + anomaly.detection.interval.ms=10000 + # The goal violation to detect. + anomaly.detection.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The interested metrics for metric anomaly analyzer. + metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_MAX,BROKER_PRODUCE_LOCAL_TIME_MS_MEAN,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MAX,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MAX,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_LOG_FLUSH_TIME_MS_MAX,BROKER_LOG_FLUSH_TIME_MS_MEAN + ## Adjust accordingly if your metrics reporter is an older version and does not produce these metrics. + #metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_50TH,BROKER_PRODUCE_LOCAL_TIME_MS_999TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_50TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_999TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_50TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_999TH,BROKER_LOG_FLUSH_TIME_MS_50TH,BROKER_LOG_FLUSH_TIME_MS_999TH + # The cluster configurations for the KafkaTopicConfigProvider + cluster.configs.file=config/clusterConfigs.json + # The maximum time in milliseconds to store the response and access details of a completed user task. + completed.user.task.retention.time.ms=21600000 + # The maximum time in milliseconds to retain the demotion history of brokers. + demotion.history.retention.time.ms=86400000 + # The maximum number of completed user tasks for which the response and access details will be cached. + max.cached.completed.user.tasks=500 + # The maximum number of user tasks for concurrently running in async endpoints across all users. + max.active.user.tasks=25 + # Enable self healing for all anomaly detectors, unless the particular anomaly detector is explicitly disabled + self.healing.enabled=true + # Enable self healing for broker failure detector + #self.healing.broker.failure.enabled=true + # Enable self healing for goal violation detector + #self.healing.goal.violation.enabled=true + # Enable self healing for metric anomaly detector + #self.healing.metric.anomaly.enabled=true + # configurations for the webserver + # ================================ + # HTTP listen port + webserver.http.port=9090 + # HTTP listen address + webserver.http.address=0.0.0.0 + # Whether CORS support is enabled for API or not + webserver.http.cors.enabled=false + # Value for Access-Control-Allow-Origin + webserver.http.cors.origin=http://localhost:8080/ + # Value for Access-Control-Request-Method + webserver.http.cors.allowmethods=OPTIONS,GET,POST + # Headers that should be exposed to the Browser (Webapp) + # This is a special header that is used by the + # User Tasks subsystem and should be explicitly + # Enabled when CORS mode is used as part of the + # Admin Interface + webserver.http.cors.exposeheaders=User-Task-ID + # REST API default prefix + # (dont forget the ending *) + webserver.api.urlprefix=/kafkacruisecontrol/* + # Location where the Cruise Control frontend is deployed + webserver.ui.diskpath=./cruise-control-ui/dist/ + # URL path prefix for UI + # (dont forget the ending *) + webserver.ui.urlprefix=/* + # Time After which request is converted to Async + webserver.request.maxBlockTimeMs=10000 + # Default Session Expiry Period + webserver.session.maxExpiryTimeMs=60000 + # Session cookie path + webserver.session.path=/ + # Server Access Logs + webserver.accesslog.enabled=true + # Location of HTTP Request Logs + webserver.accesslog.path=access.log + # HTTP Request Log retention days + webserver.accesslog.retention.days=14 + clusterConfig: | + { + "min.insync.replicas": 3 + } + listenersConfig: + internalListeners: + - type: "plaintext" + name: "internal" + containerPort: 29092 + usedForInnerBrokerCommunication: true + - type: "plaintext" + name: "controller" + containerPort: 29093 + usedForInnerBrokerCommunication: false + usedForControllerCommunication: true + externalListeners: + - type: "plaintext" + name: "external" + externalStartingPort: 19090 + containerPort: 9094 + diff --git a/config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml b/config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml new file mode 100644 index 000000000..9484f4b78 --- /dev/null +++ b/config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml @@ -0,0 +1,311 @@ +--- +# Self-signed issuer for creating TLS certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: envoygateway-selfsigned-issuer + namespace: kafka +spec: + selfSigned: {} +--- +# TLS certificate for Envoy Gateway +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: envoygateway-tls-cert + namespace: kafka +spec: + secretName: envoygateway-tls-secret + issuerRef: + name: envoygateway-selfsigned-issuer + kind: Issuer + dnsNames: + - "*.kafka.cluster.local" + - "kafka.cluster.local" + commonName: "kafka.cluster.local" +--- +apiVersion: kafka.banzaicloud.io/v1beta1 +kind: KafkaCluster +metadata: + labels: + controller-tools.k8s.io: "1.0" + name: kafka + namespace: kafka +spec: + kRaft: true + monitoringConfig: + jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0" + headlessServiceEnabled: true + propagateLabels: false + oneBrokerPerNode: false + clusterImage: "ghcr.io/adobe/koperator/kafka:2.13-3.9.1" + ingressController: "envoygateway" + envoyGatewayConfig: + gatewayClassName: "eg" + tlsSecretName: "envoygateway-tls-secret" + brokerHostnameTemplate: "broker-%id.kafka.cluster.local" + listenersConfig: + internalListeners: + - type: "plaintext" + name: "internal" + containerPort: 29092 + usedForInnerBrokerCommunication: true + - type: "plaintext" + name: "controller" + containerPort: 29093 + usedForInnerBrokerCommunication: false + usedForControllerCommunication: true + externalListeners: + - accessMethod: LoadBalancer + containerPort: 29095 + externalStartingPort: -1 + name: envoyg + type: plaintext + usedForInnerBrokerCommunication: false + readOnlyConfig: | + auto.create.topics.enable=false + cruise.control.metrics.topic.auto.create=true + cruise.control.metrics.topic.num.partitions=1 + cruise.control.metrics.topic.replication.factor=2 + brokerConfigGroups: + default: + storageConfigs: + - mountPath: "/kafka-logs" + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + broker: + processRoles: + - broker + storageConfigs: + - mountPath: "/kafka-logs-broker" + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + brokerAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9020" + brokers: + - id: 0 + brokerConfigGroup: "broker" + - id: 1 + brokerConfigGroup: "broker" + - id: 2 + brokerConfigGroup: "broker" + - id: 3 + brokerConfigGroup: "default" + brokerConfig: + processRoles: + - controller + # - broker + - id: 4 + brokerConfigGroup: "default" + brokerConfig: + processRoles: + - controller + - id: 5 + brokerConfigGroup: "default" + brokerConfig: + processRoles: + - controller + rollingUpgradeConfig: + failureThreshold: 1 + cruiseControlConfig: + cruiseControlTaskSpec: + RetryDurationMinutes: 5 + topicConfig: + partitions: 12 + replicationFactor: 3 + config: | + # Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. + # + # This is an example property file for Kafka Cruise Control. See KafkaCruiseControlConfig for more details. + # Configuration for the metadata client. + # ======================================= + # The maximum interval in milliseconds between two metadata refreshes. + #metadata.max.age.ms=300000 + # Client id for the Cruise Control. It is used for the metadata client. + #client.id=kafka-cruise-control + # The size of TCP send buffer bytes for the metadata client. + #send.buffer.bytes=131072 + # The size of TCP receive buffer size for the metadata client. + #receive.buffer.bytes=131072 + # The time to wait before disconnect an idle TCP connection. + #connections.max.idle.ms=540000 + # The time to wait before reconnect to a given host. + #reconnect.backoff.ms=50 + # The time to wait for a response from a host after sending a request. + #request.timeout.ms=30000 + # Configurations for the load monitor + # ======================================= + # The number of metric fetcher thread to fetch metrics for the Kafka cluster + num.metric.fetchers=1 + # The metric sampler class + metric.sampler.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.CruiseControlMetricsReporterSampler + # Configurations for CruiseControlMetricsReporterSampler + metric.reporter.topic.pattern=__CruiseControlMetrics + # The sample store class name + sample.store.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.KafkaSampleStore + # The config for the Kafka sample store to save the partition metric samples + partition.metric.sample.store.topic=__KafkaCruiseControlPartitionMetricSamples + # The config for the Kafka sample store to save the model training samples + broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples + # The replication factor of Kafka metric sample store topic + sample.store.topic.replication.factor=2 + # The config for the number of Kafka sample store consumer threads + num.sample.loading.threads=8 + # The partition assignor class for the metric samplers + metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor + # The metric sampling interval in milliseconds + metric.sampling.interval.ms=120000 + metric.anomaly.detection.interval.ms=180000 + # The partition metrics window size in milliseconds + partition.metrics.window.ms=300000 + # The number of partition metric windows to keep in memory + num.partition.metrics.windows=1 + # The minimum partition metric samples required for a partition in each window + min.samples.per.partition.metrics.window=1 + # The broker metrics window size in milliseconds + broker.metrics.window.ms=300000 + # The number of broker metric windows to keep in memory + num.broker.metrics.windows=20 + # The minimum broker metric samples required for a partition in each window + min.samples.per.broker.metrics.window=1 + # The configuration for the BrokerCapacityConfigFileResolver (supports JBOD and non-JBOD broker capacities) + capacity.config.file=config/capacity.json + #capacity.config.file=config/capacityJBOD.json + # Configurations for the analyzer + # ======================================= + # The list of goals to optimize the Kafka cluster for with pre-computed proposals + default.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal + # The list of supported goals + goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.kafkaassigner.KafkaAssignerDiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PreferredLeaderElectionGoal + # The list of supported hard goals + hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The minimum percentage of well monitored partitions out of all the partitions + min.monitored.partition.percentage=0.95 + # The balance threshold for CPU + cpu.balance.threshold=1.1 + # The balance threshold for disk + disk.balance.threshold=1.1 + # The balance threshold for network inbound utilization + network.inbound.balance.threshold=1.1 + # The balance threshold for network outbound utilization + network.outbound.balance.threshold=1.1 + # The balance threshold for the replica count + replica.count.balance.threshold=1.1 + # The capacity threshold for CPU in percentage + cpu.capacity.threshold=0.8 + # The capacity threshold for disk in percentage + disk.capacity.threshold=0.8 + # The capacity threshold for network inbound utilization in percentage + network.inbound.capacity.threshold=0.8 + # The capacity threshold for network outbound utilization in percentage + network.outbound.capacity.threshold=0.8 + # The threshold to define the cluster to be in a low CPU utilization state + cpu.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + disk.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low network inbound utilization state + network.inbound.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + network.outbound.low.utilization.threshold=0.0 + # The metric anomaly percentile upper threshold + metric.anomaly.percentile.upper.threshold=90.0 + # The metric anomaly percentile lower threshold + metric.anomaly.percentile.lower.threshold=10.0 + # How often should the cached proposal be expired and recalculated if necessary + proposal.expiration.ms=60000 + # The maximum number of replicas that can reside on a broker at any given time. + max.replicas.per.broker=10000 + # The number of threads to use for proposal candidate precomputing. + num.proposal.precompute.threads=1 + # the topics that should be excluded from the partition movement. + #topics.excluded.from.partition.movement + # Configurations for the executor + # ======================================= + # The max number of partitions to move in/out on a given broker at a given time. + num.concurrent.partition.movements.per.broker=10 + # The interval between two execution progress checks. + execution.progress.check.interval.ms=10000 + # Configurations for anomaly detector + # ======================================= + # The goal violation notifier class + anomaly.notifier.class=com.linkedin.kafka.cruisecontrol.detector.notifier.SelfHealingNotifier + # The metric anomaly finder class + metric.anomaly.finder.class=com.linkedin.kafka.cruisecontrol.detector.KafkaMetricAnomalyFinder + # The anomaly detection interval + anomaly.detection.interval.ms=10000 + # The goal violation to detect. + anomaly.detection.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The interested metrics for metric anomaly analyzer. + metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_MAX,BROKER_PRODUCE_LOCAL_TIME_MS_MEAN,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MAX,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MAX,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_LOG_FLUSH_TIME_MS_MAX,BROKER_LOG_FLUSH_TIME_MS_MEAN + ## Adjust accordingly if your metrics reporter is an older version and does not produce these metrics. + #metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_50TH,BROKER_PRODUCE_LOCAL_TIME_MS_999TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_50TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_999TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_50TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_999TH,BROKER_LOG_FLUSH_TIME_MS_50TH,BROKER_LOG_FLUSH_TIME_MS_999TH + # The cluster configurations for the KafkaTopicConfigProvider + cluster.configs.file=config/clusterConfigs.json + # The maximum time in milliseconds to store the response and access details of a completed user task. + completed.user.task.retention.time.ms=21600000 + # The maximum time in milliseconds to retain the demotion history of brokers. + demotion.history.retention.time.ms=86400000 + # The maximum number of completed user tasks for which the response and access details will be cached. + max.cached.completed.user.tasks=500 + # The maximum number of user tasks for concurrently running in async endpoints across all users. + max.active.user.tasks=25 + # Enable self healing for all anomaly detectors, unless the particular anomaly detector is explicitly disabled + self.healing.enabled=true + # Enable self healing for broker failure detector + #self.healing.broker.failure.enabled=true + # Enable self healing for goal violation detector + #self.healing.goal.violation.enabled=true + # Enable self healing for metric anomaly detector + #self.healing.metric.anomaly.enabled=true + # configurations for the webserver + # ================================ + # HTTP listen port + webserver.http.port=9090 + # HTTP listen address + webserver.http.address=0.0.0.0 + # Whether CORS support is enabled for API or not + webserver.http.cors.enabled=false + # Value for Access-Control-Allow-Origin + webserver.http.cors.origin=http://localhost:8080/ + # Value for Access-Control-Request-Method + webserver.http.cors.allowmethods=OPTIONS,GET,POST + # Headers that should be exposed to the Browser (Webapp) + # This is a special header that is used by the + # User Tasks subsystem and should be explicitly + # Enabled when CORS mode is used as part of the + # Admin Interface + webserver.http.cors.exposeheaders=User-Task-ID + # REST API default prefix + # (dont forget the ending *) + webserver.api.urlprefix=/kafkacruisecontrol/* + # Location where the Cruise Control frontend is deployed + webserver.ui.diskpath=./cruise-control-ui/dist/ + # URL path prefix for UI + # (dont forget the ending *) + webserver.ui.urlprefix=/* + # Time After which request is converted to Async + webserver.request.maxBlockTimeMs=10000 + # Default Session Expiry Period + webserver.session.maxExpiryTimeMs=60000 + # Session cookie path + webserver.session.path=/ + # Server Access Logs + webserver.accesslog.enabled=true + # Location of HTTP Request Logs + webserver.accesslog.path=access.log + # HTTP Request Log retention days + webserver.accesslog.retention.days=14 + clusterConfig: | + { + "min.insync.replicas": 3 + } + diff --git a/config/samples/simplekafkacluster_with_envoy.yaml b/config/samples/simplekafkacluster_with_envoy.yaml new file mode 100644 index 000000000..464862e0d --- /dev/null +++ b/config/samples/simplekafkacluster_with_envoy.yaml @@ -0,0 +1,258 @@ +apiVersion: kafka.banzaicloud.io/v1beta1 +kind: KafkaCluster +metadata: + labels: + controller-tools.k8s.io: "1.0" + name: kafka + namespace: kafka +spec: + monitoringConfig: + jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0" + headlessServiceEnabled: true + zkAddresses: + - "zookeeper-server-client.zookeeper:2181" + propagateLabels: false + oneBrokerPerNode: false + clusterImage: "ghcr.io/adobe/koperator/kafka:2.13-3.9.1" + ingressController: "envoy" + listenersConfig: + internalListeners: + - type: "plaintext" + name: "internal" + containerPort: 29092 + usedForInnerBrokerCommunication: true + - type: "plaintext" + name: "controller" + containerPort: 29093 + usedForInnerBrokerCommunication: false + usedForControllerCommunication: true + externalListeners: + - type: "plaintext" + name: "external" + externalStartingPort: 19090 + containerPort: 9094 + readOnlyConfig: | + auto.create.topics.enable=false + cruise.control.metrics.topic.auto.create=true + cruise.control.metrics.topic.num.partitions=1 + cruise.control.metrics.topic.replication.factor=2 + brokerConfigGroups: + default: + storageConfigs: + - mountPath: "/kafka-logs" + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + brokerAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9020" + brokers: + - id: 0 + brokerConfigGroup: "default" + - id: 1 + brokerConfigGroup: "default" + - id: 2 + brokerConfigGroup: "default" + rollingUpgradeConfig: + failureThreshold: 1 + cruiseControlConfig: + cruiseControlTaskSpec: + RetryDurationMinutes: 5 + topicConfig: + partitions: 12 + replicationFactor: 3 + config: | + # Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. + # + # This is an example property file for Kafka Cruise Control. See KafkaCruiseControlConfig for more details. + # Configuration for the metadata client. + # ======================================= + # The maximum interval in milliseconds between two metadata refreshes. + #metadata.max.age.ms=300000 + # Client id for the Cruise Control. It is used for the metadata client. + #client.id=kafka-cruise-control + # The size of TCP send buffer bytes for the metadata client. + #send.buffer.bytes=131072 + # The size of TCP receive buffer size for the metadata client. + #receive.buffer.bytes=131072 + # The time to wait before disconnect an idle TCP connection. + #connections.max.idle.ms=540000 + # The time to wait before reconnect to a given host. + #reconnect.backoff.ms=50 + # The time to wait for a response from a host after sending a request. + #request.timeout.ms=30000 + # Configurations for the load monitor + # ======================================= + # The number of metric fetcher thread to fetch metrics for the Kafka cluster + num.metric.fetchers=1 + # The metric sampler class + metric.sampler.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.CruiseControlMetricsReporterSampler + # Configurations for CruiseControlMetricsReporterSampler + metric.reporter.topic.pattern=__CruiseControlMetrics + # The sample store class name + sample.store.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.KafkaSampleStore + # The config for the Kafka sample store to save the partition metric samples + partition.metric.sample.store.topic=__KafkaCruiseControlPartitionMetricSamples + # The config for the Kafka sample store to save the model training samples + broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples + # The replication factor of Kafka metric sample store topic + sample.store.topic.replication.factor=2 + # The config for the number of Kafka sample store consumer threads + num.sample.loading.threads=8 + # The partition assignor class for the metric samplers + metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor + # The metric sampling interval in milliseconds + metric.sampling.interval.ms=120000 + metric.anomaly.detection.interval.ms=180000 + # The partition metrics window size in milliseconds + partition.metrics.window.ms=300000 + # The number of partition metric windows to keep in memory + num.partition.metrics.windows=1 + # The minimum partition metric samples required for a partition in each window + min.samples.per.partition.metrics.window=1 + # The broker metrics window size in milliseconds + broker.metrics.window.ms=300000 + # The number of broker metric windows to keep in memory + num.broker.metrics.windows=20 + # The minimum broker metric samples required for a partition in each window + min.samples.per.broker.metrics.window=1 + # The configuration for the BrokerCapacityConfigFileResolver (supports JBOD and non-JBOD broker capacities) + capacity.config.file=config/capacity.json + #capacity.config.file=config/capacityJBOD.json + # Configurations for the analyzer + # ======================================= + # The list of goals to optimize the Kafka cluster for with pre-computed proposals + default.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal + # The list of supported goals + goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.kafkaassigner.KafkaAssignerDiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PreferredLeaderElectionGoal + # The list of supported hard goals + hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The minimum percentage of well monitored partitions out of all the partitions + min.monitored.partition.percentage=0.95 + # The balance threshold for CPU + cpu.balance.threshold=1.1 + # The balance threshold for disk + disk.balance.threshold=1.1 + # The balance threshold for network inbound utilization + network.inbound.balance.threshold=1.1 + # The balance threshold for network outbound utilization + network.outbound.balance.threshold=1.1 + # The balance threshold for the replica count + replica.count.balance.threshold=1.1 + # The capacity threshold for CPU in percentage + cpu.capacity.threshold=0.8 + # The capacity threshold for disk in percentage + disk.capacity.threshold=0.8 + # The capacity threshold for network inbound utilization in percentage + network.inbound.capacity.threshold=0.8 + # The capacity threshold for network outbound utilization in percentage + network.outbound.capacity.threshold=0.8 + # The threshold to define the cluster to be in a low CPU utilization state + cpu.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + disk.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low network inbound utilization state + network.inbound.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + network.outbound.low.utilization.threshold=0.0 + # The metric anomaly percentile upper threshold + metric.anomaly.percentile.upper.threshold=90.0 + # The metric anomaly percentile lower threshold + metric.anomaly.percentile.lower.threshold=10.0 + # How often should the cached proposal be expired and recalculated if necessary + proposal.expiration.ms=60000 + # The maximum number of replicas that can reside on a broker at any given time. + max.replicas.per.broker=10000 + # The number of threads to use for proposal candidate precomputing. + num.proposal.precompute.threads=1 + # the topics that should be excluded from the partition movement. + #topics.excluded.from.partition.movement + # Configurations for the executor + # ======================================= + # The max number of partitions to move in/out on a given broker at a given time. + num.concurrent.partition.movements.per.broker=10 + # The interval between two execution progress checks. + execution.progress.check.interval.ms=10000 + # Configurations for anomaly detector + # ======================================= + # The goal violation notifier class + anomaly.notifier.class=com.linkedin.kafka.cruisecontrol.detector.notifier.SelfHealingNotifier + # The metric anomaly finder class + metric.anomaly.finder.class=com.linkedin.kafka.cruisecontrol.detector.KafkaMetricAnomalyFinder + # The anomaly detection interval + anomaly.detection.interval.ms=10000 + # The goal violation to detect. + anomaly.detection.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The interested metrics for metric anomaly analyzer. + metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_MAX,BROKER_PRODUCE_LOCAL_TIME_MS_MEAN,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MAX,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MAX,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_LOG_FLUSH_TIME_MS_MAX,BROKER_LOG_FLUSH_TIME_MS_MEAN + ## Adjust accordingly if your metrics reporter is an older version and does not produce these metrics. + #metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_50TH,BROKER_PRODUCE_LOCAL_TIME_MS_999TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_50TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_999TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_50TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_999TH,BROKER_LOG_FLUSH_TIME_MS_50TH,BROKER_LOG_FLUSH_TIME_MS_999TH + # The zk path to store failed broker information. + failed.brokers.zk.path=/CruiseControlBrokerList + # Topic config provider class + topic.config.provider.class=com.linkedin.kafka.cruisecontrol.config.KafkaTopicConfigProvider + # The cluster configurations for the KafkaTopicConfigProvider + cluster.configs.file=config/clusterConfigs.json + # The maximum time in milliseconds to store the response and access details of a completed user task. + completed.user.task.retention.time.ms=21600000 + # The maximum time in milliseconds to retain the demotion history of brokers. + demotion.history.retention.time.ms=86400000 + # The maximum number of completed user tasks for which the response and access details will be cached. + max.cached.completed.user.tasks=500 + # The maximum number of user tasks for concurrently running in async endpoints across all users. + max.active.user.tasks=25 + # Enable self healing for all anomaly detectors, unless the particular anomaly detector is explicitly disabled + self.healing.enabled=true + # Enable self healing for broker failure detector + #self.healing.broker.failure.enabled=true + # Enable self healing for goal violation detector + #self.healing.goal.violation.enabled=true + # Enable self healing for metric anomaly detector + #self.healing.metric.anomaly.enabled=true + # configurations for the webserver + # ================================ + # HTTP listen port + webserver.http.port=9090 + # HTTP listen address + webserver.http.address=0.0.0.0 + # Whether CORS support is enabled for API or not + webserver.http.cors.enabled=false + # Value for Access-Control-Allow-Origin + webserver.http.cors.origin=http://localhost:8080/ + # Value for Access-Control-Request-Method + webserver.http.cors.allowmethods=OPTIONS,GET,POST + # Headers that should be exposed to the Browser (Webapp) + # This is a special header that is used by the + # User Tasks subsystem and should be explicitly + # Enabled when CORS mode is used as part of the + # Admin Interface + webserver.http.cors.exposeheaders=User-Task-ID + # REST API default prefix + # (dont forget the ending *) + webserver.api.urlprefix=/kafkacruisecontrol/* + # Location where the Cruise Control frontend is deployed + webserver.ui.diskpath=./cruise-control-ui/dist/ + # URL path prefix for UI + # (dont forget the ending *) + webserver.ui.urlprefix=/* + # Time After which request is converted to Async + webserver.request.maxBlockTimeMs=10000 + # Default Session Expiry Period + webserver.session.maxExpiryTimeMs=60000 + # Session cookie path + webserver.session.path=/ + # Server Access Logs + webserver.accesslog.enabled=true + # Location of HTTP Request Logs + webserver.accesslog.path=access.log + # HTTP Request Log retention days + webserver.accesslog.retention.days=14 + clusterConfig: | + { + "min.insync.replicas": 3 + } + + diff --git a/config/samples/simplekafkacluster_with_envoygateway.yaml b/config/samples/simplekafkacluster_with_envoygateway.yaml new file mode 100644 index 000000000..52f660909 --- /dev/null +++ b/config/samples/simplekafkacluster_with_envoygateway.yaml @@ -0,0 +1,288 @@ +--- +# Self-signed issuer for creating TLS certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: envoygateway-selfsigned-issuer + namespace: kafka +spec: + selfSigned: {} +--- +# TLS certificate for Envoy Gateway +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: envoygateway-tls-cert + namespace: kafka +spec: + secretName: envoygateway-tls-secret + issuerRef: + name: envoygateway-selfsigned-issuer + kind: Issuer + dnsNames: + - "*.kafka.cluster.local" + - "kafka.cluster.local" + commonName: "kafka.cluster.local" +--- +apiVersion: kafka.banzaicloud.io/v1beta1 +kind: KafkaCluster +metadata: + labels: + controller-tools.k8s.io: "1.0" + name: kafka + namespace: kafka +spec: + monitoringConfig: + jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0" + headlessServiceEnabled: true + zkAddresses: + - "zookeeper-server-client.zookeeper:2181" + propagateLabels: false + oneBrokerPerNode: false + clusterImage: "ghcr.io/adobe/koperator/kafka:2.13-3.9.1" + ingressController: "envoygateway" + envoyGatewayConfig: + gatewayClassName: "eg" + tlsSecretName: "envoygateway-tls-secret" + brokerHostnameTemplate: "broker-%id.kafka.cluster.local" + listenersConfig: + internalListeners: + - type: "plaintext" + name: "internal" + containerPort: 29092 + usedForInnerBrokerCommunication: true + - type: "plaintext" + name: "controller" + containerPort: 29093 + usedForInnerBrokerCommunication: false + usedForControllerCommunication: true + externalListeners: + - accessMethod: LoadBalancer + containerPort: 29095 + externalStartingPort: -1 + name: envoyg + type: plaintext + usedForInnerBrokerCommunication: false + readOnlyConfig: | + auto.create.topics.enable=false + cruise.control.metrics.topic.auto.create=true + cruise.control.metrics.topic.num.partitions=1 + cruise.control.metrics.topic.replication.factor=2 + brokerConfigGroups: + default: + storageConfigs: + - mountPath: "/kafka-logs" + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + brokerAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9020" + brokers: + - id: 0 + brokerConfigGroup: "default" + - id: 1 + brokerConfigGroup: "default" + - id: 2 + brokerConfigGroup: "default" + rollingUpgradeConfig: + failureThreshold: 1 + cruiseControlConfig: + cruiseControlTaskSpec: + RetryDurationMinutes: 5 + topicConfig: + partitions: 12 + replicationFactor: 3 + config: | + # Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. + # + # This is an example property file for Kafka Cruise Control. See KafkaCruiseControlConfig for more details. + # Configuration for the metadata client. + # ======================================= + # The maximum interval in milliseconds between two metadata refreshes. + #metadata.max.age.ms=300000 + # Client id for the Cruise Control. It is used for the metadata client. + #client.id=kafka-cruise-control + # The size of TCP send buffer bytes for the metadata client. + #send.buffer.bytes=131072 + # The size of TCP receive buffer size for the metadata client. + #receive.buffer.bytes=131072 + # The time to wait before disconnect an idle TCP connection. + #connections.max.idle.ms=540000 + # The time to wait before reconnect to a given host. + #reconnect.backoff.ms=50 + # The time to wait for a response from a host after sending a request. + #request.timeout.ms=30000 + # Configurations for the load monitor + # ======================================= + # The number of metric fetcher thread to fetch metrics for the Kafka cluster + num.metric.fetchers=1 + # The metric sampler class + metric.sampler.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.CruiseControlMetricsReporterSampler + # Configurations for CruiseControlMetricsReporterSampler + metric.reporter.topic.pattern=__CruiseControlMetrics + # The sample store class name + sample.store.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.KafkaSampleStore + # The config for the Kafka sample store to save the partition metric samples + partition.metric.sample.store.topic=__KafkaCruiseControlPartitionMetricSamples + # The config for the Kafka sample store to save the model training samples + broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples + # The replication factor of Kafka metric sample store topic + sample.store.topic.replication.factor=2 + # The config for the number of Kafka sample store consumer threads + num.sample.loading.threads=8 + # The partition assignor class for the metric samplers + metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor + # The metric sampling interval in milliseconds + metric.sampling.interval.ms=120000 + metric.anomaly.detection.interval.ms=180000 + # The partition metrics window size in milliseconds + partition.metrics.window.ms=300000 + # The number of partition metric windows to keep in memory + num.partition.metrics.windows=1 + # The minimum partition metric samples required for a partition in each window + min.samples.per.partition.metrics.window=1 + # The broker metrics window size in milliseconds + broker.metrics.window.ms=300000 + # The number of broker metric windows to keep in memory + num.broker.metrics.windows=20 + # The minimum broker metric samples required for a partition in each window + min.samples.per.broker.metrics.window=1 + # The configuration for the BrokerCapacityConfigFileResolver (supports JBOD and non-JBOD broker capacities) + capacity.config.file=config/capacity.json + #capacity.config.file=config/capacityJBOD.json + # Configurations for the analyzer + # ======================================= + # The list of goals to optimize the Kafka cluster for with pre-computed proposals + default.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal + # The list of supported goals + goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.kafkaassigner.KafkaAssignerDiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PreferredLeaderElectionGoal + # The list of supported hard goals + hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The minimum percentage of well monitored partitions out of all the partitions + min.monitored.partition.percentage=0.95 + # The balance threshold for CPU + cpu.balance.threshold=1.1 + # The balance threshold for disk + disk.balance.threshold=1.1 + # The balance threshold for network inbound utilization + network.inbound.balance.threshold=1.1 + # The balance threshold for network outbound utilization + network.outbound.balance.threshold=1.1 + # The balance threshold for the replica count + replica.count.balance.threshold=1.1 + # The capacity threshold for CPU in percentage + cpu.capacity.threshold=0.8 + # The capacity threshold for disk in percentage + disk.capacity.threshold=0.8 + # The capacity threshold for network inbound utilization in percentage + network.inbound.capacity.threshold=0.8 + # The capacity threshold for network outbound utilization in percentage + network.outbound.capacity.threshold=0.8 + # The threshold to define the cluster to be in a low CPU utilization state + cpu.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + disk.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low network inbound utilization state + network.inbound.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + network.outbound.low.utilization.threshold=0.0 + # The metric anomaly percentile upper threshold + metric.anomaly.percentile.upper.threshold=90.0 + # The metric anomaly percentile lower threshold + metric.anomaly.percentile.lower.threshold=10.0 + # How often should the cached proposal be expired and recalculated if necessary + proposal.expiration.ms=60000 + # The maximum number of replicas that can reside on a broker at any given time. + max.replicas.per.broker=10000 + # The number of threads to use for proposal candidate precomputing. + num.proposal.precompute.threads=1 + # the topics that should be excluded from the partition movement. + #topics.excluded.from.partition.movement + # Configurations for the executor + # ======================================= + # The max number of partitions to move in/out on a given broker at a given time. + num.concurrent.partition.movements.per.broker=10 + # The interval between two execution progress checks. + execution.progress.check.interval.ms=10000 + # Configurations for anomaly detector + # ======================================= + # The goal violation notifier class + anomaly.notifier.class=com.linkedin.kafka.cruisecontrol.detector.notifier.SelfHealingNotifier + # The metric anomaly finder class + metric.anomaly.finder.class=com.linkedin.kafka.cruisecontrol.detector.KafkaMetricAnomalyFinder + # The anomaly detection interval + anomaly.detection.interval.ms=10000 + # The goal violation to detect. + anomaly.detection.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The interested metrics for metric anomaly analyzer. + metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_MAX,BROKER_PRODUCE_LOCAL_TIME_MS_MEAN,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MAX,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MAX,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_LOG_FLUSH_TIME_MS_MAX,BROKER_LOG_FLUSH_TIME_MS_MEAN + ## Adjust accordingly if your metrics reporter is an older version and does not produce these metrics. + #metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_50TH,BROKER_PRODUCE_LOCAL_TIME_MS_999TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_50TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_999TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_50TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_999TH,BROKER_LOG_FLUSH_TIME_MS_50TH,BROKER_LOG_FLUSH_TIME_MS_999TH + # The zk path to store failed broker information. + failed.brokers.zk.path=/CruiseControlBrokerList + # Topic config provider class + topic.config.provider.class=com.linkedin.kafka.cruisecontrol.config.KafkaTopicConfigProvider + # The cluster configurations for the KafkaTopicConfigProvider + cluster.configs.file=config/clusterConfigs.json + # The maximum time in milliseconds to store the response and access details of a completed user task. + completed.user.task.retention.time.ms=21600000 + # The maximum time in milliseconds to retain the demotion history of brokers. + demotion.history.retention.time.ms=86400000 + # The maximum number of completed user tasks for which the response and access details will be cached. + max.cached.completed.user.tasks=500 + # The maximum number of user tasks for concurrently running in async endpoints across all users. + max.active.user.tasks=25 + # Enable self healing for all anomaly detectors, unless the particular anomaly detector is explicitly disabled + self.healing.enabled=true + # Enable self healing for broker failure detector + #self.healing.broker.failure.enabled=true + # Enable self healing for goal violation detector + #self.healing.goal.violation.enabled=true + # Enable self healing for metric anomaly detector + #self.healing.metric.anomaly.enabled=true + # configurations for the webserver + # ================================ + # HTTP listen port + webserver.http.port=9090 + # HTTP listen address + webserver.http.address=0.0.0.0 + # Whether CORS support is enabled for API or not + webserver.http.cors.enabled=false + # Value for Access-Control-Allow-Origin + webserver.http.cors.origin=http://localhost:8080/ + # Value for Access-Control-Request-Method + webserver.http.cors.allowmethods=OPTIONS,GET,POST + # Headers that should be exposed to the Browser (Webapp) + # This is a special header that is used by the + # User Tasks subsystem and should be explicitly + # Enabled when CORS mode is used as part of the + # Admin Interface + webserver.http.cors.exposeheaders=User-Task-ID + # REST API default prefix + # (dont forget the ending *) + webserver.api.urlprefix=/kafkacruisecontrol/* + # Location where the Cruise Control frontend is deployed + webserver.ui.diskpath=./cruise-control-ui/dist/ + # URL path prefix for UI + # (dont forget the ending *) + webserver.ui.urlprefix=/* + # Time After which request is converted to Async + webserver.request.maxBlockTimeMs=10000 + # Default Session Expiry Period + webserver.session.maxExpiryTimeMs=60000 + # Session cookie path + webserver.session.path=/ + # Server Access Logs + webserver.accesslog.enabled=true + # Location of HTTP Request Logs + webserver.accesslog.path=access.log + # HTTP Request Log retention days + webserver.accesslog.retention.days=14 + clusterConfig: | + { + "min.insync.replicas": 3 + } diff --git a/config/test/crd/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml b/config/test/crd/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml new file mode 100644 index 000000000..15412b869 --- /dev/null +++ b/config/test/crd/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml @@ -0,0 +1,515 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 + gateway.networking.k8s.io/bundle-version: v1.5.0 + gateway.networking.k8s.io/channel: standard + name: gatewayclasses.gateway.networking.k8s.io +spec: + group: gateway.networking.k8s.io + names: + categories: + - gateway-api + kind: GatewayClass + listKind: GatewayClassList + plural: gatewayclasses + shortNames: + - gc + singular: gatewayclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.controllerName + name: Controller + type: string + - jsonPath: .status.conditions[?(@.type=="Accepted")].status + name: Accepted + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.description + name: Description + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: |- + GatewayClass describes a class of Gateways available to the user for creating + Gateway resources. + + It is recommended that this resource be used as a template for Gateways. This + means that a Gateway is based on the state of the GatewayClass at the time it + was created and changes to the GatewayClass or associated parameters are not + propagated down to existing Gateways. This recommendation is intended to + limit the blast radius of changes to GatewayClass or associated parameters. + If implementations choose to propagate GatewayClass changes to existing + Gateways, that MUST be clearly documented by the implementation. + + Whenever one or more Gateways are using a GatewayClass, implementations SHOULD + add the `gateway-exists-finalizer.gateway.networking.k8s.io` finalizer on the + associated GatewayClass. This ensures that a GatewayClass associated with a + Gateway is not deleted while in use. + + GatewayClass is a Cluster level resource. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of GatewayClass. + properties: + controllerName: + description: |- + ControllerName is the name of the controller that is managing Gateways of + this class. The value of this field MUST be a domain prefixed path. + + Example: "example.net/gateway-controller". + + This field is not mutable and cannot be empty. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + description: + description: Description helps describe a GatewayClass with more details. + maxLength: 64 + type: string + parametersRef: + description: |- + ParametersRef is a reference to a resource that contains the configuration + parameters corresponding to the GatewayClass. This is optional if the + controller does not require any additional configuration. + + ParametersRef can reference a standard Kubernetes resource, i.e. ConfigMap, + or an implementation-specific custom resource. The resource can be + cluster-scoped or namespace-scoped. + + If the referent cannot be found, refers to an unsupported kind, or when + the data within that resource is malformed, the GatewayClass SHOULD be + rejected with the "Accepted" status condition set to "False" and an + "InvalidParameters" reason. + + A Gateway for this GatewayClass may provide its own `parametersRef`. When both are specified, + the merging behavior is implementation specific. + It is generally recommended that GatewayClass provides defaults that can be overridden by a Gateway. + + Support: Implementation-specific + properties: + group: + description: Group is the group of the referent. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. + This field is required when referring to a Namespace-scoped resource and + MUST be unset when referring to a Cluster-scoped resource. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + required: + - controllerName + type: object + status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Accepted + description: |- + Status defines the current state of GatewayClass. + + Implementations MUST populate status on all GatewayClass resources which + specify their controller name. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Accepted + description: |- + Conditions is the current status from the controller for + this GatewayClass. + + Controllers should prefer to publish conditions using values + of GatewayClassConditionType for the type of each Condition. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + supportedFeatures: + description: |- + SupportedFeatures is the set of features the GatewayClass support. + It MUST be sorted in ascending alphabetical order by the Name key. + items: + properties: + name: + description: |- + FeatureName is used to describe distinct features that are covered by + conformance tests. + type: string + required: + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.controllerName + name: Controller + type: string + - jsonPath: .status.conditions[?(@.type=="Accepted")].status + name: Accepted + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.description + name: Description + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + GatewayClass describes a class of Gateways available to the user for creating + Gateway resources. + + It is recommended that this resource be used as a template for Gateways. This + means that a Gateway is based on the state of the GatewayClass at the time it + was created and changes to the GatewayClass or associated parameters are not + propagated down to existing Gateways. This recommendation is intended to + limit the blast radius of changes to GatewayClass or associated parameters. + If implementations choose to propagate GatewayClass changes to existing + Gateways, that MUST be clearly documented by the implementation. + + Whenever one or more Gateways are using a GatewayClass, implementations SHOULD + add the `gateway-exists-finalizer.gateway.networking.k8s.io` finalizer on the + associated GatewayClass. This ensures that a GatewayClass associated with a + Gateway is not deleted while in use. + + GatewayClass is a Cluster level resource. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of GatewayClass. + properties: + controllerName: + description: |- + ControllerName is the name of the controller that is managing Gateways of + this class. The value of this field MUST be a domain prefixed path. + + Example: "example.net/gateway-controller". + + This field is not mutable and cannot be empty. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + description: + description: Description helps describe a GatewayClass with more details. + maxLength: 64 + type: string + parametersRef: + description: |- + ParametersRef is a reference to a resource that contains the configuration + parameters corresponding to the GatewayClass. This is optional if the + controller does not require any additional configuration. + + ParametersRef can reference a standard Kubernetes resource, i.e. ConfigMap, + or an implementation-specific custom resource. The resource can be + cluster-scoped or namespace-scoped. + + If the referent cannot be found, refers to an unsupported kind, or when + the data within that resource is malformed, the GatewayClass SHOULD be + rejected with the "Accepted" status condition set to "False" and an + "InvalidParameters" reason. + + A Gateway for this GatewayClass may provide its own `parametersRef`. When both are specified, + the merging behavior is implementation specific. + It is generally recommended that GatewayClass provides defaults that can be overridden by a Gateway. + + Support: Implementation-specific + properties: + group: + description: Group is the group of the referent. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. + This field is required when referring to a Namespace-scoped resource and + MUST be unset when referring to a Cluster-scoped resource. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + required: + - controllerName + type: object + status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Accepted + description: |- + Status defines the current state of GatewayClass. + + Implementations MUST populate status on all GatewayClass resources which + specify their controller name. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Accepted + description: |- + Conditions is the current status from the controller for + this GatewayClass. + + Controllers should prefer to publish conditions using values + of GatewayClassConditionType for the type of each Condition. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + supportedFeatures: + description: |- + SupportedFeatures is the set of features the GatewayClass support. + It MUST be sorted in ascending alphabetical order by the Name key. + items: + properties: + name: + description: |- + FeatureName is used to describe distinct features that are covered by + conformance tests. + type: string + required: + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/config/test/crd/gateway-api/gateway.networking.k8s.io_gateways.yaml b/config/test/crd/gateway-api/gateway.networking.k8s.io_gateways.yaml new file mode 100644 index 000000000..169e74fcb --- /dev/null +++ b/config/test/crd/gateway-api/gateway.networking.k8s.io_gateways.yaml @@ -0,0 +1,3283 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 + gateway.networking.k8s.io/bundle-version: v1.5.0 + gateway.networking.k8s.io/channel: standard + name: gateways.gateway.networking.k8s.io +spec: + group: gateway.networking.k8s.io + names: + categories: + - gateway-api + kind: Gateway + listKind: GatewayList + plural: gateways + shortNames: + - gtw + singular: gateway + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.gatewayClassName + name: Class + type: string + - jsonPath: .status.addresses[*].value + name: Address + type: string + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + Gateway represents an instance of a service-traffic handling infrastructure + by binding Listeners to a set of IP addresses. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of Gateway. + properties: + addresses: + description: |- + Addresses requested for this Gateway. This is optional and behavior can + depend on the implementation. If a value is set in the spec and the + requested address is invalid or unavailable, the implementation MUST + indicate this in an associated entry in GatewayStatus.Conditions. + + The Addresses field represents a request for the address(es) on the + "outside of the Gateway", that traffic bound for this Gateway will use. + This could be the IP address or hostname of an external load balancer or + other networking infrastructure, or some other address that traffic will + be sent to. + + If no Addresses are specified, the implementation MAY schedule the + Gateway in an implementation-specific manner, assigning an appropriate + set of Addresses. + + The implementation MUST bind all Listeners to every GatewayAddress that + it assigns to the Gateway and add a corresponding entry in + GatewayStatus.Addresses. + + Support: Extended + items: + description: GatewaySpecAddress describes an address that can be + bound to a Gateway. + oneOf: + - properties: + type: + enum: + - IPAddress + value: + anyOf: + - format: ipv4 + - format: ipv6 + - properties: + type: + not: + enum: + - IPAddress + properties: + type: + default: IPAddress + description: Type of the address. + maxLength: 253 + minLength: 1 + pattern: ^Hostname|IPAddress|NamedAddress|[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + value: + description: |- + When a value is unspecified, an implementation SHOULD automatically + assign an address matching the requested type if possible. + + If an implementation does not support an empty value, they MUST set the + "Programmed" condition in status to False with a reason of "AddressNotAssigned". + + Examples: `1.2.3.4`, `128::1`, `my-ip-address`. + maxLength: 253 + type: string + type: object + x-kubernetes-validations: + - message: Hostname value must be empty or contain only valid characters + (matching ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$) + rule: 'self.type == ''Hostname'' ? (!has(self.value) || self.value.matches(r"""^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$""")): + true' + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: IPAddress values must be unique + rule: 'self.all(a1, a1.type == ''IPAddress'' && has(a1.value) ? + self.exists_one(a2, a2.type == a1.type && has(a2.value) && a2.value + == a1.value) : true )' + - message: Hostname values must be unique + rule: 'self.all(a1, a1.type == ''Hostname'' && has(a1.value) ? + self.exists_one(a2, a2.type == a1.type && has(a2.value) && a2.value + == a1.value) : true )' + allowedListeners: + description: |- + AllowedListeners defines which ListenerSets can be attached to this Gateway. + The default value is to allow no ListenerSets. + properties: + namespaces: + default: + from: None + description: |- + Namespaces defines which namespaces ListenerSets can be attached to this Gateway. + The default value is to allow no ListenerSets. + properties: + from: + default: None + description: |- + From indicates where ListenerSets can attach to this Gateway. Possible + values are: + + * Same: Only ListenerSets in the same namespace may be attached to this Gateway. + * Selector: ListenerSets in namespaces selected by the selector may be attached to this Gateway. + * All: ListenerSets in all namespaces may be attached to this Gateway. + * None: Only listeners defined in the Gateway's spec are allowed + + The default value None + enum: + - All + - Selector + - Same + - None + type: string + selector: + description: |- + Selector must be specified when From is set to "Selector". In that case, + only ListenerSets in Namespaces matching this Selector will be selected by this + Gateway. This field is ignored for other values of "From". + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: object + gatewayClassName: + description: |- + GatewayClassName used for this Gateway. This is the name of a + GatewayClass resource. + maxLength: 253 + minLength: 1 + type: string + infrastructure: + description: |- + Infrastructure defines infrastructure level attributes about this Gateway instance. + + Support: Extended + properties: + annotations: + additionalProperties: + description: |- + AnnotationValue is the value of an annotation in Gateway API. This is used + for validation of maps such as TLS options. This roughly matches Kubernetes + annotation validation, although the length validation in that case is based + on the entire size of the annotations struct. + maxLength: 4096 + minLength: 0 + type: string + description: |- + Annotations that SHOULD be applied to any resources created in response to this Gateway. + + For implementations creating other Kubernetes objects, this should be the `metadata.annotations` field on resources. + For other implementations, this refers to any relevant (implementation specific) "annotations" concepts. + + An implementation may chose to add additional implementation-specific annotations as they see fit. + + Support: Extended + maxProperties: 8 + type: object + x-kubernetes-validations: + - message: Annotation keys must be in the form of an optional + DNS subdomain prefix followed by a required name segment of + up to 63 characters. + rule: self.all(key, key.matches(r"""^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?([A-Za-z0-9][-A-Za-z0-9_.]{0,61})?[A-Za-z0-9]$""")) + - message: If specified, the annotation key's prefix must be a + DNS subdomain not longer than 253 characters in total. + rule: self.all(key, key.split("/")[0].size() < 253) + labels: + additionalProperties: + description: |- + LabelValue is the value of a label in the Gateway API. This is used for validation + of maps such as Gateway infrastructure labels. This matches the Kubernetes + label validation rules: + * must be 63 characters or less (can be empty), + * unless empty, must begin and end with an alphanumeric character ([a-z0-9A-Z]), + * could contain dashes (-), underscores (_), dots (.), and alphanumerics between. + + Valid values include: + + * MyValue + * my.name + * 123-my-value + maxLength: 63 + minLength: 0 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + description: |- + Labels that SHOULD be applied to any resources created in response to this Gateway. + + For implementations creating other Kubernetes objects, this should be the `metadata.labels` field on resources. + For other implementations, this refers to any relevant (implementation specific) "labels" concepts. + + An implementation may chose to add additional implementation-specific labels as they see fit. + + If an implementation maps these labels to Pods, or any other resource that would need to be recreated when labels + change, it SHOULD clearly warn about this behavior in documentation. + + Support: Extended + maxProperties: 8 + type: object + x-kubernetes-validations: + - message: Label keys must be in the form of an optional DNS subdomain + prefix followed by a required name segment of up to 63 characters. + rule: self.all(key, key.matches(r"""^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?([A-Za-z0-9][-A-Za-z0-9_.]{0,61})?[A-Za-z0-9]$""")) + - message: If specified, the label key's prefix must be a DNS + subdomain not longer than 253 characters in total. + rule: self.all(key, key.split("/")[0].size() < 253) + parametersRef: + description: |- + ParametersRef is a reference to a resource that contains the configuration + parameters corresponding to the Gateway. This is optional if the + controller does not require any additional configuration. + + This follows the same semantics as GatewayClass's `parametersRef`, but on a per-Gateway basis + + The Gateway's GatewayClass may provide its own `parametersRef`. When both are specified, + the merging behavior is implementation specific. + It is generally recommended that GatewayClass provides defaults that can be overridden by a Gateway. + + If the referent cannot be found, refers to an unsupported kind, or when + the data within that resource is malformed, the Gateway SHOULD be + rejected with the "Accepted" status condition set to "False" and an + "InvalidParameters" reason. + + Support: Implementation-specific + properties: + group: + description: Group is the group of the referent. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + type: object + listeners: + description: |- + Listeners associated with this Gateway. Listeners define + logical endpoints that are bound on this Gateway's addresses. + At least one Listener MUST be specified. + + ## Distinct Listeners + + Each Listener in a set of Listeners (for example, in a single Gateway) + MUST be _distinct_, in that a traffic flow MUST be able to be assigned to + exactly one listener. (This section uses "set of Listeners" rather than + "Listeners in a single Gateway" because implementations MAY merge configuration + from multiple Gateways onto a single data plane, and these rules _also_ + apply in that case). + + Practically, this means that each listener in a set MUST have a unique + combination of Port, Protocol, and, if supported by the protocol, Hostname. + + Some combinations of port, protocol, and TLS settings are considered + Core support and MUST be supported by implementations based on the objects + they support: + + HTTPRoute + + 1. HTTPRoute, Port: 80, Protocol: HTTP + 2. HTTPRoute, Port: 443, Protocol: HTTPS, TLS Mode: Terminate, TLS keypair provided + + TLSRoute + + 1. TLSRoute, Port: 443, Protocol: TLS, TLS Mode: Passthrough + + "Distinct" Listeners have the following property: + + **The implementation can match inbound requests to a single distinct + Listener**. + + When multiple Listeners share values for fields (for + example, two Listeners with the same Port value), the implementation + can match requests to only one of the Listeners using other + Listener fields. + + When multiple listeners have the same value for the Protocol field, then + each of the Listeners with matching Protocol values MUST have different + values for other fields. + + The set of fields that MUST be different for a Listener differs per protocol. + The following rules define the rules for what fields MUST be considered for + Listeners to be distinct with each protocol currently defined in the + Gateway API spec. + + The set of listeners that all share a protocol value MUST have _different_ + values for _at least one_ of these fields to be distinct: + + * **HTTP, HTTPS, TLS**: Port, Hostname + * **TCP, UDP**: Port + + One **very** important rule to call out involves what happens when an + implementation: + + * Supports TCP protocol Listeners, as well as HTTP, HTTPS, or TLS protocol + Listeners, and + * sees HTTP, HTTPS, or TLS protocols with the same `port` as one with TCP + Protocol. + + In this case all the Listeners that share a port with the + TCP Listener are not distinct and so MUST NOT be accepted. + + If an implementation does not support TCP Protocol Listeners, then the + previous rule does not apply, and the TCP Listeners SHOULD NOT be + accepted. + + Note that the `tls` field is not used for determining if a listener is distinct, because + Listeners that _only_ differ on TLS config will still conflict in all cases. + + ### Listeners that are distinct only by Hostname + + When the Listeners are distinct based only on Hostname, inbound request + hostnames MUST match from the most specific to least specific Hostname + values to choose the correct Listener and its associated set of Routes. + + Exact matches MUST be processed before wildcard matches, and wildcard + matches MUST be processed before fallback (empty Hostname value) + matches. For example, `"foo.example.com"` takes precedence over + `"*.example.com"`, and `"*.example.com"` takes precedence over `""`. + + Additionally, if there are multiple wildcard entries, more specific + wildcard entries must be processed before less specific wildcard entries. + For example, `"*.foo.example.com"` takes precedence over `"*.example.com"`. + + The precise definition here is that the higher the number of dots in the + hostname to the right of the wildcard character, the higher the precedence. + + The wildcard character will match any number of characters _and dots_ to + the left, however, so `"*.example.com"` will match both + `"foo.bar.example.com"` _and_ `"bar.example.com"`. + + ## Handling indistinct Listeners + + If a set of Listeners contains Listeners that are not distinct, then those + Listeners are _Conflicted_, and the implementation MUST set the "Conflicted" + condition in the Listener Status to "True". + + The words "indistinct" and "conflicted" are considered equivalent for the + purpose of this documentation. + + Implementations MAY choose to accept a Gateway with some Conflicted + Listeners only if they only accept the partial Listener set that contains + no Conflicted Listeners. + + Specifically, an implementation MAY accept a partial Listener set subject to + the following rules: + + * The implementation MUST NOT pick one conflicting Listener as the winner. + ALL indistinct Listeners must not be accepted for processing. + * At least one distinct Listener MUST be present, or else the Gateway effectively + contains _no_ Listeners, and must be rejected from processing as a whole. + + The implementation MUST set a "ListenersNotValid" condition on the + Gateway Status when the Gateway contains Conflicted Listeners whether or + not they accept the Gateway. That Condition SHOULD clearly + indicate in the Message which Listeners are conflicted, and which are + Accepted. Additionally, the Listener status for those listeners SHOULD + indicate which Listeners are conflicted and not Accepted. + + ## General Listener behavior + + Note that, for all distinct Listeners, requests SHOULD match at most one Listener. + For example, if Listeners are defined for "foo.example.com" and "*.example.com", a + request to "foo.example.com" SHOULD only be routed using routes attached + to the "foo.example.com" Listener (and not the "*.example.com" Listener). + + This concept is known as "Listener Isolation", and it is an Extended feature + of Gateway API. Implementations that do not support Listener Isolation MUST + clearly document this, and MUST NOT claim support for the + `GatewayHTTPListenerIsolation` feature. + + Implementations that _do_ support Listener Isolation SHOULD claim support + for the Extended `GatewayHTTPListenerIsolation` feature and pass the associated + conformance tests. + + ## Compatible Listeners + + A Gateway's Listeners are considered _compatible_ if: + + 1. They are distinct. + 2. The implementation can serve them in compliance with the Addresses + requirement that all Listeners are available on all assigned + addresses. + + Compatible combinations in Extended support are expected to vary across + implementations. A combination that is compatible for one implementation + may not be compatible for another. + + For example, an implementation that cannot serve both TCP and UDP listeners + on the same address, or cannot mix HTTPS and generic TLS listens on the same port + would not consider those cases compatible, even though they are distinct. + + Implementations MAY merge separate Gateways onto a single set of + Addresses if all Listeners across all Gateways are compatible. + + In a future release the MinItems=1 requirement MAY be dropped. + + Support: Core + items: + description: |- + Listener embodies the concept of a logical endpoint where a Gateway accepts + network connections. + properties: + allowedRoutes: + default: + namespaces: + from: Same + description: |- + AllowedRoutes defines the types of routes that MAY be attached to a + Listener and the trusted namespaces where those Route resources MAY be + present. + + Although a client request may match multiple route rules, only one rule + may ultimately receive the request. Matching precedence MUST be + determined in order of the following criteria: + + * The most specific match as defined by the Route type. + * The oldest Route based on creation timestamp. For example, a Route with + a creation timestamp of "2020-09-08 01:02:03" is given precedence over + a Route with a creation timestamp of "2020-09-08 01:02:04". + * If everything else is equivalent, the Route appearing first in + alphabetical order (namespace/name) should be given precedence. For + example, foo/bar is given precedence over foo/baz. + + All valid rules within a Route attached to this Listener should be + implemented. Invalid Route rules can be ignored (sometimes that will mean + the full Route). If a Route rule transitions from valid to invalid, + support for that Route rule should be dropped to ensure consistency. For + example, even if a filter specified by a Route rule is invalid, the rest + of the rules within that Route should still be supported. + + Support: Core + properties: + kinds: + description: |- + Kinds specifies the groups and kinds of Routes that are allowed to bind + to this Gateway Listener. When unspecified or empty, the kinds of Routes + selected are determined using the Listener protocol. + + A RouteGroupKind MUST correspond to kinds of Routes that are compatible + with the application protocol specified in the Listener's Protocol field. + If an implementation does not support or recognize this resource type, it + MUST set the "ResolvedRefs" condition to False for this Listener with the + "InvalidRouteKinds" reason. + + Support: Core + items: + description: RouteGroupKind indicates the group and kind + of a Route resource. + properties: + group: + default: gateway.networking.k8s.io + description: Group is the group of the Route. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is the kind of the Route. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + required: + - kind + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + namespaces: + default: + from: Same + description: |- + Namespaces indicates namespaces from which Routes may be attached to this + Listener. This is restricted to the namespace of this Gateway by default. + + Support: Core + properties: + from: + default: Same + description: |- + From indicates where Routes will be selected for this Gateway. Possible + values are: + + * All: Routes in all namespaces may be used by this Gateway. + * Selector: Routes in namespaces selected by the selector may be used by + this Gateway. + * Same: Only Routes in the same namespace may be used by this Gateway. + + Support: Core + enum: + - All + - Selector + - Same + type: string + selector: + description: |- + Selector must be specified when From is set to "Selector". In that case, + only Routes in Namespaces matching this Selector will be selected by this + Gateway. This field is ignored for other values of "From". + + Support: Core + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: object + hostname: + description: |- + Hostname specifies the virtual hostname to match for protocol types that + define this concept. When unspecified, all hostnames are matched. This + field is ignored for protocols that don't require hostname based + matching. + + Implementations MUST apply Hostname matching appropriately for each of + the following protocols: + + * TLS: The Listener Hostname MUST match the SNI. + * HTTP: The Listener Hostname MUST match the Host header of the request. + * HTTPS: The Listener Hostname SHOULD match both the SNI and Host header. + Note that this does not require the SNI and Host header to be the same. + The semantics of this are described in more detail below. + + To ensure security, Section 11.1 of RFC-6066 emphasizes that server + implementations that rely on SNI hostname matching MUST also verify + hostnames within the application protocol. + + Section 9.1.2 of RFC-7540 provides a mechanism for servers to reject the + reuse of a connection by responding with the HTTP 421 Misdirected Request + status code. This indicates that the origin server has rejected the + request because it appears to have been misdirected. + + To detect misdirected requests, Gateways SHOULD match the authority of + the requests with all the SNI hostname(s) configured across all the + Gateway Listeners on the same port and protocol: + + * If another Listener has an exact match or more specific wildcard entry, + the Gateway SHOULD return a 421. + * If the current Listener (selected by SNI matching during ClientHello) + does not match the Host: + * If another Listener does match the Host, the Gateway SHOULD return a + 421. + * If no other Listener matches the Host, the Gateway MUST return a + 404. + + For HTTPRoute and TLSRoute resources, there is an interaction with the + `spec.hostnames` array. When both listener and route specify hostnames, + there MUST be an intersection between the values for a Route to be + accepted. For more information, refer to the Route specific Hostnames + documentation. + + Hostnames that are prefixed with a wildcard label (`*.`) are interpreted + as a suffix match. That means that a match for `*.example.com` would match + both `test.example.com`, and `foo.test.example.com`, but not `example.com`. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + name: + description: |- + Name is the name of the Listener. This name MUST be unique within a + Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + port: + description: |- + Port is the network port. Multiple listeners may use the + same port, subject to the Listener compatibility rules. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + description: |- + Protocol specifies the network protocol this listener expects to receive. + + Support: Core + maxLength: 255 + minLength: 1 + pattern: ^[a-zA-Z0-9]([-a-zA-Z0-9]*[a-zA-Z0-9])?$|[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9]+$ + type: string + tls: + description: |- + TLS is the TLS configuration for the Listener. This field is required if + the Protocol field is "HTTPS" or "TLS". It is invalid to set this field + if the Protocol field is "HTTP", "TCP", or "UDP". + + The association of SNIs to Certificate defined in ListenerTLSConfig is + defined based on the Hostname field for this listener. + + The GatewayClass MUST use the longest matching SNI out of all + available certificates for any TLS handshake. + + Support: Core + properties: + certificateRefs: + description: |- + CertificateRefs contains a series of references to Kubernetes objects that + contains TLS certificates and private keys. These certificates are used to + establish a TLS handshake for requests that match the hostname of the + associated listener. + + A single CertificateRef to a Kubernetes Secret has "Core" support. + Implementations MAY choose to support attaching multiple certificates to + a Listener, but this behavior is implementation-specific. + + References to a resource in different namespace are invalid UNLESS there + is a ReferenceGrant in the target namespace that allows the certificate + to be attached. If a ReferenceGrant does not allow this reference, the + "ResolvedRefs" condition MUST be set to False for this listener with the + "RefNotPermitted" reason. + + This field is required to have at least one element when the mode is set + to "Terminate" (default) and is optional otherwise. + + CertificateRefs can reference to standard Kubernetes resources, i.e. + Secret, or implementation-specific custom resources. + + Support: Core - A single reference to a Kubernetes Secret of type kubernetes.io/tls + + Support: Implementation-specific (More than one reference or other resource types) + items: + description: |- + SecretObjectReference identifies an API object including its namespace, + defaulting to Secret. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + + References to objects with invalid Group and Kind are not valid, and must + be rejected by the implementation, with appropriate Conditions set + on the containing object. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-type: atomic + mode: + default: Terminate + description: |- + Mode defines the TLS behavior for the TLS session initiated by the client. + There are two possible modes: + + - Terminate: The TLS session between the downstream client and the + Gateway is terminated at the Gateway. This mode requires certificates + to be specified in some way, such as populating the certificateRefs + field. + - Passthrough: The TLS session is NOT terminated by the Gateway. This + implies that the Gateway can't decipher the TLS stream except for + the ClientHello message of the TLS protocol. The certificateRefs field + is ignored in this mode. + + Support: Core + enum: + - Terminate + - Passthrough + type: string + options: + additionalProperties: + description: |- + AnnotationValue is the value of an annotation in Gateway API. This is used + for validation of maps such as TLS options. This roughly matches Kubernetes + annotation validation, although the length validation in that case is based + on the entire size of the annotations struct. + maxLength: 4096 + minLength: 0 + type: string + description: |- + Options are a list of key/value pairs to enable extended TLS + configuration for each implementation. For example, configuring the + minimum TLS version or supported cipher suites. + + A set of common keys MAY be defined by the API in the future. To avoid + any ambiguity, implementation-specific definitions MUST use + domain-prefixed names, such as `example.com/my-custom-option`. + Un-prefixed names are reserved for key names defined by Gateway API. + + Support: Implementation-specific + maxProperties: 16 + type: object + type: object + x-kubernetes-validations: + - message: certificateRefs or options must be specified when + mode is Terminate + rule: 'self.mode == ''Terminate'' ? size(self.certificateRefs) + > 0 || size(self.options) > 0 : true' + required: + - name + - port + - protocol + type: object + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: tls must not be specified for protocols ['HTTP', 'TCP', + 'UDP'] + rule: 'self.all(l, l.protocol in [''HTTP'', ''TCP'', ''UDP''] ? + !has(l.tls) : true)' + - message: tls mode must be Terminate for protocol HTTPS + rule: 'self.all(l, (l.protocol == ''HTTPS'' && has(l.tls)) ? (l.tls.mode + == '''' || l.tls.mode == ''Terminate'') : true)' + - message: tls mode must be set for protocol TLS + rule: 'self.all(l, (l.protocol == ''TLS'' ? has(l.tls) && has(l.tls.mode) + && l.tls.mode != '''' : true))' + - message: hostname must not be specified for protocols ['TCP', 'UDP'] + rule: 'self.all(l, l.protocol in [''TCP'', ''UDP''] ? (!has(l.hostname) + || l.hostname == '''') : true)' + - message: Listener name must be unique within the Gateway + rule: self.all(l1, self.exists_one(l2, l1.name == l2.name)) + - message: Combination of port, protocol and hostname must be unique + for each listener + rule: 'self.all(l1, self.exists_one(l2, l1.port == l2.port && l1.protocol + == l2.protocol && (has(l1.hostname) && has(l2.hostname) ? l1.hostname + == l2.hostname : !has(l1.hostname) && !has(l2.hostname))))' + tls: + description: |- + TLS specifies frontend and backend tls configuration for entire gateway. + + Support: Extended + properties: + backend: + description: |- + Backend describes TLS configuration for gateway when connecting + to backends. + + Note that this contains only details for the Gateway as a TLS client, + and does _not_ imply behavior about how to choose which backend should + get a TLS connection. That is determined by the presence of a BackendTLSPolicy. + + Support: Core + properties: + clientCertificateRef: + description: |- + ClientCertificateRef references an object that contains a client certificate + and its associated private key. It can reference standard Kubernetes resources, + i.e., Secret, or implementation-specific custom resources. + + A ClientCertificateRef is considered invalid if: + + * It refers to a resource that cannot be resolved (e.g., the referenced resource + does not exist) or is misconfigured (e.g., a Secret does not contain the keys + named `tls.crt` and `tls.key`). In this case, the `ResolvedRefs` condition + on the Gateway MUST be set to False with the Reason `InvalidClientCertificateRef` + and the Message of the Condition MUST indicate why the reference is invalid. + + * It refers to a resource in another namespace UNLESS there is a ReferenceGrant + in the target namespace that allows the certificate to be attached. + If a ReferenceGrant does not allow this reference, the `ResolvedRefs` condition + on the Gateway MUST be set to False with the Reason `RefNotPermitted`. + + Implementations MAY choose to perform further validation of the certificate + content (e.g., checking expiry or enforcing specific formats). In such cases, + an implementation-specific Reason and Message MUST be set. + + Support: Core - Reference to a Kubernetes TLS Secret (with the type `kubernetes.io/tls`). + Support: Implementation-specific - Other resource kinds or Secrets with a + different type (e.g., `Opaque`). + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + type: object + frontend: + description: |- + Frontend describes TLS config when client connects to Gateway. + Support: Core + properties: + default: + description: |- + Default specifies the default client certificate validation configuration + for all Listeners handling HTTPS traffic, unless a per-port configuration + is defined. + + support: Core + properties: + validation: + description: |- + Validation holds configuration information for validating the frontend (client). + Setting this field will result in mutual authentication when connecting to the gateway. + In browsers this may result in a dialog appearing + that requests a user to specify the client certificate. + The maximum depth of a certificate chain accepted in verification is Implementation specific. + + Support: Core + properties: + caCertificateRefs: + description: |- + CACertificateRefs contains one or more references to Kubernetes + objects that contain a PEM-encoded TLS CA certificate bundle, which + is used as a trust anchor to validate the certificates presented by + the client. + + A CACertificateRef is invalid if: + + * It refers to a resource that cannot be resolved (e.g., the + referenced resource does not exist) or is misconfigured (e.g., a + ConfigMap does not contain a key named `ca.crt`). In this case, the + Reason on all matching HTTPS listeners must be set to `InvalidCACertificateRef` + and the Message of the Condition must indicate which reference is invalid and why. + + * It refers to an unknown or unsupported kind of resource. In this + case, the Reason on all matching HTTPS listeners must be set to + `InvalidCACertificateKind` and the Message of the Condition must explain + which kind of resource is unknown or unsupported. + + * It refers to a resource in another namespace UNLESS there is a + ReferenceGrant in the target namespace that allows the CA + certificate to be attached. If a ReferenceGrant does not allow this + reference, the `ResolvedRefs` on all matching HTTPS listeners condition + MUST be set with the Reason `RefNotPermitted`. + + Implementations MAY choose to perform further validation of the + certificate content (e.g., checking expiry or enforcing specific formats). + In such cases, an implementation-specific Reason and Message MUST be set. + + In all cases, the implementation MUST ensure that the `ResolvedRefs` + condition is set to `status: False` on all targeted listeners (i.e., + listeners serving HTTPS on a matching port). The condition MUST + include a Reason and Message that indicate the cause of the error. If + ALL CACertificateRefs are invalid, the implementation MUST also ensure + the `Accepted` condition on the listener is set to `status: False`, with + the Reason `NoValidCACertificate`. + Implementations MAY choose to support attaching multiple CA certificates + to a listener, but this behavior is implementation-specific. + + Support: Core - A single reference to a Kubernetes ConfigMap, with the + CA certificate in a key named `ca.crt`. + + Support: Implementation-specific - More than one reference, other kinds + of resources, or a single reference that includes multiple certificates. + items: + description: |- + ObjectReference identifies an API object including its namespace. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + + References to objects with invalid Group and Kind are not valid, and must + be rejected by the implementation, with appropriate Conditions set + on the containing object. + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When set to the empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. For + example "ConfigMap" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + mode: + default: AllowValidOnly + description: |- + FrontendValidationMode defines the mode for validating the client certificate. + There are two possible modes: + + - AllowValidOnly: In this mode, the gateway will accept connections only if + the client presents a valid certificate. This certificate must successfully + pass validation against the CA certificates specified in `CACertificateRefs`. + - AllowInsecureFallback: In this mode, the gateway will accept connections + even if the client certificate is not presented or fails verification. + + This approach delegates client authorization to the backend and introduce + a significant security risk. It should be used in testing environments or + on a temporary basis in non-testing environments. + + Defaults to AllowValidOnly. + + Support: Core + enum: + - AllowValidOnly + - AllowInsecureFallback + type: string + required: + - caCertificateRefs + type: object + type: object + perPort: + description: |- + PerPort specifies tls configuration assigned per port. + Per port configuration is optional. Once set this configuration overrides + the default configuration for all Listeners handling HTTPS traffic + that match this port. + Each override port requires a unique TLS configuration. + + support: Core + items: + properties: + port: + description: |- + The Port indicates the Port Number to which the TLS configuration will be + applied. This configuration will be applied to all Listeners handling HTTPS + traffic that match this port. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + tls: + description: |- + TLS store the configuration that will be applied to all Listeners handling + HTTPS traffic and matching given port. + + Support: Core + properties: + validation: + description: |- + Validation holds configuration information for validating the frontend (client). + Setting this field will result in mutual authentication when connecting to the gateway. + In browsers this may result in a dialog appearing + that requests a user to specify the client certificate. + The maximum depth of a certificate chain accepted in verification is Implementation specific. + + Support: Core + properties: + caCertificateRefs: + description: |- + CACertificateRefs contains one or more references to Kubernetes + objects that contain a PEM-encoded TLS CA certificate bundle, which + is used as a trust anchor to validate the certificates presented by + the client. + + A CACertificateRef is invalid if: + + * It refers to a resource that cannot be resolved (e.g., the + referenced resource does not exist) or is misconfigured (e.g., a + ConfigMap does not contain a key named `ca.crt`). In this case, the + Reason on all matching HTTPS listeners must be set to `InvalidCACertificateRef` + and the Message of the Condition must indicate which reference is invalid and why. + + * It refers to an unknown or unsupported kind of resource. In this + case, the Reason on all matching HTTPS listeners must be set to + `InvalidCACertificateKind` and the Message of the Condition must explain + which kind of resource is unknown or unsupported. + + * It refers to a resource in another namespace UNLESS there is a + ReferenceGrant in the target namespace that allows the CA + certificate to be attached. If a ReferenceGrant does not allow this + reference, the `ResolvedRefs` on all matching HTTPS listeners condition + MUST be set with the Reason `RefNotPermitted`. + + Implementations MAY choose to perform further validation of the + certificate content (e.g., checking expiry or enforcing specific formats). + In such cases, an implementation-specific Reason and Message MUST be set. + + In all cases, the implementation MUST ensure that the `ResolvedRefs` + condition is set to `status: False` on all targeted listeners (i.e., + listeners serving HTTPS on a matching port). The condition MUST + include a Reason and Message that indicate the cause of the error. If + ALL CACertificateRefs are invalid, the implementation MUST also ensure + the `Accepted` condition on the listener is set to `status: False`, with + the Reason `NoValidCACertificate`. + Implementations MAY choose to support attaching multiple CA certificates + to a listener, but this behavior is implementation-specific. + + Support: Core - A single reference to a Kubernetes ConfigMap, with the + CA certificate in a key named `ca.crt`. + + Support: Implementation-specific - More than one reference, other kinds + of resources, or a single reference that includes multiple certificates. + items: + description: |- + ObjectReference identifies an API object including its namespace. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + + References to objects with invalid Group and Kind are not valid, and must + be rejected by the implementation, with appropriate Conditions set + on the containing object. + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When set to the empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. + For example "ConfigMap" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + mode: + default: AllowValidOnly + description: |- + FrontendValidationMode defines the mode for validating the client certificate. + There are two possible modes: + + - AllowValidOnly: In this mode, the gateway will accept connections only if + the client presents a valid certificate. This certificate must successfully + pass validation against the CA certificates specified in `CACertificateRefs`. + - AllowInsecureFallback: In this mode, the gateway will accept connections + even if the client certificate is not presented or fails verification. + + This approach delegates client authorization to the backend and introduce + a significant security risk. It should be used in testing environments or + on a temporary basis in non-testing environments. + + Defaults to AllowValidOnly. + + Support: Core + enum: + - AllowValidOnly + - AllowInsecureFallback + type: string + required: + - caCertificateRefs + type: object + type: object + required: + - port + - tls + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: Port for TLS configuration must be unique within + the Gateway + rule: self.all(t1, self.exists_one(t2, t1.port == t2.port)) + required: + - default + type: object + type: object + required: + - gatewayClassName + - listeners + type: object + status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Accepted + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: Status defines the current state of Gateway. + properties: + addresses: + description: |- + Addresses lists the network addresses that have been bound to the + Gateway. + + This list may differ from the addresses provided in the spec under some + conditions: + + * no addresses are specified, all addresses are dynamically assigned + * a combination of specified and dynamic addresses are assigned + * a specified address was unusable (e.g. already in use) + items: + description: GatewayStatusAddress describes a network address that + is bound to a Gateway. + oneOf: + - properties: + type: + enum: + - IPAddress + value: + anyOf: + - format: ipv4 + - format: ipv6 + - properties: + type: + not: + enum: + - IPAddress + properties: + type: + default: IPAddress + description: Type of the address. + maxLength: 253 + minLength: 1 + pattern: ^Hostname|IPAddress|NamedAddress|[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + value: + description: |- + Value of the address. The validity of the values will depend + on the type and support by the controller. + + Examples: `1.2.3.4`, `128::1`, `my-ip-address`. + maxLength: 253 + minLength: 1 + type: string + required: + - value + type: object + x-kubernetes-validations: + - message: Hostname value must only contain valid characters (matching + ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$) + rule: 'self.type == ''Hostname'' ? self.value.matches(r"""^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"""): + true' + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + attachedListenerSets: + description: |- + AttachedListenerSets represents the total number of ListenerSets that have been + successfully attached to this Gateway. + + A ListenerSet is successfully attached to a Gateway when all the following conditions are met: + - The ListenerSet is selected by the Gateway's AllowedListeners field + - The ListenerSet has a valid ParentRef selecting the Gateway + - The ListenerSet's status has the condition "Accepted: true" + + Uses for this field include troubleshooting AttachedListenerSets attachment and + measuring blast radius/impact of changes to a Gateway. + format: int32 + type: integer + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Accepted + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: |- + Conditions describe the current conditions of the Gateway. + + Implementations should prefer to express Gateway conditions + using the `GatewayConditionType` and `GatewayConditionReason` + constants so that operators and tools can converge on a common + vocabulary to describe Gateway state. + + Known condition types are: + + * "Accepted" + * "Programmed" + * "Ready" + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + listeners: + description: Listeners provide status for each unique listener port + defined in the Spec. + items: + description: ListenerStatus is the status associated with a Listener. + properties: + attachedRoutes: + description: |- + AttachedRoutes represents the total number of Routes that have been + successfully attached to this Listener. + + Successful attachment of a Route to a Listener is based solely on the + combination of the AllowedRoutes field on the corresponding Listener + and the Route's ParentRefs field. A Route is successfully attached to + a Listener when it is selected by the Listener's AllowedRoutes field + AND the Route has a valid ParentRef selecting the whole Gateway + resource or a specific Listener as a parent resource (more detail on + attachment semantics can be found in the documentation on the various + Route kinds ParentRefs fields). Listener or Route status does not impact + successful attachment, i.e. the AttachedRoutes field count MUST be set + for Listeners, even if the Accepted condition of an individual Listener is set + to "False". The AttachedRoutes number represents the number of Routes with + the Accepted condition set to "True" that have been attached to this Listener. + Routes with any other value for the Accepted condition MUST NOT be included + in this count. + + Uses for this field include troubleshooting Route attachment and + measuring blast radius/impact of changes to a Listener. + format: int32 + type: integer + conditions: + description: Conditions describe the current condition of this + listener. + items: + description: Condition contains details for one aspect of + the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + name: + description: Name is the name of the Listener that this status + corresponds to. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + supportedKinds: + description: |- + SupportedKinds is the list indicating the Kinds supported by this + listener. This MUST represent the kinds supported by an implementation for + that Listener configuration. + + If kinds are specified in Spec that are not supported, they MUST NOT + appear in this list and an implementation MUST set the "ResolvedRefs" + condition to "False" with the "InvalidRouteKinds" reason. If both valid + and invalid Route kinds are specified, the implementation MUST + reference the valid Route kinds that have been specified. + items: + description: RouteGroupKind indicates the group and kind of + a Route resource. + properties: + group: + default: gateway.networking.k8s.io + description: Group is the group of the Route. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is the kind of the Route. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + required: + - kind + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + required: + - attachedRoutes + - conditions + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.gatewayClassName + name: Class + type: string + - jsonPath: .status.addresses[*].value + name: Address + type: string + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + Gateway represents an instance of a service-traffic handling infrastructure + by binding Listeners to a set of IP addresses. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of Gateway. + properties: + addresses: + description: |- + Addresses requested for this Gateway. This is optional and behavior can + depend on the implementation. If a value is set in the spec and the + requested address is invalid or unavailable, the implementation MUST + indicate this in an associated entry in GatewayStatus.Conditions. + + The Addresses field represents a request for the address(es) on the + "outside of the Gateway", that traffic bound for this Gateway will use. + This could be the IP address or hostname of an external load balancer or + other networking infrastructure, or some other address that traffic will + be sent to. + + If no Addresses are specified, the implementation MAY schedule the + Gateway in an implementation-specific manner, assigning an appropriate + set of Addresses. + + The implementation MUST bind all Listeners to every GatewayAddress that + it assigns to the Gateway and add a corresponding entry in + GatewayStatus.Addresses. + + Support: Extended + items: + description: GatewaySpecAddress describes an address that can be + bound to a Gateway. + oneOf: + - properties: + type: + enum: + - IPAddress + value: + anyOf: + - format: ipv4 + - format: ipv6 + - properties: + type: + not: + enum: + - IPAddress + properties: + type: + default: IPAddress + description: Type of the address. + maxLength: 253 + minLength: 1 + pattern: ^Hostname|IPAddress|NamedAddress|[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + value: + description: |- + When a value is unspecified, an implementation SHOULD automatically + assign an address matching the requested type if possible. + + If an implementation does not support an empty value, they MUST set the + "Programmed" condition in status to False with a reason of "AddressNotAssigned". + + Examples: `1.2.3.4`, `128::1`, `my-ip-address`. + maxLength: 253 + type: string + type: object + x-kubernetes-validations: + - message: Hostname value must be empty or contain only valid characters + (matching ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$) + rule: 'self.type == ''Hostname'' ? (!has(self.value) || self.value.matches(r"""^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$""")): + true' + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: IPAddress values must be unique + rule: 'self.all(a1, a1.type == ''IPAddress'' && has(a1.value) ? + self.exists_one(a2, a2.type == a1.type && has(a2.value) && a2.value + == a1.value) : true )' + - message: Hostname values must be unique + rule: 'self.all(a1, a1.type == ''Hostname'' && has(a1.value) ? + self.exists_one(a2, a2.type == a1.type && has(a2.value) && a2.value + == a1.value) : true )' + allowedListeners: + description: |- + AllowedListeners defines which ListenerSets can be attached to this Gateway. + The default value is to allow no ListenerSets. + properties: + namespaces: + default: + from: None + description: |- + Namespaces defines which namespaces ListenerSets can be attached to this Gateway. + The default value is to allow no ListenerSets. + properties: + from: + default: None + description: |- + From indicates where ListenerSets can attach to this Gateway. Possible + values are: + + * Same: Only ListenerSets in the same namespace may be attached to this Gateway. + * Selector: ListenerSets in namespaces selected by the selector may be attached to this Gateway. + * All: ListenerSets in all namespaces may be attached to this Gateway. + * None: Only listeners defined in the Gateway's spec are allowed + + The default value None + enum: + - All + - Selector + - Same + - None + type: string + selector: + description: |- + Selector must be specified when From is set to "Selector". In that case, + only ListenerSets in Namespaces matching this Selector will be selected by this + Gateway. This field is ignored for other values of "From". + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: object + gatewayClassName: + description: |- + GatewayClassName used for this Gateway. This is the name of a + GatewayClass resource. + maxLength: 253 + minLength: 1 + type: string + infrastructure: + description: |- + Infrastructure defines infrastructure level attributes about this Gateway instance. + + Support: Extended + properties: + annotations: + additionalProperties: + description: |- + AnnotationValue is the value of an annotation in Gateway API. This is used + for validation of maps such as TLS options. This roughly matches Kubernetes + annotation validation, although the length validation in that case is based + on the entire size of the annotations struct. + maxLength: 4096 + minLength: 0 + type: string + description: |- + Annotations that SHOULD be applied to any resources created in response to this Gateway. + + For implementations creating other Kubernetes objects, this should be the `metadata.annotations` field on resources. + For other implementations, this refers to any relevant (implementation specific) "annotations" concepts. + + An implementation may chose to add additional implementation-specific annotations as they see fit. + + Support: Extended + maxProperties: 8 + type: object + x-kubernetes-validations: + - message: Annotation keys must be in the form of an optional + DNS subdomain prefix followed by a required name segment of + up to 63 characters. + rule: self.all(key, key.matches(r"""^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?([A-Za-z0-9][-A-Za-z0-9_.]{0,61})?[A-Za-z0-9]$""")) + - message: If specified, the annotation key's prefix must be a + DNS subdomain not longer than 253 characters in total. + rule: self.all(key, key.split("/")[0].size() < 253) + labels: + additionalProperties: + description: |- + LabelValue is the value of a label in the Gateway API. This is used for validation + of maps such as Gateway infrastructure labels. This matches the Kubernetes + label validation rules: + * must be 63 characters or less (can be empty), + * unless empty, must begin and end with an alphanumeric character ([a-z0-9A-Z]), + * could contain dashes (-), underscores (_), dots (.), and alphanumerics between. + + Valid values include: + + * MyValue + * my.name + * 123-my-value + maxLength: 63 + minLength: 0 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + description: |- + Labels that SHOULD be applied to any resources created in response to this Gateway. + + For implementations creating other Kubernetes objects, this should be the `metadata.labels` field on resources. + For other implementations, this refers to any relevant (implementation specific) "labels" concepts. + + An implementation may chose to add additional implementation-specific labels as they see fit. + + If an implementation maps these labels to Pods, or any other resource that would need to be recreated when labels + change, it SHOULD clearly warn about this behavior in documentation. + + Support: Extended + maxProperties: 8 + type: object + x-kubernetes-validations: + - message: Label keys must be in the form of an optional DNS subdomain + prefix followed by a required name segment of up to 63 characters. + rule: self.all(key, key.matches(r"""^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?([A-Za-z0-9][-A-Za-z0-9_.]{0,61})?[A-Za-z0-9]$""")) + - message: If specified, the label key's prefix must be a DNS + subdomain not longer than 253 characters in total. + rule: self.all(key, key.split("/")[0].size() < 253) + parametersRef: + description: |- + ParametersRef is a reference to a resource that contains the configuration + parameters corresponding to the Gateway. This is optional if the + controller does not require any additional configuration. + + This follows the same semantics as GatewayClass's `parametersRef`, but on a per-Gateway basis + + The Gateway's GatewayClass may provide its own `parametersRef`. When both are specified, + the merging behavior is implementation specific. + It is generally recommended that GatewayClass provides defaults that can be overridden by a Gateway. + + If the referent cannot be found, refers to an unsupported kind, or when + the data within that resource is malformed, the Gateway SHOULD be + rejected with the "Accepted" status condition set to "False" and an + "InvalidParameters" reason. + + Support: Implementation-specific + properties: + group: + description: Group is the group of the referent. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + type: object + listeners: + description: |- + Listeners associated with this Gateway. Listeners define + logical endpoints that are bound on this Gateway's addresses. + At least one Listener MUST be specified. + + ## Distinct Listeners + + Each Listener in a set of Listeners (for example, in a single Gateway) + MUST be _distinct_, in that a traffic flow MUST be able to be assigned to + exactly one listener. (This section uses "set of Listeners" rather than + "Listeners in a single Gateway" because implementations MAY merge configuration + from multiple Gateways onto a single data plane, and these rules _also_ + apply in that case). + + Practically, this means that each listener in a set MUST have a unique + combination of Port, Protocol, and, if supported by the protocol, Hostname. + + Some combinations of port, protocol, and TLS settings are considered + Core support and MUST be supported by implementations based on the objects + they support: + + HTTPRoute + + 1. HTTPRoute, Port: 80, Protocol: HTTP + 2. HTTPRoute, Port: 443, Protocol: HTTPS, TLS Mode: Terminate, TLS keypair provided + + TLSRoute + + 1. TLSRoute, Port: 443, Protocol: TLS, TLS Mode: Passthrough + + "Distinct" Listeners have the following property: + + **The implementation can match inbound requests to a single distinct + Listener**. + + When multiple Listeners share values for fields (for + example, two Listeners with the same Port value), the implementation + can match requests to only one of the Listeners using other + Listener fields. + + When multiple listeners have the same value for the Protocol field, then + each of the Listeners with matching Protocol values MUST have different + values for other fields. + + The set of fields that MUST be different for a Listener differs per protocol. + The following rules define the rules for what fields MUST be considered for + Listeners to be distinct with each protocol currently defined in the + Gateway API spec. + + The set of listeners that all share a protocol value MUST have _different_ + values for _at least one_ of these fields to be distinct: + + * **HTTP, HTTPS, TLS**: Port, Hostname + * **TCP, UDP**: Port + + One **very** important rule to call out involves what happens when an + implementation: + + * Supports TCP protocol Listeners, as well as HTTP, HTTPS, or TLS protocol + Listeners, and + * sees HTTP, HTTPS, or TLS protocols with the same `port` as one with TCP + Protocol. + + In this case all the Listeners that share a port with the + TCP Listener are not distinct and so MUST NOT be accepted. + + If an implementation does not support TCP Protocol Listeners, then the + previous rule does not apply, and the TCP Listeners SHOULD NOT be + accepted. + + Note that the `tls` field is not used for determining if a listener is distinct, because + Listeners that _only_ differ on TLS config will still conflict in all cases. + + ### Listeners that are distinct only by Hostname + + When the Listeners are distinct based only on Hostname, inbound request + hostnames MUST match from the most specific to least specific Hostname + values to choose the correct Listener and its associated set of Routes. + + Exact matches MUST be processed before wildcard matches, and wildcard + matches MUST be processed before fallback (empty Hostname value) + matches. For example, `"foo.example.com"` takes precedence over + `"*.example.com"`, and `"*.example.com"` takes precedence over `""`. + + Additionally, if there are multiple wildcard entries, more specific + wildcard entries must be processed before less specific wildcard entries. + For example, `"*.foo.example.com"` takes precedence over `"*.example.com"`. + + The precise definition here is that the higher the number of dots in the + hostname to the right of the wildcard character, the higher the precedence. + + The wildcard character will match any number of characters _and dots_ to + the left, however, so `"*.example.com"` will match both + `"foo.bar.example.com"` _and_ `"bar.example.com"`. + + ## Handling indistinct Listeners + + If a set of Listeners contains Listeners that are not distinct, then those + Listeners are _Conflicted_, and the implementation MUST set the "Conflicted" + condition in the Listener Status to "True". + + The words "indistinct" and "conflicted" are considered equivalent for the + purpose of this documentation. + + Implementations MAY choose to accept a Gateway with some Conflicted + Listeners only if they only accept the partial Listener set that contains + no Conflicted Listeners. + + Specifically, an implementation MAY accept a partial Listener set subject to + the following rules: + + * The implementation MUST NOT pick one conflicting Listener as the winner. + ALL indistinct Listeners must not be accepted for processing. + * At least one distinct Listener MUST be present, or else the Gateway effectively + contains _no_ Listeners, and must be rejected from processing as a whole. + + The implementation MUST set a "ListenersNotValid" condition on the + Gateway Status when the Gateway contains Conflicted Listeners whether or + not they accept the Gateway. That Condition SHOULD clearly + indicate in the Message which Listeners are conflicted, and which are + Accepted. Additionally, the Listener status for those listeners SHOULD + indicate which Listeners are conflicted and not Accepted. + + ## General Listener behavior + + Note that, for all distinct Listeners, requests SHOULD match at most one Listener. + For example, if Listeners are defined for "foo.example.com" and "*.example.com", a + request to "foo.example.com" SHOULD only be routed using routes attached + to the "foo.example.com" Listener (and not the "*.example.com" Listener). + + This concept is known as "Listener Isolation", and it is an Extended feature + of Gateway API. Implementations that do not support Listener Isolation MUST + clearly document this, and MUST NOT claim support for the + `GatewayHTTPListenerIsolation` feature. + + Implementations that _do_ support Listener Isolation SHOULD claim support + for the Extended `GatewayHTTPListenerIsolation` feature and pass the associated + conformance tests. + + ## Compatible Listeners + + A Gateway's Listeners are considered _compatible_ if: + + 1. They are distinct. + 2. The implementation can serve them in compliance with the Addresses + requirement that all Listeners are available on all assigned + addresses. + + Compatible combinations in Extended support are expected to vary across + implementations. A combination that is compatible for one implementation + may not be compatible for another. + + For example, an implementation that cannot serve both TCP and UDP listeners + on the same address, or cannot mix HTTPS and generic TLS listens on the same port + would not consider those cases compatible, even though they are distinct. + + Implementations MAY merge separate Gateways onto a single set of + Addresses if all Listeners across all Gateways are compatible. + + In a future release the MinItems=1 requirement MAY be dropped. + + Support: Core + items: + description: |- + Listener embodies the concept of a logical endpoint where a Gateway accepts + network connections. + properties: + allowedRoutes: + default: + namespaces: + from: Same + description: |- + AllowedRoutes defines the types of routes that MAY be attached to a + Listener and the trusted namespaces where those Route resources MAY be + present. + + Although a client request may match multiple route rules, only one rule + may ultimately receive the request. Matching precedence MUST be + determined in order of the following criteria: + + * The most specific match as defined by the Route type. + * The oldest Route based on creation timestamp. For example, a Route with + a creation timestamp of "2020-09-08 01:02:03" is given precedence over + a Route with a creation timestamp of "2020-09-08 01:02:04". + * If everything else is equivalent, the Route appearing first in + alphabetical order (namespace/name) should be given precedence. For + example, foo/bar is given precedence over foo/baz. + + All valid rules within a Route attached to this Listener should be + implemented. Invalid Route rules can be ignored (sometimes that will mean + the full Route). If a Route rule transitions from valid to invalid, + support for that Route rule should be dropped to ensure consistency. For + example, even if a filter specified by a Route rule is invalid, the rest + of the rules within that Route should still be supported. + + Support: Core + properties: + kinds: + description: |- + Kinds specifies the groups and kinds of Routes that are allowed to bind + to this Gateway Listener. When unspecified or empty, the kinds of Routes + selected are determined using the Listener protocol. + + A RouteGroupKind MUST correspond to kinds of Routes that are compatible + with the application protocol specified in the Listener's Protocol field. + If an implementation does not support or recognize this resource type, it + MUST set the "ResolvedRefs" condition to False for this Listener with the + "InvalidRouteKinds" reason. + + Support: Core + items: + description: RouteGroupKind indicates the group and kind + of a Route resource. + properties: + group: + default: gateway.networking.k8s.io + description: Group is the group of the Route. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is the kind of the Route. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + required: + - kind + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + namespaces: + default: + from: Same + description: |- + Namespaces indicates namespaces from which Routes may be attached to this + Listener. This is restricted to the namespace of this Gateway by default. + + Support: Core + properties: + from: + default: Same + description: |- + From indicates where Routes will be selected for this Gateway. Possible + values are: + + * All: Routes in all namespaces may be used by this Gateway. + * Selector: Routes in namespaces selected by the selector may be used by + this Gateway. + * Same: Only Routes in the same namespace may be used by this Gateway. + + Support: Core + enum: + - All + - Selector + - Same + type: string + selector: + description: |- + Selector must be specified when From is set to "Selector". In that case, + only Routes in Namespaces matching this Selector will be selected by this + Gateway. This field is ignored for other values of "From". + + Support: Core + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: object + hostname: + description: |- + Hostname specifies the virtual hostname to match for protocol types that + define this concept. When unspecified, all hostnames are matched. This + field is ignored for protocols that don't require hostname based + matching. + + Implementations MUST apply Hostname matching appropriately for each of + the following protocols: + + * TLS: The Listener Hostname MUST match the SNI. + * HTTP: The Listener Hostname MUST match the Host header of the request. + * HTTPS: The Listener Hostname SHOULD match both the SNI and Host header. + Note that this does not require the SNI and Host header to be the same. + The semantics of this are described in more detail below. + + To ensure security, Section 11.1 of RFC-6066 emphasizes that server + implementations that rely on SNI hostname matching MUST also verify + hostnames within the application protocol. + + Section 9.1.2 of RFC-7540 provides a mechanism for servers to reject the + reuse of a connection by responding with the HTTP 421 Misdirected Request + status code. This indicates that the origin server has rejected the + request because it appears to have been misdirected. + + To detect misdirected requests, Gateways SHOULD match the authority of + the requests with all the SNI hostname(s) configured across all the + Gateway Listeners on the same port and protocol: + + * If another Listener has an exact match or more specific wildcard entry, + the Gateway SHOULD return a 421. + * If the current Listener (selected by SNI matching during ClientHello) + does not match the Host: + * If another Listener does match the Host, the Gateway SHOULD return a + 421. + * If no other Listener matches the Host, the Gateway MUST return a + 404. + + For HTTPRoute and TLSRoute resources, there is an interaction with the + `spec.hostnames` array. When both listener and route specify hostnames, + there MUST be an intersection between the values for a Route to be + accepted. For more information, refer to the Route specific Hostnames + documentation. + + Hostnames that are prefixed with a wildcard label (`*.`) are interpreted + as a suffix match. That means that a match for `*.example.com` would match + both `test.example.com`, and `foo.test.example.com`, but not `example.com`. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + name: + description: |- + Name is the name of the Listener. This name MUST be unique within a + Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + port: + description: |- + Port is the network port. Multiple listeners may use the + same port, subject to the Listener compatibility rules. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + description: |- + Protocol specifies the network protocol this listener expects to receive. + + Support: Core + maxLength: 255 + minLength: 1 + pattern: ^[a-zA-Z0-9]([-a-zA-Z0-9]*[a-zA-Z0-9])?$|[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9]+$ + type: string + tls: + description: |- + TLS is the TLS configuration for the Listener. This field is required if + the Protocol field is "HTTPS" or "TLS". It is invalid to set this field + if the Protocol field is "HTTP", "TCP", or "UDP". + + The association of SNIs to Certificate defined in ListenerTLSConfig is + defined based on the Hostname field for this listener. + + The GatewayClass MUST use the longest matching SNI out of all + available certificates for any TLS handshake. + + Support: Core + properties: + certificateRefs: + description: |- + CertificateRefs contains a series of references to Kubernetes objects that + contains TLS certificates and private keys. These certificates are used to + establish a TLS handshake for requests that match the hostname of the + associated listener. + + A single CertificateRef to a Kubernetes Secret has "Core" support. + Implementations MAY choose to support attaching multiple certificates to + a Listener, but this behavior is implementation-specific. + + References to a resource in different namespace are invalid UNLESS there + is a ReferenceGrant in the target namespace that allows the certificate + to be attached. If a ReferenceGrant does not allow this reference, the + "ResolvedRefs" condition MUST be set to False for this listener with the + "RefNotPermitted" reason. + + This field is required to have at least one element when the mode is set + to "Terminate" (default) and is optional otherwise. + + CertificateRefs can reference to standard Kubernetes resources, i.e. + Secret, or implementation-specific custom resources. + + Support: Core - A single reference to a Kubernetes Secret of type kubernetes.io/tls + + Support: Implementation-specific (More than one reference or other resource types) + items: + description: |- + SecretObjectReference identifies an API object including its namespace, + defaulting to Secret. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + + References to objects with invalid Group and Kind are not valid, and must + be rejected by the implementation, with appropriate Conditions set + on the containing object. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-type: atomic + mode: + default: Terminate + description: |- + Mode defines the TLS behavior for the TLS session initiated by the client. + There are two possible modes: + + - Terminate: The TLS session between the downstream client and the + Gateway is terminated at the Gateway. This mode requires certificates + to be specified in some way, such as populating the certificateRefs + field. + - Passthrough: The TLS session is NOT terminated by the Gateway. This + implies that the Gateway can't decipher the TLS stream except for + the ClientHello message of the TLS protocol. The certificateRefs field + is ignored in this mode. + + Support: Core + enum: + - Terminate + - Passthrough + type: string + options: + additionalProperties: + description: |- + AnnotationValue is the value of an annotation in Gateway API. This is used + for validation of maps such as TLS options. This roughly matches Kubernetes + annotation validation, although the length validation in that case is based + on the entire size of the annotations struct. + maxLength: 4096 + minLength: 0 + type: string + description: |- + Options are a list of key/value pairs to enable extended TLS + configuration for each implementation. For example, configuring the + minimum TLS version or supported cipher suites. + + A set of common keys MAY be defined by the API in the future. To avoid + any ambiguity, implementation-specific definitions MUST use + domain-prefixed names, such as `example.com/my-custom-option`. + Un-prefixed names are reserved for key names defined by Gateway API. + + Support: Implementation-specific + maxProperties: 16 + type: object + type: object + x-kubernetes-validations: + - message: certificateRefs or options must be specified when + mode is Terminate + rule: 'self.mode == ''Terminate'' ? size(self.certificateRefs) + > 0 || size(self.options) > 0 : true' + required: + - name + - port + - protocol + type: object + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: tls must not be specified for protocols ['HTTP', 'TCP', + 'UDP'] + rule: 'self.all(l, l.protocol in [''HTTP'', ''TCP'', ''UDP''] ? + !has(l.tls) : true)' + - message: tls mode must be Terminate for protocol HTTPS + rule: 'self.all(l, (l.protocol == ''HTTPS'' && has(l.tls)) ? (l.tls.mode + == '''' || l.tls.mode == ''Terminate'') : true)' + - message: tls mode must be set for protocol TLS + rule: 'self.all(l, (l.protocol == ''TLS'' ? has(l.tls) && has(l.tls.mode) + && l.tls.mode != '''' : true))' + - message: hostname must not be specified for protocols ['TCP', 'UDP'] + rule: 'self.all(l, l.protocol in [''TCP'', ''UDP''] ? (!has(l.hostname) + || l.hostname == '''') : true)' + - message: Listener name must be unique within the Gateway + rule: self.all(l1, self.exists_one(l2, l1.name == l2.name)) + - message: Combination of port, protocol and hostname must be unique + for each listener + rule: 'self.all(l1, self.exists_one(l2, l1.port == l2.port && l1.protocol + == l2.protocol && (has(l1.hostname) && has(l2.hostname) ? l1.hostname + == l2.hostname : !has(l1.hostname) && !has(l2.hostname))))' + tls: + description: |- + TLS specifies frontend and backend tls configuration for entire gateway. + + Support: Extended + properties: + backend: + description: |- + Backend describes TLS configuration for gateway when connecting + to backends. + + Note that this contains only details for the Gateway as a TLS client, + and does _not_ imply behavior about how to choose which backend should + get a TLS connection. That is determined by the presence of a BackendTLSPolicy. + + Support: Core + properties: + clientCertificateRef: + description: |- + ClientCertificateRef references an object that contains a client certificate + and its associated private key. It can reference standard Kubernetes resources, + i.e., Secret, or implementation-specific custom resources. + + A ClientCertificateRef is considered invalid if: + + * It refers to a resource that cannot be resolved (e.g., the referenced resource + does not exist) or is misconfigured (e.g., a Secret does not contain the keys + named `tls.crt` and `tls.key`). In this case, the `ResolvedRefs` condition + on the Gateway MUST be set to False with the Reason `InvalidClientCertificateRef` + and the Message of the Condition MUST indicate why the reference is invalid. + + * It refers to a resource in another namespace UNLESS there is a ReferenceGrant + in the target namespace that allows the certificate to be attached. + If a ReferenceGrant does not allow this reference, the `ResolvedRefs` condition + on the Gateway MUST be set to False with the Reason `RefNotPermitted`. + + Implementations MAY choose to perform further validation of the certificate + content (e.g., checking expiry or enforcing specific formats). In such cases, + an implementation-specific Reason and Message MUST be set. + + Support: Core - Reference to a Kubernetes TLS Secret (with the type `kubernetes.io/tls`). + Support: Implementation-specific - Other resource kinds or Secrets with a + different type (e.g., `Opaque`). + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + type: object + frontend: + description: |- + Frontend describes TLS config when client connects to Gateway. + Support: Core + properties: + default: + description: |- + Default specifies the default client certificate validation configuration + for all Listeners handling HTTPS traffic, unless a per-port configuration + is defined. + + support: Core + properties: + validation: + description: |- + Validation holds configuration information for validating the frontend (client). + Setting this field will result in mutual authentication when connecting to the gateway. + In browsers this may result in a dialog appearing + that requests a user to specify the client certificate. + The maximum depth of a certificate chain accepted in verification is Implementation specific. + + Support: Core + properties: + caCertificateRefs: + description: |- + CACertificateRefs contains one or more references to Kubernetes + objects that contain a PEM-encoded TLS CA certificate bundle, which + is used as a trust anchor to validate the certificates presented by + the client. + + A CACertificateRef is invalid if: + + * It refers to a resource that cannot be resolved (e.g., the + referenced resource does not exist) or is misconfigured (e.g., a + ConfigMap does not contain a key named `ca.crt`). In this case, the + Reason on all matching HTTPS listeners must be set to `InvalidCACertificateRef` + and the Message of the Condition must indicate which reference is invalid and why. + + * It refers to an unknown or unsupported kind of resource. In this + case, the Reason on all matching HTTPS listeners must be set to + `InvalidCACertificateKind` and the Message of the Condition must explain + which kind of resource is unknown or unsupported. + + * It refers to a resource in another namespace UNLESS there is a + ReferenceGrant in the target namespace that allows the CA + certificate to be attached. If a ReferenceGrant does not allow this + reference, the `ResolvedRefs` on all matching HTTPS listeners condition + MUST be set with the Reason `RefNotPermitted`. + + Implementations MAY choose to perform further validation of the + certificate content (e.g., checking expiry or enforcing specific formats). + In such cases, an implementation-specific Reason and Message MUST be set. + + In all cases, the implementation MUST ensure that the `ResolvedRefs` + condition is set to `status: False` on all targeted listeners (i.e., + listeners serving HTTPS on a matching port). The condition MUST + include a Reason and Message that indicate the cause of the error. If + ALL CACertificateRefs are invalid, the implementation MUST also ensure + the `Accepted` condition on the listener is set to `status: False`, with + the Reason `NoValidCACertificate`. + Implementations MAY choose to support attaching multiple CA certificates + to a listener, but this behavior is implementation-specific. + + Support: Core - A single reference to a Kubernetes ConfigMap, with the + CA certificate in a key named `ca.crt`. + + Support: Implementation-specific - More than one reference, other kinds + of resources, or a single reference that includes multiple certificates. + items: + description: |- + ObjectReference identifies an API object including its namespace. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + + References to objects with invalid Group and Kind are not valid, and must + be rejected by the implementation, with appropriate Conditions set + on the containing object. + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When set to the empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. For + example "ConfigMap" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + mode: + default: AllowValidOnly + description: |- + FrontendValidationMode defines the mode for validating the client certificate. + There are two possible modes: + + - AllowValidOnly: In this mode, the gateway will accept connections only if + the client presents a valid certificate. This certificate must successfully + pass validation against the CA certificates specified in `CACertificateRefs`. + - AllowInsecureFallback: In this mode, the gateway will accept connections + even if the client certificate is not presented or fails verification. + + This approach delegates client authorization to the backend and introduce + a significant security risk. It should be used in testing environments or + on a temporary basis in non-testing environments. + + Defaults to AllowValidOnly. + + Support: Core + enum: + - AllowValidOnly + - AllowInsecureFallback + type: string + required: + - caCertificateRefs + type: object + type: object + perPort: + description: |- + PerPort specifies tls configuration assigned per port. + Per port configuration is optional. Once set this configuration overrides + the default configuration for all Listeners handling HTTPS traffic + that match this port. + Each override port requires a unique TLS configuration. + + support: Core + items: + properties: + port: + description: |- + The Port indicates the Port Number to which the TLS configuration will be + applied. This configuration will be applied to all Listeners handling HTTPS + traffic that match this port. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + tls: + description: |- + TLS store the configuration that will be applied to all Listeners handling + HTTPS traffic and matching given port. + + Support: Core + properties: + validation: + description: |- + Validation holds configuration information for validating the frontend (client). + Setting this field will result in mutual authentication when connecting to the gateway. + In browsers this may result in a dialog appearing + that requests a user to specify the client certificate. + The maximum depth of a certificate chain accepted in verification is Implementation specific. + + Support: Core + properties: + caCertificateRefs: + description: |- + CACertificateRefs contains one or more references to Kubernetes + objects that contain a PEM-encoded TLS CA certificate bundle, which + is used as a trust anchor to validate the certificates presented by + the client. + + A CACertificateRef is invalid if: + + * It refers to a resource that cannot be resolved (e.g., the + referenced resource does not exist) or is misconfigured (e.g., a + ConfigMap does not contain a key named `ca.crt`). In this case, the + Reason on all matching HTTPS listeners must be set to `InvalidCACertificateRef` + and the Message of the Condition must indicate which reference is invalid and why. + + * It refers to an unknown or unsupported kind of resource. In this + case, the Reason on all matching HTTPS listeners must be set to + `InvalidCACertificateKind` and the Message of the Condition must explain + which kind of resource is unknown or unsupported. + + * It refers to a resource in another namespace UNLESS there is a + ReferenceGrant in the target namespace that allows the CA + certificate to be attached. If a ReferenceGrant does not allow this + reference, the `ResolvedRefs` on all matching HTTPS listeners condition + MUST be set with the Reason `RefNotPermitted`. + + Implementations MAY choose to perform further validation of the + certificate content (e.g., checking expiry or enforcing specific formats). + In such cases, an implementation-specific Reason and Message MUST be set. + + In all cases, the implementation MUST ensure that the `ResolvedRefs` + condition is set to `status: False` on all targeted listeners (i.e., + listeners serving HTTPS on a matching port). The condition MUST + include a Reason and Message that indicate the cause of the error. If + ALL CACertificateRefs are invalid, the implementation MUST also ensure + the `Accepted` condition on the listener is set to `status: False`, with + the Reason `NoValidCACertificate`. + Implementations MAY choose to support attaching multiple CA certificates + to a listener, but this behavior is implementation-specific. + + Support: Core - A single reference to a Kubernetes ConfigMap, with the + CA certificate in a key named `ca.crt`. + + Support: Implementation-specific - More than one reference, other kinds + of resources, or a single reference that includes multiple certificates. + items: + description: |- + ObjectReference identifies an API object including its namespace. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + + References to objects with invalid Group and Kind are not valid, and must + be rejected by the implementation, with appropriate Conditions set + on the containing object. + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When set to the empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. + For example "ConfigMap" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + mode: + default: AllowValidOnly + description: |- + FrontendValidationMode defines the mode for validating the client certificate. + There are two possible modes: + + - AllowValidOnly: In this mode, the gateway will accept connections only if + the client presents a valid certificate. This certificate must successfully + pass validation against the CA certificates specified in `CACertificateRefs`. + - AllowInsecureFallback: In this mode, the gateway will accept connections + even if the client certificate is not presented or fails verification. + + This approach delegates client authorization to the backend and introduce + a significant security risk. It should be used in testing environments or + on a temporary basis in non-testing environments. + + Defaults to AllowValidOnly. + + Support: Core + enum: + - AllowValidOnly + - AllowInsecureFallback + type: string + required: + - caCertificateRefs + type: object + type: object + required: + - port + - tls + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: Port for TLS configuration must be unique within + the Gateway + rule: self.all(t1, self.exists_one(t2, t1.port == t2.port)) + required: + - default + type: object + type: object + required: + - gatewayClassName + - listeners + type: object + status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Accepted + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: Status defines the current state of Gateway. + properties: + addresses: + description: |- + Addresses lists the network addresses that have been bound to the + Gateway. + + This list may differ from the addresses provided in the spec under some + conditions: + + * no addresses are specified, all addresses are dynamically assigned + * a combination of specified and dynamic addresses are assigned + * a specified address was unusable (e.g. already in use) + items: + description: GatewayStatusAddress describes a network address that + is bound to a Gateway. + oneOf: + - properties: + type: + enum: + - IPAddress + value: + anyOf: + - format: ipv4 + - format: ipv6 + - properties: + type: + not: + enum: + - IPAddress + properties: + type: + default: IPAddress + description: Type of the address. + maxLength: 253 + minLength: 1 + pattern: ^Hostname|IPAddress|NamedAddress|[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + value: + description: |- + Value of the address. The validity of the values will depend + on the type and support by the controller. + + Examples: `1.2.3.4`, `128::1`, `my-ip-address`. + maxLength: 253 + minLength: 1 + type: string + required: + - value + type: object + x-kubernetes-validations: + - message: Hostname value must only contain valid characters (matching + ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$) + rule: 'self.type == ''Hostname'' ? self.value.matches(r"""^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"""): + true' + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + attachedListenerSets: + description: |- + AttachedListenerSets represents the total number of ListenerSets that have been + successfully attached to this Gateway. + + A ListenerSet is successfully attached to a Gateway when all the following conditions are met: + - The ListenerSet is selected by the Gateway's AllowedListeners field + - The ListenerSet has a valid ParentRef selecting the Gateway + - The ListenerSet's status has the condition "Accepted: true" + + Uses for this field include troubleshooting AttachedListenerSets attachment and + measuring blast radius/impact of changes to a Gateway. + format: int32 + type: integer + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Accepted + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: |- + Conditions describe the current conditions of the Gateway. + + Implementations should prefer to express Gateway conditions + using the `GatewayConditionType` and `GatewayConditionReason` + constants so that operators and tools can converge on a common + vocabulary to describe Gateway state. + + Known condition types are: + + * "Accepted" + * "Programmed" + * "Ready" + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + listeners: + description: Listeners provide status for each unique listener port + defined in the Spec. + items: + description: ListenerStatus is the status associated with a Listener. + properties: + attachedRoutes: + description: |- + AttachedRoutes represents the total number of Routes that have been + successfully attached to this Listener. + + Successful attachment of a Route to a Listener is based solely on the + combination of the AllowedRoutes field on the corresponding Listener + and the Route's ParentRefs field. A Route is successfully attached to + a Listener when it is selected by the Listener's AllowedRoutes field + AND the Route has a valid ParentRef selecting the whole Gateway + resource or a specific Listener as a parent resource (more detail on + attachment semantics can be found in the documentation on the various + Route kinds ParentRefs fields). Listener or Route status does not impact + successful attachment, i.e. the AttachedRoutes field count MUST be set + for Listeners, even if the Accepted condition of an individual Listener is set + to "False". The AttachedRoutes number represents the number of Routes with + the Accepted condition set to "True" that have been attached to this Listener. + Routes with any other value for the Accepted condition MUST NOT be included + in this count. + + Uses for this field include troubleshooting Route attachment and + measuring blast radius/impact of changes to a Listener. + format: int32 + type: integer + conditions: + description: Conditions describe the current condition of this + listener. + items: + description: Condition contains details for one aspect of + the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + name: + description: Name is the name of the Listener that this status + corresponds to. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + supportedKinds: + description: |- + SupportedKinds is the list indicating the Kinds supported by this + listener. This MUST represent the kinds supported by an implementation for + that Listener configuration. + + If kinds are specified in Spec that are not supported, they MUST NOT + appear in this list and an implementation MUST set the "ResolvedRefs" + condition to "False" with the "InvalidRouteKinds" reason. If both valid + and invalid Route kinds are specified, the implementation MUST + reference the valid Route kinds that have been specified. + items: + description: RouteGroupKind indicates the group and kind of + a Route resource. + properties: + group: + default: gateway.networking.k8s.io + description: Group is the group of the Route. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is the kind of the Route. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + required: + - kind + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + required: + - attachedRoutes + - conditions + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/config/test/crd/gateway-api/gateway.networking.k8s.io_tcproutes.yaml b/config/test/crd/gateway-api/gateway.networking.k8s.io_tcproutes.yaml new file mode 100644 index 000000000..88211eff0 --- /dev/null +++ b/config/test/crd/gateway-api/gateway.networking.k8s.io_tcproutes.yaml @@ -0,0 +1,756 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 + gateway.networking.k8s.io/bundle-version: v1.5.0 + gateway.networking.k8s.io/channel: experimental + name: tcproutes.gateway.networking.k8s.io +spec: + group: gateway.networking.k8s.io + names: + categories: + - gateway-api + kind: TCPRoute + listKind: TCPRouteList + plural: tcproutes + singular: tcproute + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha2 + schema: + openAPIV3Schema: + description: |- + TCPRoute provides a way to route TCP requests. When combined with a Gateway + listener, it can be used to forward connections on the port specified by the + listener to a set of backends specified by the TCPRoute. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of TCPRoute. + properties: + parentRefs: + description: |- + ParentRefs references the resources (usually Gateways) that a Route wants + to be attached to. Note that the referenced parent resource needs to + allow this for the attachment to be complete. For Gateways, that means + the Gateway needs to allow attachment from Routes of this kind and + namespace. For Services, that means the Service must either be in the same + namespace for a "producer" route, or the mesh implementation must support + and allow "consumer" routes for the referenced Service. ReferenceGrant is + not applicable for governing ParentRefs to Services - it is not possible to + create a "producer" route for a Service in a different namespace from the + Route. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + ParentRefs must be _distinct_. This means either that: + + * They select different objects. If this is the case, then parentRef + entries are distinct. In terms of fields, this means that the + multi-part key defined by `group`, `kind`, `namespace`, and `name` must + be unique across all parentRef entries in the Route. + * They do not select different objects, but for each optional field used, + each ParentRef that selects the same object must set the same set of + optional fields to different values. If one ParentRef sets a + combination of optional fields, all must set the same combination. + + Some examples: + + * If one ParentRef sets `sectionName`, all ParentRefs referencing the + same object must also set `sectionName`. + * If one ParentRef sets `port`, all ParentRefs referencing the same + object must also set `port`. + * If one ParentRef sets `sectionName` and `port`, all ParentRefs + referencing the same object must also set `sectionName` and `port`. + + It is possible to separately reference multiple distinct objects that may + be collapsed by an implementation. For example, some implementations may + choose to merge compatible Gateway Listeners together. If that is the + case, the list of routes attached to those resources should also be + merged. + + Note that for ParentRefs that cross namespace boundaries, there are specific + rules. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example, + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable other kinds of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + items: + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + maxItems: 32 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: sectionName or port must be specified when parentRefs includes + 2 or more references to the same parent + rule: 'self.all(p1, self.all(p2, p1.group == p2.group && p1.kind + == p2.kind && p1.name == p2.name && (((!has(p1.__namespace__) + || p1.__namespace__ == '''') && (!has(p2.__namespace__) || p2.__namespace__ + == '''')) || (has(p1.__namespace__) && has(p2.__namespace__) && + p1.__namespace__ == p2.__namespace__)) ? ((!has(p1.sectionName) + || p1.sectionName == '''') == (!has(p2.sectionName) || p2.sectionName + == '''') && (!has(p1.port) || p1.port == 0) == (!has(p2.port) + || p2.port == 0)): true))' + - message: sectionName or port must be unique when parentRefs includes + 2 or more references to the same parent + rule: self.all(p1, self.exists_one(p2, p1.group == p2.group && p1.kind + == p2.kind && p1.name == p2.name && (((!has(p1.__namespace__) + || p1.__namespace__ == '') && (!has(p2.__namespace__) || p2.__namespace__ + == '')) || (has(p1.__namespace__) && has(p2.__namespace__) && + p1.__namespace__ == p2.__namespace__ )) && (((!has(p1.sectionName) + || p1.sectionName == '') && (!has(p2.sectionName) || p2.sectionName + == '')) || ( has(p1.sectionName) && has(p2.sectionName) && p1.sectionName + == p2.sectionName)) && (((!has(p1.port) || p1.port == 0) && (!has(p2.port) + || p2.port == 0)) || (has(p1.port) && has(p2.port) && p1.port + == p2.port)))) + rules: + description: Rules are a list of TCP matchers and actions. + items: + description: TCPRouteRule is the configuration for a given rule. + properties: + backendRefs: + description: |- + BackendRefs defines the backend(s) where matching requests should be + sent. If unspecified or invalid (refers to a nonexistent resource or a + Service with no endpoints), the underlying implementation MUST actively + reject connection attempts to this backend. Connection rejections must + respect weight; if an invalid backend is requested to have 80% of + connections, then 80% of connections must be rejected instead. + + Support: Core for Kubernetes Service + + Support: Extended for Kubernetes ServiceImport + + Support: Implementation-specific for any other resource + + Support for weight: Extended + items: + description: |- + BackendRef defines how a Route should forward a request to a Kubernetes + resource. + + Note that when a namespace different than the local namespace is specified, a + ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + + When the BackendRef points to a Kubernetes Service, implementations SHOULD + honor the appProtocol field if it is set for the target Service Port. + + Implementations supporting appProtocol SHOULD recognize the Kubernetes + Standard Application Protocols defined in KEP-3726. + + If a Service appProtocol isn't specified, an implementation MAY infer the + backend protocol through its own means. Implementations MAY infer the + protocol from the Route type referring to the backend Service. + + If a Route is not able to send traffic to the backend using the specified + protocol then the backend is considered invalid. Implementations MUST set the + "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. + + + Note that when the BackendTLSPolicy object is enabled by the implementation, + there are some extra rules about validity to consider here. See the fields + where this struct is used for more information about the exact behavior. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + Support: Core (Services with a type other than ExternalName) + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + weight: + default: 1 + description: |- + Weight specifies the proportion of requests forwarded to the referenced + backend. This is computed as weight/(sum of all weights in this + BackendRefs list). For non-zero values, there may be some epsilon from + the exact proportion defined here depending on the precision an + implementation supports. Weight is not a percentage and the sum of + weights does not need to equal 100. + + If only one backend is specified and it has a weight greater than 0, 100% + of the traffic is forwarded to that backend. If weight is set to 0, no + traffic should be forwarded for this entry. If unspecified, weight + defaults to 1. + + Support for this field varies based on the context where used. + format: int32 + maximum: 1000000 + minimum: 0 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') + ? has(self.port) : true' + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + name: + description: |- + Name is the name of the route rule. This name MUST be unique within a Route if it is set. + + Support: Extended + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - backendRefs + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: Rule name must be unique within the route + rule: self.all(l1, !has(l1.name) || self.exists_one(l2, has(l2.name) + && l1.name == l2.name)) + useDefaultGateways: + description: |- + UseDefaultGateways indicates the default Gateway scope to use for this + Route. If unset (the default) or set to None, the Route will not be + attached to any default Gateway; if set, it will be attached to any + default Gateway supporting the named scope, subject to the usual rules + about which Routes a Gateway is allowed to claim. + + Think carefully before using this functionality! The set of default + Gateways supporting the requested scope can change over time without + any notice to the Route author, and in many situations it will not be + appropriate to request a default Gateway for a given Route -- for + example, a Route with specific security requirements should almost + certainly not use a default Gateway. + enum: + - All + - None + type: string + required: + - rules + type: object + status: + description: Status defines the current state of TCPRoute. + properties: + parents: + description: |- + Parents is a list of parent resources (usually Gateways) that are + associated with the route, and the status of the route with respect to + each parent. When this route attaches to a parent, the controller that + manages the parent must add an entry to this list when the controller + first sees the route and should update the entry as appropriate when the + route or gateway is modified. + + Note that parent references that cannot be resolved by an implementation + of this API will not be added to this list. Implementations of this API + can only populate Route status for the Gateways/parent resources they are + responsible for. + + A maximum of 32 Gateways will be represented in this list. An empty list + means the route has not been attached to any Gateway. + items: + description: |- + RouteParentStatus describes the status of a route with respect to an + associated Parent. + properties: + conditions: + description: |- + Conditions describes the status of the route with respect to the Gateway. + Note that the route's availability is also subject to the Gateway's own + status conditions and listener status. + + If the Route's ParentRef specifies an existing Gateway that supports + Routes of this kind AND that Gateway's controller has sufficient access, + then that Gateway's controller MUST set the "Accepted" condition on the + Route, to indicate whether the route has been accepted or rejected by the + Gateway, and why. + + A Route MUST be considered "Accepted" if at least one of the Route's + rules is implemented by the Gateway. + + There are a number of cases where the "Accepted" condition may not be set + due to lack of controller visibility, that includes when: + + * The Route refers to a nonexistent parent. + * The Route is of a type that the controller does not support. + * The Route is in a namespace to which the controller does not have access. + items: + description: Condition contains details for one aspect of + the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controllerName: + description: |- + ControllerName is a domain/path string that indicates the name of the + controller that wrote this status. This corresponds with the + controllerName field on GatewayClass. + + Example: "example.net/gateway-controller". + + The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are + valid Kubernetes names + (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). + + Controllers MUST populate this field when writing status. Controllers should ensure that + entries to status populated with their ControllerName are cleaned up when they are no + longer necessary. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + parentRef: + description: |- + ParentRef corresponds with a ParentRef in the spec that this + RouteParentStatus struct describes the status of. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + required: + - conditions + - controllerName + - parentRef + type: object + maxItems: 32 + type: array + x-kubernetes-list-type: atomic + required: + - parents + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/controllers/kafkacluster_controller.go b/controllers/kafkacluster_controller.go index 4b825d773..15d8b0601 100644 --- a/controllers/kafkacluster_controller.go +++ b/controllers/kafkacluster_controller.go @@ -48,6 +48,7 @@ import ( "github.com/banzaicloud/koperator/pkg/resources/cruisecontrol" "github.com/banzaicloud/koperator/pkg/resources/cruisecontrolmonitoring" "github.com/banzaicloud/koperator/pkg/resources/envoy" + "github.com/banzaicloud/koperator/pkg/resources/envoygateway" "github.com/banzaicloud/koperator/pkg/resources/kafka" "github.com/banzaicloud/koperator/pkg/resources/kafkamonitoring" "github.com/banzaicloud/koperator/pkg/resources/nodeportexternalaccess" @@ -88,6 +89,9 @@ type KafkaClusterReconciler struct { // +kubebuilder:rbac:groups=kafka.banzaicloud.io,resources=kafkaclusters/finalizers,verbs=create;update;patch;delete // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=projectcontour.io,resources=httpproxies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tlsroutes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tcproutes,verbs=get;list;watch;create;update;patch;delete func (r *KafkaClusterReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { log := logr.FromContextOrDiscard(ctx) @@ -122,6 +126,7 @@ func (r *KafkaClusterReconciler) Reconcile(ctx context.Context, request ctrl.Req envoy.New(r.Client, instance), nodeportexternalaccess.New(r.Client, instance), contouringress.New(r.Client, instance), + envoygateway.New(r.Client, instance), kafkamonitoring.New(r.Client, instance), cruisecontrolmonitoring.New(r.Client, instance), kafka.New(r.Client, r.DirectClient, instance, r.KafkaClientProvider), diff --git a/controllers/tests/kafkacluster_controller_envoygateway_test.go b/controllers/tests/kafkacluster_controller_envoygateway_test.go new file mode 100644 index 000000000..da1f3624f --- /dev/null +++ b/controllers/tests/kafkacluster_controller_envoygateway_test.go @@ -0,0 +1,173 @@ +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "context" + "fmt" + "sync/atomic" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/banzaicloud/koperator/api/v1beta1" +) + +var _ = Describe("KafkaClusterWithEnvoyGatewayIngressController", Label("envoygateway"), func() { + var ( + count uint64 = 0 + namespace string + namespaceObj *corev1.Namespace + kafkaCluster *v1beta1.KafkaCluster + ) + + BeforeEach(func() { + atomic.AddUint64(&count, 1) + namespace = fmt.Sprintf("kafkaenvoygatewaytest-%v", count) + namespaceObj = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + + kafkaCluster = createMinimalKafkaClusterCR(fmt.Sprintf("kafkacluster-%d", count), namespace) + kafkaCluster.Spec.IngressController = "envoygateway" + kafkaCluster.Spec.EnvoyGatewayConfig = v1beta1.EnvoyGatewayIngressConfig{ + GatewayClassName: "eg", + BrokerHostnameTemplate: "broker-%id.kafka.cluster.local", + } + + envoyGatewayListener := kafkaCluster.Spec.ListenersConfig.ExternalListeners[0] + envoyGatewayListener.AccessMethod = corev1.ServiceTypeLoadBalancer + envoyGatewayListener.ExternalStartingPort = 19090 + envoyGatewayListener.Type = "plaintext" + envoyGatewayListener.Name = "listener1" + + kafkaCluster.Spec.ListenersConfig.ExternalListeners[0] = envoyGatewayListener + }) + + JustBeforeEach(func(ctx SpecContext) { + By("creating namespace " + namespace) + err := k8sClient.Create(ctx, namespaceObj) + Expect(err).NotTo(HaveOccurred()) + + By("creating kafka cluster object " + kafkaCluster.Name + " in namespace " + namespace) + err = k8sClient.Create(ctx, kafkaCluster) + Expect(err).NotTo(HaveOccurred()) + + waitForClusterRunningState(ctx, kafkaCluster, namespace) + }) + + JustAfterEach(func(ctx SpecContext) { + By("deleting Kafka cluster object " + kafkaCluster.Name + " in namespace " + namespace) + err := k8sClient.Delete(ctx, kafkaCluster) + Expect(err).NotTo(HaveOccurred()) + + kafkaCluster = nil + }) + + When("configuring Envoy Gateway ingress with TCP routes", func() { + It("should reconcile Gateway and TCPRoute objects properly", func(ctx SpecContext) { + expectEnvoyGateway(ctx, kafkaCluster, "listener1") + expectEnvoyGatewayTCPRoutes(ctx, kafkaCluster, "listener1") + }) + }) +}) + +func expectEnvoyGatewayLabels(labels map[string]string, eListenerName, crName string) { + Expect(labels).To(HaveKeyWithValue(v1beta1.AppLabelKey, "envoygateway")) + Expect(labels).To(HaveKeyWithValue("eListenerName", eListenerName)) + Expect(labels).To(HaveKeyWithValue(v1beta1.KafkaCRLabelKey, crName)) +} + +func expectEnvoyGateway(ctx context.Context, kafkaCluster *v1beta1.KafkaCluster, eListenerName string) { + var gateway gatewayv1.Gateway + gatewayName := fmt.Sprintf("kafka-gateway-%s", eListenerName) + Eventually(ctx, func() error { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: kafkaCluster.Namespace, Name: gatewayName}, &gateway) + return err + }).Should(Succeed()) + + expectEnvoyGatewayLabels(gateway.Labels, eListenerName, kafkaCluster.Name) + Expect(string(gateway.Spec.GatewayClassName)).To(Equal("eg")) + + // Check listeners + if kafkaCluster.Spec.KRaftMode { + // 2 brokers + 1 anycast = 3 listeners + Expect(gateway.Spec.Listeners).To(HaveLen(3)) + } else { + // 3 brokers + 1 anycast = 4 listeners + Expect(gateway.Spec.Listeners).To(HaveLen(4)) + } + + // Verify broker listeners + brokerCount := len(kafkaCluster.Spec.Brokers) + for i := 0; i < brokerCount; i++ { + listener := gateway.Spec.Listeners[i] + Expect(string(listener.Name)).To(Equal(fmt.Sprintf("broker-%d", kafkaCluster.Spec.Brokers[i].Id))) + Expect(listener.Port).To(BeEquivalentTo(19090 + kafkaCluster.Spec.Brokers[i].Id)) + Expect(listener.Protocol).To(Equal(gatewayv1.TCPProtocolType)) + } + + // Verify anycast listener + anycastListener := gateway.Spec.Listeners[brokerCount] + Expect(string(anycastListener.Name)).To(Equal("anycast")) + // Anycast listener should use the default anycast port (29092), not ExternalStartingPort + Expect(anycastListener.Port).To(BeEquivalentTo(29092)) + Expect(anycastListener.Protocol).To(Equal(gatewayv1.TCPProtocolType)) +} + +func expectEnvoyGatewayTCPRoutes(ctx context.Context, kafkaCluster *v1beta1.KafkaCluster, eListenerName string) { + brokerCount := len(kafkaCluster.Spec.Brokers) + + // Check TCPRoute for each broker + for i := 0; i < brokerCount; i++ { + var tcpRoute gatewayv1alpha2.TCPRoute + tcpRouteName := fmt.Sprintf("kafka-tcproute-%s-%d", eListenerName, kafkaCluster.Spec.Brokers[i].Id) + Eventually(ctx, func() error { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: kafkaCluster.Namespace, Name: tcpRouteName}, &tcpRoute) + return err + }).Should(Succeed()) + + expectEnvoyGatewayLabels(tcpRoute.Labels, eListenerName, kafkaCluster.Name) + + // Verify parent reference + Expect(tcpRoute.Spec.ParentRefs).To(HaveLen(1)) + Expect(string(tcpRoute.Spec.ParentRefs[0].Name)).To(Equal(fmt.Sprintf("kafka-gateway-%s", eListenerName))) + Expect(string(*tcpRoute.Spec.ParentRefs[0].SectionName)).To(Equal(fmt.Sprintf("broker-%d", kafkaCluster.Spec.Brokers[i].Id))) + + // Verify backend reference + Expect(tcpRoute.Spec.Rules).To(HaveLen(1)) + Expect(tcpRoute.Spec.Rules[0].BackendRefs).To(HaveLen(1)) + Expect(string(tcpRoute.Spec.Rules[0].BackendRefs[0].Name)).To(Equal(fmt.Sprintf("%s-all-broker", kafkaCluster.Name))) + } + + // Check anycast TCPRoute + var anycastTCPRoute gatewayv1alpha2.TCPRoute + anycastTCPRouteName := fmt.Sprintf("kafka-tcproute-%s-anycast", eListenerName) + Eventually(ctx, func() error { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: kafkaCluster.Namespace, Name: anycastTCPRouteName}, &anycastTCPRoute) + return err + }).Should(Succeed()) + + expectEnvoyGatewayLabels(anycastTCPRoute.Labels, eListenerName, kafkaCluster.Name) + Expect(anycastTCPRoute.Spec.ParentRefs).To(HaveLen(1)) + Expect(string(*anycastTCPRoute.Spec.ParentRefs[0].SectionName)).To(Equal("anycast")) +} diff --git a/controllers/tests/suite_test.go b/controllers/tests/suite_test.go index 10ecaa739..e91cfeff1 100644 --- a/controllers/tests/suite_test.go +++ b/controllers/tests/suite_test.go @@ -55,6 +55,8 @@ import ( cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" contour "github.com/projectcontour/contour/apis/projectcontour/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" banzaicloudv1alpha1 "github.com/banzaicloud/koperator/api/v1alpha1" banzaicloudv1beta1 "github.com/banzaicloud/koperator/api/v1beta1" @@ -92,6 +94,7 @@ var _ = BeforeSuite(func(ctx SpecContext) { filepath.Join("..", "..", "config", "base", "crds"), filepath.Join("..", "..", "config", "test", "crd", "cert-manager"), filepath.Join("..", "..", "config", "test", "crd", "projectcontour"), + filepath.Join("..", "..", "config", "test", "crd", "gateway-api"), }, ControlPlaneStartTimeout: timeout, ControlPlaneStopTimeout: timeout, @@ -122,6 +125,8 @@ var _ = BeforeSuite(func(ctx SpecContext) { Expect(banzaicloudv1alpha1.AddToScheme(scheme)).To(Succeed()) Expect(banzaicloudv1beta1.AddToScheme(scheme)).To(Succeed()) Expect(contour.AddToScheme(scheme)).To(Succeed()) + Expect(gatewayv1.Install(scheme)).To(Succeed()) + Expect(gatewayv1alpha2.Install(scheme)).To(Succeed()) // +kubebuilder:scaffold:scheme diff --git a/go.mod b/go.mod index b1e0c7217..cfb708bf8 100644 --- a/go.mod +++ b/go.mod @@ -133,7 +133,7 @@ require ( k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 // indirect - sigs.k8s.io/gateway-api v1.6.0 // indirect + sigs.k8s.io/gateway-api v1.6.0 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/yaml v1.6.0 ) diff --git a/main.go b/main.go index 0a62f715a..7f90b5d01 100644 --- a/main.go +++ b/main.go @@ -47,6 +47,8 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" contour "github.com/projectcontour/contour/apis/projectcontour/v1" @@ -74,6 +76,9 @@ func init() { _ = banzaicloudv1beta1.AddToScheme(scheme) _ = contour.AddToScheme(scheme) + + _ = gatewayv1.Install(scheme) + _ = gatewayv1alpha2.Install(scheme) // +kubebuilder:scaffold:scheme } diff --git a/pkg/k8sutil/resource_test.go b/pkg/k8sutil/resource_test.go new file mode 100644 index 000000000..6206c545e --- /dev/null +++ b/pkg/k8sutil/resource_test.go @@ -0,0 +1,277 @@ +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8sutil + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/banzaicloud/koperator/api/v1beta1" +) + +// MockClient is a mock implementation of client.Client +type MockClient struct { + mock.Mock +} + +func (m *MockClient) Get(ctx context.Context, key types.NamespacedName, obj client.Object, opts ...client.GetOption) error { + args := m.Called(ctx, key, obj, opts) + return args.Error(0) +} + +func (m *MockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + args := m.Called(ctx, list, opts) + return args.Error(0) +} + +func (m *MockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + args := m.Called(ctx, obj, patch, opts) + return args.Error(0) +} + +func (m *MockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Status() client.StatusWriter { + args := m.Called() + return args.Get(0).(client.StatusWriter) +} + +func (m *MockClient) Scheme() *runtime.Scheme { + args := m.Called() + return args.Get(0).(*runtime.Scheme) +} + +func (m *MockClient) RESTMapper() meta.RESTMapper { + args := m.Called() + return args.Get(0).(meta.RESTMapper) +} + +func (m *MockClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + args := m.Called(obj) + return args.Get(0).(schema.GroupVersionKind), args.Error(1) +} + +func (m *MockClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + args := m.Called(obj) + return args.Bool(0), args.Error(1) +} + +func (m *MockClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) SubResource(subResource string) client.SubResourceClient { + args := m.Called(subResource) + return args.Get(0).(client.SubResourceClient) +} + +func TestReconcile_CreateResource(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + } + + // Create a test ConfigMap + desired := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "key": "value", + }, + } + + // Mock Get to return NotFound error (resource doesn't exist) + mockClient.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.NewNotFound(schema.GroupResource{}, "test-configmap")) + // Mock Create to succeed + mockClient.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + err := Reconcile(logr.Discard(), mockClient, desired, cluster) + assert.NoError(t, err) + + // Verify that Create was called + mockClient.AssertCalled(t, "Create", mock.Anything, mock.Anything, mock.Anything) +} + +func TestReconcile_UpdateResource(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + } + + // Create a test ConfigMap + desired := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "key": "value", + }, + } + + // Mock Get to return existing resource + existing := desired.DeepCopy() + existing.Data["existing-key"] = "existing-value" + mockClient.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + obj := args.Get(2).(client.Object) + // Set the existing data + if cm, ok := obj.(*corev1.ConfigMap); ok { + cm.Data = map[string]string{ + "existing-key": "existing-value", + } + } + }) + // Mock Update to succeed + mockClient.On("Update", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + err := Reconcile(logr.Discard(), mockClient, desired, cluster) + assert.NoError(t, err) +} + +func TestReconcile_ErrorHandling(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + } + + // Create a test ConfigMap + desired := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "key": "value", + }, + } + + // Mock Get to return an error + mockClient.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(assert.AnError) + + err := Reconcile(logr.Discard(), mockClient, desired, cluster) + assert.Error(t, err) +} + +func TestReconcile_CreateErrorHandling(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + } + + // Create a test ConfigMap + desired := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "key": "value", + }, + } + + // Mock Get to return NotFound error (resource doesn't exist) + mockClient.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(assert.AnError) + // Mock Create to return an error + mockClient.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(assert.AnError) + + err := Reconcile(logr.Discard(), mockClient, desired, cluster) + assert.Error(t, err) +} + +func TestReconcile_UpdateErrorHandling(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + } + + // Create a test ConfigMap + desired := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "key": "value", + }, + } + + // Mock Get to return a different existing resource (so update is needed) + existing := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "key": "different-value", + }, + } + mockClient.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + obj := args.Get(2).(*corev1.ConfigMap) + *obj = *existing + }) + // Mock Update to return an error + mockClient.On("Update", mock.Anything, mock.Anything, mock.Anything).Return(assert.AnError) + + err := Reconcile(logr.Discard(), mockClient, desired, cluster) + assert.Error(t, err) +} diff --git a/pkg/resources/envoy/deployment.go b/pkg/resources/envoy/deployment.go index eceebce86..dd1df8be9 100644 --- a/pkg/resources/envoy/deployment.go +++ b/pkg/resources/envoy/deployment.go @@ -118,7 +118,7 @@ func (r *Reconciler) deployment(log logr.Logger, extListener v1beta1.ExternalLis TopologySpreadConstraints: ingressConfig.EnvoyConfig.GetTopologySpreadConstaints(), Containers: []corev1.Container{ { - Name: "envoy", + Name: envoyContainerName, Image: ingressConfig.EnvoyConfig.GetEnvoyImage(), Args: arguments, Ports: append(exposedPorts, diff --git a/pkg/resources/envoy/envoy.go b/pkg/resources/envoy/envoy.go index 318205599..dd435a53b 100644 --- a/pkg/resources/envoy/envoy.go +++ b/pkg/resources/envoy/envoy.go @@ -38,6 +38,13 @@ import ( envoyutils "github.com/banzaicloud/koperator/pkg/util/envoy" ) +const ( + // envoyContainerName is the name of the envoy container and the "envoy" ingressController value. + envoyContainerName = "envoy" + // envoyIngressAppLabelValue is the v1beta1.AppLabelKey value for envoy ingress resources. + envoyIngressAppLabelValue = "envoyingress" +) + // labelsForEnvoyIngress returns the labels for selecting the resources // belonging to the given kafka CR name. func labelsForEnvoyIngress(crName, eLName string) map[string]string { @@ -45,7 +52,7 @@ func labelsForEnvoyIngress(crName, eLName string) map[string]string { } func labelsForEnvoyIngressWithoutEListenerName(crName string) map[string]string { - return map[string]string{v1beta1.AppLabelKey: "envoyingress", v1beta1.KafkaCRLabelKey: crName} + return map[string]string{v1beta1.AppLabelKey: envoyIngressAppLabelValue, v1beta1.KafkaCRLabelKey: crName} } // Reconciler implements the Component Reconciler diff --git a/pkg/resources/envoy/envoy_test.go b/pkg/resources/envoy/envoy_test.go new file mode 100644 index 000000000..f57b7f222 --- /dev/null +++ b/pkg/resources/envoy/envoy_test.go @@ -0,0 +1,302 @@ +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envoy + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/banzaicloud/koperator/api/v1beta1" + "github.com/banzaicloud/koperator/pkg/util" +) + +// MockClient is a mock implementation of client.Client +type MockClient struct { + mock.Mock +} + +func (m *MockClient) Get(ctx context.Context, key types.NamespacedName, obj client.Object, opts ...client.GetOption) error { + args := m.Called(ctx, key, obj, opts) + return args.Error(0) +} + +func (m *MockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + args := m.Called(ctx, list, opts) + return args.Error(0) +} + +func (m *MockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + args := m.Called(ctx, obj, patch, opts) + return args.Error(0) +} + +func (m *MockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Status() client.StatusWriter { + args := m.Called() + return args.Get(0).(client.StatusWriter) +} + +func (m *MockClient) Scheme() *runtime.Scheme { + args := m.Called() + return args.Get(0).(*runtime.Scheme) +} + +func (m *MockClient) RESTMapper() meta.RESTMapper { + args := m.Called() + return args.Get(0).(meta.RESTMapper) +} + +func (m *MockClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + args := m.Called(obj) + return args.Get(0).(schema.GroupVersionKind), args.Error(1) +} + +func (m *MockClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + args := m.Called(obj) + return args.Bool(0), args.Error(1) +} + +func (m *MockClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) SubResource(subResource string) client.SubResourceClient { + args := m.Called(subResource) + return args.Get(0).(client.SubResourceClient) +} + +func TestNew(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + } + + reconciler := New(mockClient, cluster) + + assert.NotNil(t, reconciler) + assert.Equal(t, mockClient, reconciler.Client) + assert.Equal(t, cluster, reconciler.KafkaCluster) +} + +func TestLabelsForEnvoyIngress(t *testing.T) { + tests := []struct { + name string + crName string + eLName string + expectedLabels map[string]string + }{ + { + name: "basic labels", + crName: "test-cluster", + eLName: "external", + expectedLabels: map[string]string{ + v1beta1.AppLabelKey: envoyIngressAppLabelValue, + v1beta1.KafkaCRLabelKey: "test-cluster", + util.ExternalListenerLabelNameKey: "external", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + labels := labelsForEnvoyIngress(tt.crName, tt.eLName) + assert.Equal(t, tt.expectedLabels, labels) + }) + } +} + +func TestLabelsForEnvoyIngressWithoutEListenerName(t *testing.T) { + tests := []struct { + name string + crName string + expectedLabels map[string]string + }{ + { + name: "basic labels without external listener", + crName: "test-cluster", + expectedLabels: map[string]string{ + v1beta1.AppLabelKey: envoyIngressAppLabelValue, + v1beta1.KafkaCRLabelKey: "test-cluster", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + labels := labelsForEnvoyIngressWithoutEListenerName(tt.crName) + assert.Equal(t, tt.expectedLabels, labels) + }) + } +} + +func TestReconcile_WithEnvoyIngressController(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + ListenersConfig: v1beta1.ListenersConfig{ + ExternalListeners: []v1beta1.ExternalListenerConfig{ + { + CommonListenerSpec: v1beta1.CommonListenerSpec{ + Name: "external", + ContainerPort: 9094, + }, + AccessMethod: corev1.ServiceTypeLoadBalancer, + ExternalStartingPort: 19090, + }, + }, + }, + IngressController: envoyContainerName, + }, + } + + reconciler := New(mockClient, cluster) + + // Mock the k8sutil.Reconcile calls + mockClient.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + mockClient.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(nil) + mockClient.On("Update", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + log := logr.Discard() + err := reconciler.Reconcile(log) + assert.NoError(t, err) +} + +func TestReconcile_WithRemoveUnusedIngressResources(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + ListenersConfig: v1beta1.ListenersConfig{ + ExternalListeners: []v1beta1.ExternalListenerConfig{ + { + CommonListenerSpec: v1beta1.CommonListenerSpec{ + Name: "external", + ContainerPort: 9094, + }, + AccessMethod: corev1.ServiceTypeNodePort, // Not LoadBalancer + ExternalStartingPort: 19090, + }, + }, + }, + IngressController: "nginx", // Not envoy + RemoveUnusedIngressResources: true, + }, + } + + reconciler := New(mockClient, cluster) + + // Mock the List call to return empty list (no resources to delete) + mockClient.On("List", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + log := logr.Discard() + err := reconciler.Reconcile(log) + assert.NoError(t, err) +} + +func TestReconcile_NoExternalListeners(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + ListenersConfig: v1beta1.ListenersConfig{ + ExternalListeners: nil, // No external listeners + }, + }, + } + + reconciler := New(mockClient, cluster) + + log := logr.Discard() + err := reconciler.Reconcile(log) + assert.NoError(t, err) +} + +func TestReconcile_ErrorHandling(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + ListenersConfig: v1beta1.ListenersConfig{ + ExternalListeners: []v1beta1.ExternalListenerConfig{ + { + CommonListenerSpec: v1beta1.CommonListenerSpec{ + Name: "external", + ContainerPort: 9094, + }, + AccessMethod: corev1.ServiceTypeLoadBalancer, + ExternalStartingPort: 19090, + }, + }, + }, + IngressController: envoyContainerName, + }, + } + + reconciler := New(mockClient, cluster) + + // Mock k8sutil.Reconcile to return an error + mockClient.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(assert.AnError) + + log := logr.Discard() + err := reconciler.Reconcile(log) + assert.Error(t, err) +} diff --git a/pkg/resources/envoygateway/envoygateway.go b/pkg/resources/envoygateway/envoygateway.go new file mode 100644 index 000000000..13f082c2a --- /dev/null +++ b/pkg/resources/envoygateway/envoygateway.go @@ -0,0 +1,170 @@ +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envoygateway + +import ( + "context" + "fmt" + "strings" + + "emperror.dev/errors" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + apiutil "github.com/banzaicloud/koperator/api/util" + "github.com/banzaicloud/koperator/api/v1beta1" + "github.com/banzaicloud/koperator/pkg/k8sutil" + "github.com/banzaicloud/koperator/pkg/resources" + "github.com/banzaicloud/koperator/pkg/util" + envoygatewayutils "github.com/banzaicloud/koperator/pkg/util/envoygateway" +) + +const ( + componentName = "envoygateway" +) + +// labelsForEnvoyGateway returns the labels for selecting the resources +// belonging to the given kafka CR name. +func labelsForEnvoyGateway(crName, eLName string) map[string]string { + return apiutil.MergeLabels(labelsForEnvoyGatewayWithoutEListenerName(crName), map[string]string{util.ExternalListenerLabelNameKey: eLName}) +} + +func labelsForEnvoyGatewayWithoutEListenerName(crName string) map[string]string { + return map[string]string{v1beta1.AppLabelKey: "envoygateway", v1beta1.KafkaCRLabelKey: crName} +} + +// Reconciler implements the Component Reconciler +type Reconciler struct { + resources.Reconciler +} + +// New creates a new reconciler for Envoy Gateway +func New(client client.Client, cluster *v1beta1.KafkaCluster) *Reconciler { + return &Reconciler{ + Reconciler: resources.Reconciler{ + Client: client, + KafkaCluster: cluster, + }, + } +} + +// Reconcile implements the reconcile logic for Envoy Gateway +func (r *Reconciler) Reconcile(log logr.Logger) error { + log = log.WithValues("component", componentName) + + log.V(1).Info("Reconciling") + for _, eListener := range r.KafkaCluster.Spec.ListenersConfig.ExternalListeners { + if r.KafkaCluster.Spec.GetIngressController() == envoygatewayutils.IngressControllerName && eListener.GetAccessMethod() == corev1.ServiceTypeLoadBalancer { + ingressConfigs, defaultControllerName, err := util.GetIngressConfigs(r.KafkaCluster.Spec, eListener) + if err != nil { + return err + } + + for name, ingressConfig := range ingressConfigs { + if !util.IsIngressConfigInUse(name, defaultControllerName, r.KafkaCluster, log) { + continue + } + + // Validate TLS configuration for envoygateway + // EnvoyGateway ONLY supports TLS termination at the gateway level + if eListener.TLSEnabled() { + if ingressConfig.EnvoyGatewayConfig == nil || ingressConfig.EnvoyGatewayConfig.TLSSecretName == "" { + return errors.New("envoygateway ingress controller requires TLSSecretName to be set in envoyGatewayConfig when TLS is enabled (externalStartingPort == -1). EnvoyGateway only supports TLS termination at the gateway level") + } + } + + // Create Gateway resource + gateway := r.gateway(eListener, ingressConfig) + err := k8sutil.Reconcile(log, r.Client, gateway, r.KafkaCluster) + if err != nil { + return err + } + + // Create TCPRoute for each broker + // Note: We always use TCPRoute because EnvoyGateway performs TLS termination + // at the gateway level, so traffic to backends is plain TCP + for _, broker := range r.KafkaCluster.Spec.Brokers { + route := r.tcpRoute(broker.Id, eListener, ingressConfig) + err := k8sutil.Reconcile(log, r.Client, route, r.KafkaCluster) + if err != nil { + return err + } + } + + // Create TCPRoute for anycast (all-broker) service + anyCastRoute := r.tcpRouteAllBroker(eListener, ingressConfig) + err = k8sutil.Reconcile(log, r.Client, anyCastRoute, r.KafkaCluster) + if err != nil { + return err + } + } + } else if r.KafkaCluster.Spec.RemoveUnusedIngressResources { + // Cleaning up unused envoy gateway resources when ingress controller is not envoygateway or externalListener access method is not LoadBalancer + deletionCounter := 0 + ctx := context.Background() + envoyGatewayResourcesGVK := []schema.GroupVersionKind{ + { + Version: gatewayv1.GroupVersion.Version, + Group: gatewayv1.GroupVersion.Group, + Kind: "Gateway", + }, + { + Version: "v1alpha2", + Group: gatewayv1.GroupVersion.Group, + Kind: "TLSRoute", + }, + { + Version: "v1alpha2", + Group: gatewayv1.GroupVersion.Group, + Kind: "TCPRoute", + }, + } + + for _, gvk := range envoyGatewayResourcesGVK { + var envoyGatewayResources unstructured.UnstructuredList + envoyGatewayResources.SetGroupVersionKind(gvk) + err := r.List(ctx, &envoyGatewayResources, + client.InNamespace(r.KafkaCluster.Namespace), + client.MatchingLabels(labelsForEnvoyGatewayWithoutEListenerName(r.KafkaCluster.Name))) + if err != nil { + return errors.WrapIfWithDetails(err, "failed to list envoy gateway resources", "gvk", gvk) + } + + for _, removeObject := range envoyGatewayResources.Items { + if !strings.Contains(removeObject.GetLabels()[util.ExternalListenerLabelNameKey], eListener.Name) || + util.ObjectManagedByClusterRegistry(&removeObject) || + !removeObject.GetDeletionTimestamp().IsZero() { + continue + } + if err := r.Delete(ctx, &removeObject); client.IgnoreNotFound(err) != nil { + return errors.Wrap(err, "error when removing envoy gateway ingress resources") + } + log.V(1).Info(fmt.Sprintf("Deleted envoy gateway ingress '%s' resource '%s' for externalListener '%s'", gvk.Kind, removeObject.GetName(), eListener.Name)) + deletionCounter++ + } + } + if deletionCounter > 0 { + log.Info(fmt.Sprintf("Removed '%d' resources for envoy gateway ingress", deletionCounter)) + } + } + } + log.V(1).Info("Reconciled") + + return nil +} diff --git a/pkg/resources/envoygateway/envoygateway_test.go b/pkg/resources/envoygateway/envoygateway_test.go new file mode 100644 index 000000000..3ca081ca2 --- /dev/null +++ b/pkg/resources/envoygateway/envoygateway_test.go @@ -0,0 +1,237 @@ +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envoygateway + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/banzaicloud/koperator/api/v1beta1" + "github.com/banzaicloud/koperator/pkg/resources" +) + +func TestGatewayGeneration(t *testing.T) { + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + Brokers: []v1beta1.Broker{ + {Id: 0}, + {Id: 1}, + {Id: 2}, + }, + EnvoyGatewayConfig: v1beta1.EnvoyGatewayIngressConfig{ + GatewayClassName: "test-gateway-class", + }, + }, + } + + reconciler := &Reconciler{ + Reconciler: resources.Reconciler{ + KafkaCluster: cluster, + }, + } + + eListener := v1beta1.ExternalListenerConfig{ + CommonListenerSpec: v1beta1.CommonListenerSpec{ + Name: "test-listener", + ContainerPort: 9092, + }, + ExternalStartingPort: 19090, + } + + ingressConfig := v1beta1.IngressConfig{ + EnvoyGatewayConfig: &cluster.Spec.EnvoyGatewayConfig, + } + + gateway := reconciler.gateway(eListener, ingressConfig) + + gw, ok := gateway.(*gatewayv1.Gateway) + if !ok { + t.Fatal("Expected Gateway type") + } + + if gw.Name != "kafka-gateway-test-listener" { + t.Errorf("Expected gateway name 'kafka-gateway-test-listener', got '%s'", gw.Name) + } + + if string(gw.Spec.GatewayClassName) != "test-gateway-class" { + t.Errorf("Expected gateway class 'test-gateway-class', got '%s'", gw.Spec.GatewayClassName) + } + + // 3 brokers + 1 anycast = 4 listeners + if len(gw.Spec.Listeners) != 4 { + t.Errorf("Expected 4 listeners, got %d", len(gw.Spec.Listeners)) + } + + // Check broker listeners + for i := 0; i < 3; i++ { + expectedName := gatewayv1.SectionName("broker-" + string(rune('0'+i))) + if gw.Spec.Listeners[i].Name != expectedName { + t.Errorf("Expected listener name '%s', got '%s'", expectedName, gw.Spec.Listeners[i].Name) + } + expectedPort := gatewayv1.PortNumber(19090 + i) + if gw.Spec.Listeners[i].Port != expectedPort { + t.Errorf("Expected port %d, got %d", expectedPort, gw.Spec.Listeners[i].Port) + } + } + + // Check anycast listener + if gw.Spec.Listeners[3].Name != "anycast" { + t.Errorf("Expected anycast listener name 'anycast', got '%s'", gw.Spec.Listeners[3].Name) + } + // Anycast listener should use the default anycast port (29092), not ExternalStartingPort + expectedAnycastPort := gatewayv1.PortNumber(29092) + if gw.Spec.Listeners[3].Port != expectedAnycastPort { + t.Errorf("Expected anycast port %d, got %d", expectedAnycastPort, gw.Spec.Listeners[3].Port) + } +} + +func TestTCPRouteGeneration(t *testing.T) { + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + Brokers: []v1beta1.Broker{ + {Id: 0}, + }, + }, + } + + reconciler := &Reconciler{ + Reconciler: resources.Reconciler{ + KafkaCluster: cluster, + }, + } + + eListener := v1beta1.ExternalListenerConfig{ + CommonListenerSpec: v1beta1.CommonListenerSpec{ + Name: "test-listener", + ContainerPort: 9092, + }, + ExternalStartingPort: 19090, + } + + ingressConfig := v1beta1.IngressConfig{ + EnvoyGatewayConfig: &v1beta1.EnvoyGatewayIngressConfig{}, + } + + route := reconciler.tcpRoute(0, eListener, ingressConfig) + + tcpRoute, ok := route.(*gatewayv1alpha2.TCPRoute) + if !ok { + t.Fatal("Expected TCPRoute type") + } + + if tcpRoute.Name != "kafka-tcproute-test-listener-0" { + t.Errorf("Expected route name 'kafka-tcproute-test-listener-0', got '%s'", tcpRoute.Name) + } + + if len(tcpRoute.Spec.ParentRefs) != 1 { + t.Errorf("Expected 1 parent ref, got %d", len(tcpRoute.Spec.ParentRefs)) + } + + if string(tcpRoute.Spec.ParentRefs[0].Name) != "kafka-gateway-test-listener" { + t.Errorf("Expected parent gateway 'kafka-gateway-test-listener', got '%s'", tcpRoute.Spec.ParentRefs[0].Name) + } + + if len(tcpRoute.Spec.Rules) != 1 { + t.Errorf("Expected 1 rule, got %d", len(tcpRoute.Spec.Rules)) + } + + if len(tcpRoute.Spec.Rules[0].BackendRefs) != 1 { + t.Errorf("Expected 1 backend ref, got %d", len(tcpRoute.Spec.Rules[0].BackendRefs)) + } + + if string(tcpRoute.Spec.Rules[0].BackendRefs[0].Name) != "test-cluster-all-broker" { + t.Errorf("Expected backend 'test-cluster-all-broker', got '%s'", tcpRoute.Spec.Rules[0].BackendRefs[0].Name) + } +} + +func TestGatewayGenerationWithTLS(t *testing.T) { + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + Brokers: []v1beta1.Broker{ + {Id: 0}, + {Id: 1}, + {Id: 2}, + }, + EnvoyGatewayConfig: v1beta1.EnvoyGatewayIngressConfig{ + GatewayClassName: "test-gateway-class", + TLSSecretName: "test-tls-secret", + }, + }, + } + + reconciler := &Reconciler{ + Reconciler: resources.Reconciler{ + KafkaCluster: cluster, + }, + } + + eListener := v1beta1.ExternalListenerConfig{ + CommonListenerSpec: v1beta1.CommonListenerSpec{ + Name: "test-listener", + ContainerPort: 9092, + }, + ExternalStartingPort: -1, // TLS enabled + } + + ingressConfig := v1beta1.IngressConfig{ + EnvoyGatewayConfig: &cluster.Spec.EnvoyGatewayConfig, + } + + gateway := reconciler.gateway(eListener, ingressConfig) + + gw, ok := gateway.(*gatewayv1.Gateway) + if !ok { + t.Fatal("Expected Gateway type") + } + + // 3 brokers + 1 anycast = 4 listeners + if len(gw.Spec.Listeners) != 4 { + t.Errorf("Expected 4 listeners, got %d", len(gw.Spec.Listeners)) + } + + // When TLS is enabled (externalStartingPort == -1), all broker listeners should use the anycast port + expectedPort := gatewayv1.PortNumber(29092) // default anycast port + for i := 0; i < 3; i++ { + if gw.Spec.Listeners[i].Port != expectedPort { + t.Errorf("Expected broker %d port %d (anycast port when TLS enabled), got %d", i, expectedPort, gw.Spec.Listeners[i].Port) + } + if gw.Spec.Listeners[i].Protocol != gatewayv1.TLSProtocolType { + t.Errorf("Expected broker %d protocol TLS, got %s", i, gw.Spec.Listeners[i].Protocol) + } + if gw.Spec.Listeners[i].TLS == nil { + t.Errorf("Expected broker %d to have TLS config", i) + } + } + + // Check anycast listener also uses the same port + if gw.Spec.Listeners[3].Port != expectedPort { + t.Errorf("Expected anycast port %d, got %d", expectedPort, gw.Spec.Listeners[3].Port) + } +} diff --git a/pkg/resources/envoygateway/gateway.go b/pkg/resources/envoygateway/gateway.go new file mode 100644 index 000000000..d0b1ef08c --- /dev/null +++ b/pkg/resources/envoygateway/gateway.go @@ -0,0 +1,132 @@ +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envoygateway + +import ( + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + apiutil "github.com/banzaicloud/koperator/api/util" + "github.com/banzaicloud/koperator/api/v1beta1" + "github.com/banzaicloud/koperator/pkg/resources/templates" + envoygatewayutils "github.com/banzaicloud/koperator/pkg/util/envoygateway" +) + +func (r *Reconciler) gateway(eListener v1beta1.ExternalListenerConfig, + ingressConfig v1beta1.IngressConfig) client.Object { + gatewayName := fmt.Sprintf(envoygatewayutils.GatewayNameTemplate, eListener.Name) + if ingressConfig.EnvoyGatewayConfig != nil && ingressConfig.EnvoyGatewayConfig.GatewayName != "" { + gatewayName = ingressConfig.EnvoyGatewayConfig.GatewayName + } + + gatewayClassName := "eg" + if ingressConfig.EnvoyGatewayConfig != nil { + gatewayClassName = ingressConfig.EnvoyGatewayConfig.GetGatewayClassName() + } + + labels := labelsForEnvoyGateway(r.KafkaCluster.Name, eListener.Name) + if r.KafkaCluster.Spec.PropagateLabels { + labels = apiutil.MergeLabels(r.KafkaCluster.Labels, labels) + } + + annotations := make(map[string]string) + if ingressConfig.EnvoyGatewayConfig != nil { + annotations = ingressConfig.EnvoyGatewayConfig.GetAnnotations() + } + + // Build listeners for the Gateway + var listeners []gatewayv1.Listener + + // Add listener for each broker + for _, broker := range r.KafkaCluster.Spec.Brokers { + listenerName := gatewayv1.SectionName(fmt.Sprintf("broker-%d", broker.Id)) + port := eListener.GetBrokerPort(broker.Id) + + listener := gatewayv1.Listener{ + Name: listenerName, + Port: port, + Protocol: gatewayv1.TCPProtocolType, + } + + if eListener.TLSEnabled() { + listener.Protocol = gatewayv1.TLSProtocolType + + // When TLS is enabled, use hostname-based routing (SNI) + // Each broker needs a unique hostname to satisfy Gateway API uniqueness constraint + if ingressConfig.EnvoyGatewayConfig != nil && ingressConfig.EnvoyGatewayConfig.BrokerHostnameTemplate != "" { + hostname := gatewayv1.Hostname(envoygatewayutils.GetBrokerHostname(ingressConfig.EnvoyGatewayConfig.BrokerHostnameTemplate, broker.Id)) + listener.Hostname = &hostname + } + + // EnvoyGateway only supports TLS termination at the gateway level + // TLSSecretName is validated to be present in the Reconcile method + listener.TLS = &gatewayv1.ListenerTLSConfig{ + Mode: func() *gatewayv1.TLSModeType { + mode := gatewayv1.TLSModeTerminate + return &mode + }(), + CertificateRefs: []gatewayv1.SecretObjectReference{ + { + Name: gatewayv1.ObjectName(ingressConfig.EnvoyGatewayConfig.TLSSecretName), + }, + }, + } + } + + listeners = append(listeners, listener) + } + + // Add anycast listener (all-broker) + anycastListenerName := gatewayv1.SectionName("anycast") + anycastPort := eListener.GetAnyCastPort() + + anycastListener := gatewayv1.Listener{ + Name: anycastListenerName, + Port: anycastPort, + Protocol: gatewayv1.TCPProtocolType, + } + + if eListener.TLSEnabled() { + anycastListener.Protocol = gatewayv1.TLSProtocolType + + // EnvoyGateway only supports TLS termination at the gateway level + // TLSSecretName is validated to be present in the Reconcile method + anycastListener.TLS = &gatewayv1.ListenerTLSConfig{ + Mode: func() *gatewayv1.TLSModeType { + mode := gatewayv1.TLSModeTerminate + return &mode + }(), + CertificateRefs: []gatewayv1.SecretObjectReference{ + { + Name: gatewayv1.ObjectName(ingressConfig.EnvoyGatewayConfig.TLSSecretName), + }, + }, + } + } + + listeners = append(listeners, anycastListener) + + gateway := &gatewayv1.Gateway{ + ObjectMeta: templates.ObjectMetaWithAnnotations(gatewayName, labels, annotations, r.KafkaCluster), + Spec: gatewayv1.GatewaySpec{ + GatewayClassName: gatewayv1.ObjectName(gatewayClassName), + Listeners: listeners, + }, + } + + return gateway +} diff --git a/pkg/resources/envoygateway/tcproute.go b/pkg/resources/envoygateway/tcproute.go new file mode 100644 index 000000000..8a62fdf5e --- /dev/null +++ b/pkg/resources/envoygateway/tcproute.go @@ -0,0 +1,135 @@ +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envoygateway + +import ( + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + apiutil "github.com/banzaicloud/koperator/api/util" + "github.com/banzaicloud/koperator/api/v1beta1" + "github.com/banzaicloud/koperator/pkg/resources/templates" + envoygatewayutils "github.com/banzaicloud/koperator/pkg/util/envoygateway" + "github.com/banzaicloud/koperator/pkg/util/kafka" +) + +func (r *Reconciler) tcpRoute(brokerId int32, eListener v1beta1.ExternalListenerConfig, + ingressConfig v1beta1.IngressConfig) client.Object { + tcpRouteName := fmt.Sprintf(envoygatewayutils.TCPRouteNameTemplate, eListener.Name, fmt.Sprintf("%d", brokerId)) + + gatewayName := fmt.Sprintf(envoygatewayutils.GatewayNameTemplate, eListener.Name) + if ingressConfig.EnvoyGatewayConfig != nil && ingressConfig.EnvoyGatewayConfig.GatewayName != "" { + gatewayName = ingressConfig.EnvoyGatewayConfig.GatewayName + } + + labels := labelsForEnvoyGateway(r.KafkaCluster.Name, eListener.Name) + if r.KafkaCluster.Spec.PropagateLabels { + labels = apiutil.MergeLabels(r.KafkaCluster.Labels, labels) + } + + // Backend service reference + serviceName := fmt.Sprintf(kafka.AllBrokerServiceTemplate, r.KafkaCluster.Name) + servicePort := eListener.ContainerPort + + tcpRoute := &gatewayv1alpha2.TCPRoute{ + ObjectMeta: templates.ObjectMeta(tcpRouteName, labels, r.KafkaCluster), + Spec: gatewayv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gatewayv1.CommonRouteSpec{ + ParentRefs: []gatewayv1.ParentReference{ + { + Name: gatewayv1.ObjectName(gatewayName), + SectionName: func() *gatewayv1.SectionName { + name := gatewayv1.SectionName(fmt.Sprintf("broker-%d", brokerId)) + return &name + }(), + }, + }, + }, + Rules: []gatewayv1alpha2.TCPRouteRule{ + { + BackendRefs: []gatewayv1.BackendRef{ + { + BackendObjectReference: gatewayv1.BackendObjectReference{ + Name: gatewayv1.ObjectName(serviceName), + Port: func() *gatewayv1.PortNumber { + port := servicePort + return &port + }(), + }, + }, + }, + }, + }, + }, + } + + return tcpRoute +} + +func (r *Reconciler) tcpRouteAllBroker(eListener v1beta1.ExternalListenerConfig, + ingressConfig v1beta1.IngressConfig) client.Object { + tcpRouteName := fmt.Sprintf(envoygatewayutils.TCPRouteNameTemplate, eListener.Name, "anycast") + + gatewayName := fmt.Sprintf(envoygatewayutils.GatewayNameTemplate, eListener.Name) + if ingressConfig.EnvoyGatewayConfig != nil && ingressConfig.EnvoyGatewayConfig.GatewayName != "" { + gatewayName = ingressConfig.EnvoyGatewayConfig.GatewayName + } + + labels := labelsForEnvoyGateway(r.KafkaCluster.Name, eListener.Name) + if r.KafkaCluster.Spec.PropagateLabels { + labels = apiutil.MergeLabels(r.KafkaCluster.Labels, labels) + } + + // Backend service reference + serviceName := fmt.Sprintf(kafka.AllBrokerServiceTemplate, r.KafkaCluster.Name) + servicePort := eListener.ContainerPort + + tcpRoute := &gatewayv1alpha2.TCPRoute{ + ObjectMeta: templates.ObjectMeta(tcpRouteName, labels, r.KafkaCluster), + Spec: gatewayv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gatewayv1.CommonRouteSpec{ + ParentRefs: []gatewayv1.ParentReference{ + { + Name: gatewayv1.ObjectName(gatewayName), + SectionName: func() *gatewayv1.SectionName { + name := gatewayv1.SectionName("anycast") + return &name + }(), + }, + }, + }, + Rules: []gatewayv1alpha2.TCPRouteRule{ + { + BackendRefs: []gatewayv1.BackendRef{ + { + BackendObjectReference: gatewayv1.BackendObjectReference{ + Name: gatewayv1.ObjectName(serviceName), + Port: func() *gatewayv1.PortNumber { + port := servicePort + return &port + }(), + }, + }, + }, + }, + }, + }, + } + + return tcpRoute +} diff --git a/pkg/resources/kafka/kafka.go b/pkg/resources/kafka/kafka.go index d737754f8..88b37e687 100644 --- a/pkg/resources/kafka/kafka.go +++ b/pkg/resources/kafka/kafka.go @@ -55,6 +55,7 @@ import ( certutil "github.com/banzaicloud/koperator/pkg/util/cert" contourutils "github.com/banzaicloud/koperator/pkg/util/contour" envoyutils "github.com/banzaicloud/koperator/pkg/util/envoy" + envoygatewayutils "github.com/banzaicloud/koperator/pkg/util/envoygateway" "github.com/banzaicloud/koperator/pkg/util/kafka" pkicommon "github.com/banzaicloud/koperator/pkg/util/pki" ) @@ -1416,7 +1417,12 @@ func (r *Reconciler) getBrokerHost(log logr.Logger, defaultHost string, broker b // portNumber = eListener.ContainerPort case corev1.ServiceTypeLoadBalancer: if eListener.TLSEnabled() { - brokerHost = iConfig.EnvoyConfig.GetBrokerHostname(broker.Id) + // Check which ingress controller is being used + if iConfig.EnvoyConfig != nil { + brokerHost = iConfig.EnvoyConfig.GetBrokerHostname(broker.Id) + } else if iConfig.EnvoyGatewayConfig != nil { + brokerHost = iConfig.EnvoyGatewayConfig.GetBrokerHostname(broker.Id) + } if brokerHost == "" { return "", errors.New("brokerHostnameTemplate is not set in the ingress service settings") } @@ -1427,94 +1433,112 @@ func (r *Reconciler) getBrokerHost(log logr.Logger, defaultHost string, broker b return fmt.Sprintf("%s:%d", brokerHost, portNumber), nil } -func (r *Reconciler) createExternalListenerStatuses(log logr.Logger) (map[string]banzaiv1beta1.ListenerStatusList, error) { - extListenerStatuses := make(map[string]banzaiv1beta1.ListenerStatusList, len(r.KafkaCluster.Spec.ListenersConfig.ExternalListeners)) - for _, eListener := range r.KafkaCluster.Spec.ListenersConfig.ExternalListeners { - var host string - var foundLBService *corev1.Service - var err error - ingressConfigs, defaultControllerName, err := util.GetIngressConfigs(r.KafkaCluster.Spec, eListener) - if err != nil { - return nil, err +func (r *Reconciler) createStandardExternalListenerStatuses(log logr.Logger, eListener banzaiv1beta1.ExternalListenerConfig) (banzaiv1beta1.ListenerStatusList, error) { + var host string + var foundLBService *corev1.Service + var err error + ingressConfigs, defaultControllerName, err := util.GetIngressConfigs(r.KafkaCluster.Spec, eListener) + if err != nil { + return nil, err + } + listenerStatusList := make(banzaiv1beta1.ListenerStatusList, 0, len(r.KafkaCluster.Spec.Brokers)+1) + for iConfigName, iConfig := range ingressConfigs { + if !util.IsIngressConfigInUse(iConfigName, defaultControllerName, r.KafkaCluster, log) { + continue } - listenerStatusList := make(banzaiv1beta1.ListenerStatusList, 0, len(r.KafkaCluster.Spec.Brokers)+1) - for iConfigName, iConfig := range ingressConfigs { - if !util.IsIngressConfigInUse(iConfigName, defaultControllerName, r.KafkaCluster, log) { - continue + if iConfig.HostnameOverride != "" { + host = iConfig.HostnameOverride + } else if eListener.GetAccessMethod() == corev1.ServiceTypeLoadBalancer && + (r.KafkaCluster.Spec.GetIngressController() == envoyutils.IngressControllerName || + r.KafkaCluster.Spec.GetIngressController() == contourutils.IngressControllerName) { + // For envoy and contour ingress controllers, get the LoadBalancer service + foundLBService, err = getServiceFromExternalListener(r.Client, r.KafkaCluster, eListener.Name, iConfigName) + if err != nil { + return nil, errors.WrapIfWithDetails(err, "could not get service corresponding to the external listener", "externalListenerName", eListener.Name) + } + lbIP, err := getLoadBalancerIP(foundLBService) + if err != nil { + return nil, errors.WrapIfWithDetails(err, "could not extract IP from LoadBalancer service", "externalListenerName", eListener.Name) } - if iConfig.HostnameOverride != "" { - host = iConfig.HostnameOverride - } else if eListener.GetAccessMethod() == corev1.ServiceTypeLoadBalancer { + host = lbIP + } + + // optionally add all brokers service to the top of the list + if eListener.GetAccessMethod() != corev1.ServiceTypeNodePort && + (r.KafkaCluster.Spec.GetIngressController() == envoyutils.IngressControllerName || + r.KafkaCluster.Spec.GetIngressController() == contourutils.IngressControllerName) { + if foundLBService == nil { foundLBService, err = getServiceFromExternalListener(r.Client, r.KafkaCluster, eListener.Name, iConfigName) if err != nil { return nil, errors.WrapIfWithDetails(err, "could not get service corresponding to the external listener", "externalListenerName", eListener.Name) } - lbIP, err := getLoadBalancerIP(foundLBService) - if err != nil { - return nil, errors.WrapIfWithDetails(err, "could not extract IP from LoadBalancer service", "externalListenerName", eListener.Name) + } + var allBrokerPort int32 = 0 + for _, port := range foundLBService.Spec.Ports { + if port.Name == "tcp-all-broker" { + allBrokerPort = port.Port + break } - host = lbIP } + if allBrokerPort == 0 { + return nil, errors.NewWithDetails("could not find port with name tcp-all-broker", "externalListenerName", eListener.Name) + } + var anyBrokerStatusName string + if iConfigName == util.IngressConfigGlobalName { + anyBrokerStatusName = "any-broker" + } else { + anyBrokerStatusName = fmt.Sprintf("any-broker-%s", iConfigName) + } + listenerStatus := banzaiv1beta1.ListenerStatus{ + Name: anyBrokerStatusName, + Address: fmt.Sprintf("%s:%d", host, allBrokerPort), + } + listenerStatusList = append(listenerStatusList, listenerStatus) + } - // optionally add all brokers service to the top of the list - if eListener.GetAccessMethod() != corev1.ServiceTypeNodePort { - if foundLBService == nil { - foundLBService, err = getServiceFromExternalListener(r.Client, r.KafkaCluster, eListener.Name, iConfigName) - if err != nil { - return nil, errors.WrapIfWithDetails(err, "could not get service corresponding to the external listener", "externalListenerName", eListener.Name) - } - } - var allBrokerPort int32 = 0 - for _, port := range foundLBService.Spec.Ports { - if port.Name == "tcp-all-broker" { - allBrokerPort = port.Port - break - } - } - if allBrokerPort == 0 { - return nil, errors.NewWithDetails("could not find port with name tcp-all-broker", "externalListenerName", eListener.Name) - } - var anyBrokerStatusName string - if iConfigName == util.IngressConfigGlobalName { - anyBrokerStatusName = "any-broker" - } else { - anyBrokerStatusName = fmt.Sprintf("any-broker-%s", iConfigName) - } + for _, broker := range r.KafkaCluster.Spec.Brokers { + brokerHostPort, err := r.getBrokerHost(log, host, broker, eListener, iConfig) + if err != nil { + return nil, errors.WrapIfWithDetails(err, "could not get brokerHost for external listener status", "brokerID", broker.Id) + } + + brokerConfig, err := broker.GetBrokerConfig(r.KafkaCluster.Spec) + if err != nil { + return nil, err + } + if util.ShouldIncludeBroker(brokerConfig, r.KafkaCluster.Status, int(broker.Id), defaultControllerName, iConfigName) { listenerStatus := banzaiv1beta1.ListenerStatus{ - Name: anyBrokerStatusName, - Address: fmt.Sprintf("%s:%d", host, allBrokerPort), + Name: fmt.Sprintf("broker-%d", broker.Id), + Address: brokerHostPort, } listenerStatusList = append(listenerStatusList, listenerStatus) } + } + } + // We have to sort the listener status list since the ingress config is a + // map and we are using that for the generation + sort.Sort(listenerStatusList) - for _, broker := range r.KafkaCluster.Spec.Brokers { - brokerHostPort, err := r.getBrokerHost(log, host, broker, eListener, iConfig) - if err != nil { - return nil, errors.WrapIfWithDetails(err, "could not get brokerHost for external listener status", "brokerID", broker.Id) - } + return listenerStatusList, nil +} - brokerConfig, err := broker.GetBrokerConfig(r.KafkaCluster.Spec) - if err != nil { - return nil, err - } - if util.ShouldIncludeBroker(brokerConfig, r.KafkaCluster.Status, int(broker.Id), defaultControllerName, iConfigName) { - listenerStatus := banzaiv1beta1.ListenerStatus{ - Name: fmt.Sprintf("broker-%d", broker.Id), - Address: brokerHostPort, - } - listenerStatusList = append(listenerStatusList, listenerStatus) - } - } +func (r *Reconciler) createExternalListenerStatuses(log logr.Logger) (map[string]banzaiv1beta1.ListenerStatusList, error) { + extListenerStatuses := make(map[string]banzaiv1beta1.ListenerStatusList, len(r.KafkaCluster.Spec.ListenersConfig.ExternalListeners)) + for _, eListener := range r.KafkaCluster.Spec.ListenersConfig.ExternalListeners { + listenerStatusList, err := r.createListenerStatuses(log, eListener) + if err != nil { + return nil, err } - // We have to sort the listener status list since the ingress config is a - // map and we are using that for the generation - sort.Sort(listenerStatusList) - extListenerStatuses[eListener.Name] = listenerStatusList } return extListenerStatuses, nil } +func (r *Reconciler) createListenerStatuses(log logr.Logger, eListener banzaiv1beta1.ExternalListenerConfig) (banzaiv1beta1.ListenerStatusList, error) { + // Handle standard external listeners + return r.createStandardExternalListenerStatuses(log, eListener) +} + func (r *Reconciler) getK8sAssignedNodeport(log logr.Logger, eListenerName string, brokerId int32) (int32, error) { log.Info("determining automatically assigned nodeport", banzaiv1beta1.BrokerIdLabelKey, brokerId, "listenerName", eListenerName) @@ -1642,6 +1666,10 @@ func getServiceFromExternalListener(client client.Client, cluster *banzaiv1beta1 iControllerServiceName = fmt.Sprintf(contourutils.ContourServiceNameWithScope, eListenerName, ingressConfigName, cluster.GetName()) iControllerServiceName = strings.ReplaceAll(iControllerServiceName, "_", "-") } + case envoygatewayutils.IngressControllerName: + // EnvoyGateway uses Gateway API resources, not LoadBalancer services + // Return an error to indicate this is not supported for EnvoyGateway + return nil, errors.New("EnvoyGateway does not use LoadBalancer services; use Gateway API resources instead") } err := client.Get(context.TODO(), types.NamespacedName{Name: iControllerServiceName, Namespace: cluster.GetNamespace()}, foundLBService) diff --git a/pkg/resources/nodeportexternalaccess/nodeportExternalAccess_test.go b/pkg/resources/nodeportexternalaccess/nodeportExternalAccess_test.go new file mode 100644 index 000000000..4914ac1af --- /dev/null +++ b/pkg/resources/nodeportexternalaccess/nodeportExternalAccess_test.go @@ -0,0 +1,233 @@ +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nodeportexternalaccess + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/banzaicloud/koperator/api/v1beta1" +) + +// MockClient is a mock implementation of client.Client +type MockClient struct { + mock.Mock +} + +func (m *MockClient) Get(ctx context.Context, key types.NamespacedName, obj client.Object, opts ...client.GetOption) error { + args := m.Called(ctx, key, obj, opts) + return args.Error(0) +} + +func (m *MockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + args := m.Called(ctx, list, opts) + return args.Error(0) +} + +func (m *MockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + args := m.Called(ctx, obj, patch, opts) + return args.Error(0) +} + +func (m *MockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Status() client.StatusWriter { + args := m.Called() + return args.Get(0).(client.StatusWriter) +} + +func (m *MockClient) Scheme() *runtime.Scheme { + args := m.Called() + return args.Get(0).(*runtime.Scheme) +} + +func (m *MockClient) RESTMapper() meta.RESTMapper { + args := m.Called() + return args.Get(0).(meta.RESTMapper) +} + +func (m *MockClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + args := m.Called(obj) + return args.Get(0).(schema.GroupVersionKind), args.Error(1) +} + +func (m *MockClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + args := m.Called(obj) + return args.Bool(0), args.Error(1) +} + +func (m *MockClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) SubResource(subResource string) client.SubResourceClient { + args := m.Called(subResource) + return args.Get(0).(client.SubResourceClient) +} + +func TestNew(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + } + + reconciler := New(mockClient, cluster) + + assert.NotNil(t, reconciler) + assert.Equal(t, mockClient, reconciler.Client) + assert.Equal(t, cluster, reconciler.KafkaCluster) +} + +func TestReconcile_WithNodePortAccessMethod(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + ListenersConfig: v1beta1.ListenersConfig{ + ExternalListeners: nil, // No external listeners to avoid service function call + }, + }, + } + + reconciler := New(mockClient, cluster) + + log := logr.Discard() + err := reconciler.Reconcile(log) + assert.NoError(t, err) +} + +func TestReconcile_WithRemoveUnusedIngressResources(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + ListenersConfig: v1beta1.ListenersConfig{ + ExternalListeners: nil, // No external listeners to avoid service function call + }, + RemoveUnusedIngressResources: true, + }, + } + + reconciler := New(mockClient, cluster) + + // Mock the Delete call + mockClient.On("Delete", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + log := logr.Discard() + err := reconciler.Reconcile(log) + assert.NoError(t, err) +} + +func TestReconcile_NoExternalListeners(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + ListenersConfig: v1beta1.ListenersConfig{ + ExternalListeners: nil, // No external listeners + }, + }, + } + + reconciler := New(mockClient, cluster) + + log := logr.Discard() + err := reconciler.Reconcile(log) + assert.NoError(t, err) +} + +func TestReconcile_ErrorHandling(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + ListenersConfig: v1beta1.ListenersConfig{ + ExternalListeners: nil, // No external listeners to avoid service function call + }, + }, + } + + reconciler := New(mockClient, cluster) + + log := logr.Discard() + err := reconciler.Reconcile(log) + assert.NoError(t, err) +} + +func TestReconcile_DeleteErrorHandling(t *testing.T) { + mockClient := &MockClient{} + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: v1beta1.KafkaClusterSpec{ + ListenersConfig: v1beta1.ListenersConfig{ + ExternalListeners: nil, // No external listeners to avoid service function call + }, + RemoveUnusedIngressResources: true, + }, + } + + reconciler := New(mockClient, cluster) + + log := logr.Discard() + err := reconciler.Reconcile(log) + assert.NoError(t, err) +} diff --git a/pkg/util/envoygateway/common.go b/pkg/util/envoygateway/common.go new file mode 100644 index 000000000..4c30c55ea --- /dev/null +++ b/pkg/util/envoygateway/common.go @@ -0,0 +1,40 @@ +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envoygateway + +import ( + "strconv" + "strings" +) + +const ( + // IngressControllerName name for envoy gateway ingress controller + IngressControllerName = "envoygateway" + + // GatewayNameTemplate template for Gateway resource name + GatewayNameTemplate = "kafka-gateway-%s" + + // TLSRouteNameTemplate template for TLSRoute resource name + TLSRouteNameTemplate = "kafka-tlsroute-%s-%s" + + // TCPRouteNameTemplate template for TCPRoute resource name + TCPRouteNameTemplate = "kafka-tcproute-%s-%s" +) + +// GetBrokerHostname returns the broker hostname for the given broker ID +// by replacing %id in the template with the actual broker ID +func GetBrokerHostname(template string, brokerId int32) string { + return strings.Replace(template, "%id", strconv.Itoa(int(brokerId)), 1) +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 87d8fecb6..18a071bfa 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -59,6 +59,7 @@ import ( "github.com/banzaicloud/koperator/pkg/util/cert" "github.com/banzaicloud/koperator/pkg/util/contour" envoyutils "github.com/banzaicloud/koperator/pkg/util/envoy" + envoygatewayutils "github.com/banzaicloud/koperator/pkg/util/envoygateway" properties "github.com/banzaicloud/koperator/properties/pkg" ) @@ -357,6 +358,34 @@ func GetIngressConfigs(kafkaClusterSpec v1beta1.KafkaClusterSpec, }, } } + case envoygatewayutils.IngressControllerName: + if eListenerConfig.Config != nil { + defaultIngressConfigName = eListenerConfig.Config.DefaultIngressConfig + ingressConfigs = make(map[string]v1beta1.IngressConfig, len(eListenerConfig.Config.IngressConfig)) + for k, iConf := range eListenerConfig.Config.IngressConfig { + if iConf.EnvoyGatewayConfig != nil { + err := mergo.Merge(iConf.EnvoyGatewayConfig, kafkaClusterSpec.EnvoyGatewayConfig) + if err != nil { + return nil, "", errors.WrapWithDetails(err, + "could not merge global envoy gateway config with local one", "envoyGatewayConfig", k) + } + err = mergo.Merge(&iConf.IngressServiceSettings, eListenerConfig.IngressServiceSettings) + if err != nil { + return nil, "", errors.WrapWithDetails(err, + "could not merge global loadbalancer config with local one", + "externalListenerName", eListenerConfig.Name) + } + ingressConfigs[k] = iConf + } + } + } else { + ingressConfigs = map[string]v1beta1.IngressConfig{ + IngressConfigGlobalName: { + IngressServiceSettings: eListenerConfig.IngressServiceSettings, + EnvoyGatewayConfig: &kafkaClusterSpec.EnvoyGatewayConfig, + }, + } + } default: return nil, "", errors.NewWithDetails("not supported ingress type", "name", kafkaClusterSpec.GetIngressController()) } diff --git a/pkg/webhooks/kafkacluster_validator.go b/pkg/webhooks/kafkacluster_validator.go index 486db357e..78e041ca7 100644 --- a/pkg/webhooks/kafkacluster_validator.go +++ b/pkg/webhooks/kafkacluster_validator.go @@ -139,6 +139,12 @@ func checkExternalListenerStartingPort(kafkaClusterSpec *banzaicloudv1beta1.Kafk var allErrs field.ErrorList const maxPort int32 = 65535 for i, extListener := range kafkaClusterSpec.ListenersConfig.ExternalListeners { + // Skip port validation when TLS is enabled (externalStartingPort == -1) + // In TLS mode, GetAnyCastPort() is used instead of externalStartingPort + brokerId + if extListener.TLSEnabled() { + continue + } + var outOfRangeBrokerIDs, collidingPortsBrokerIDs []int32 for _, broker := range kafkaClusterSpec.Brokers { externalPort := util.GetExternalPortForBroker(extListener.ExternalStartingPort, broker.Id) diff --git a/pkg/webhooks/kafkacluster_validator_test.go b/pkg/webhooks/kafkacluster_validator_test.go index 952c3f389..42f734922 100644 --- a/pkg/webhooks/kafkacluster_validator_test.go +++ b/pkg/webhooks/kafkacluster_validator_test.go @@ -248,6 +248,23 @@ func TestCheckExternalListenerStartingPort(t *testing.T) { "test-external2", int32(8081), int32(8080), int32(29092), []int32{11})), ), }, + { + // When TLS is enabled (externalStartingPort == -1), port validation should be skipped + // because GetAnyCastPort() is used instead of externalStartingPort + brokerId + testName: "valid config: TLS enabled with externalStartingPort -1 (should skip port validation)", + kafkaClusterSpec: v1beta1.KafkaClusterSpec{ + Brokers: []v1beta1.Broker{{Id: 0}, {Id: 1}, {Id: 2}}, + ListenersConfig: v1beta1.ListenersConfig{ + ExternalListeners: []v1beta1.ExternalListenerConfig{ + { + CommonListenerSpec: v1beta1.CommonListenerSpec{Name: "envoygateway"}, + ExternalStartingPort: -1, // TLS enabled + }, + }, + }, + }, + expected: nil, + }, } for _, testCase := range testCases { diff --git a/run-e2e.sh b/run-e2e.sh index 572cb0d4c..772dcbad8 100755 --- a/run-e2e.sh +++ b/run-e2e.sh @@ -1,15 +1,28 @@ #!/bin/bash +# Check if cloud-provider-kind is available in PATH +if ! command -v cloud-provider-kind &> /dev/null; then + echo "Error: cloud-provider-kind is not installed or not in PATH" + echo "Please install it using: brew install cloud-provider-kind" + exit 1 +fi + export IMG_E2E=koperator_e2e_test:latest +export KUBECONFIG=/tmp/kind kind delete clusters e2e-kind kind create cluster --config=tests/e2e/platforms/kind/kind_config.yaml --name=e2e-kind kubectl label node e2e-kind-control-plane node.kubernetes.io/exclude-from-external-load-balancers- docker build . -t koperator_e2e_test kind load docker-image koperator_e2e_test:latest --name e2e-kind kind load docker-image ghcr.io/adobe/koperator/kafka:2.13-3.9.1 --name e2e-kind +kind load docker-image ghcr.io/adobe/zookeeper-operator/zookeeper:3.8.4-0.2.15-adobe-20250923 --name e2e-kind kind load docker-image adobe/cruise-control:3.0.3-adbe-20250804 --name e2e-kind -sudo ~/go/bin/cloud-provider-kind & +sudo cloud-provider-kind &>/tmp/cloud-provider-kind.log & + make test-e2e + +kind delete cluster e2e-kind +sudo pkill -9 -f cloud-provider-kind diff --git a/tests/e2e/const.go b/tests/e2e/const.go index 158cc8e64..58898b9b4 100644 --- a/tests/e2e/const.go +++ b/tests/e2e/const.go @@ -110,6 +110,7 @@ func apiGroupKoperatorDependencies() map[string]string { "zookeeper": "zookeeper.pravega.io", "prometheus": "monitoring.coreos.com", contourName: "projectcontour.io", + "envoy-gateway": "gateway.networking.k8s.io", } } diff --git a/tests/e2e/global.go b/tests/e2e/global.go index fcdeff1c5..79e0ac64a 100644 --- a/tests/e2e/global.go +++ b/tests/e2e/global.go @@ -56,6 +56,25 @@ var ( }, } + // envoyGatewayHelmDescriptor describes the Envoy Gateway Helm component + // The Helm chart installs Gateway API CRDs and Envoy Gateway CRDs automatically + envoyGatewayHelmDescriptor = helmDescriptor{ + Repository: "", + ChartName: "oci://docker.io/envoyproxy/gateway-helm", + ChartVersion: EnvoyGatewayVersion, + ReleaseName: "eg", + Namespace: "envoy-gateway-system", + SetValues: map[string]string{ + "deployment.envoyGateway.resources.limits.cpu": "500m", + "deployment.envoyGateway.resources.limits.memory": "1024Mi", + "deployment.envoyGateway.resources.requests.cpu": "100m", + "deployment.envoyGateway.resources.requests.memory": "256Mi", + }, + HelmExtraArguments: map[string][]string{ + "install": {"--timeout", "10m"}, + }, + } + // koperatorLocalHelmDescriptor describes the Koperator Helm component with // a local chart and version. koperatorLocalHelmDescriptor = func() helmDescriptor { diff --git a/tests/e2e/kcat.go b/tests/e2e/kcat.go index 0c04f4d37..af0db642c 100644 --- a/tests/e2e/kcat.go +++ b/tests/e2e/kcat.go @@ -23,6 +23,11 @@ import ( ginkgo "github.com/onsi/ginkgo/v2" ) +const ( + // kcatTLSParams defines the TLS parameters for kcat when using SSL security protocol + kcatTLSParams = "-X security.protocol=SSL -X ssl.key.location=/ssl/certs/tls.key -X ssl.certificate.location=/ssl/certs/tls.crt -X ssl.ca.location=/ssl/certs/ca.crt" +) + // consumingMessagesInternally consuming messages based on parameters from Kafka cluster. // It returns messages in string slice. func consumingMessagesInternally(kubectlOptions k8s.KubectlOptions, kcatPodName string, internalKafkaAddress string, topicName string, tlsMode bool) (string, error) { @@ -30,7 +35,7 @@ func consumingMessagesInternally(kubectlOptions k8s.KubectlOptions, kcatPodName kcatTLSParameters := "" if tlsMode { - kcatTLSParameters += "-X security.protocol=SSL -X ssl.key.location=/ssl/certs/tls.key -X ssl.certificate.location=/ssl/certs/tls.crt -X ssl.ca.location=/ssl/certs/ca.crt" + kcatTLSParameters += kcatTLSParams } consumedMessages, err := k8s.RunKubectlAndGetOutputContextE(ginkgo.GinkgoT(), @@ -55,7 +60,7 @@ func producingMessagesInternally(kubectlOptions k8s.KubectlOptions, kcatPodName kcatTLSParameters := "" if tlsMode { - kcatTLSParameters += "-X security.protocol=SSL -X ssl.key.location=/ssl/certs/tls.key -X ssl.certificate.location=/ssl/certs/tls.crt -X ssl.ca.location=/ssl/certs/ca.crt" + kcatTLSParameters += kcatTLSParams } _, err := k8s.RunKubectlAndGetOutputContextE(ginkgo.GinkgoT(), diff --git a/tests/e2e/koperator_suite_test.go b/tests/e2e/koperator_suite_test.go index 189f41305..2e2b9f5fe 100644 --- a/tests/e2e/koperator_suite_test.go +++ b/tests/e2e/koperator_suite_test.go @@ -21,8 +21,8 @@ import ( "testing" "github.com/gruntwork-io/terratest/modules/k8s" - ginkgo "github.com/onsi/ginkgo/v2" - gomega "github.com/onsi/gomega" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -58,15 +58,15 @@ var _ = ginkgo.When("Testing e2e test altogether", ginkgo.Ordered, func() { snapshotCluster(snapshottedInfo) testInstall() testInstallZookeeperCluster() - testInstallKafkaCluster("../../config/samples/simplekafkacluster.yaml") + testInstallNoIngressKafkaCluster("Installing Kafka cluster (Zookeeper-based, plaintext, no ingress)", "../../config/samples/simplekafkacluster.yaml") testProduceConsumeInternal() testJmxExporter() testUninstallKafkaCluster() - testInstallKafkaCluster("../../config/samples/simplekafkacluster_ssl.yaml") + testInstallNoIngressKafkaCluster("Installing Kafka cluster (Zookeeper-based, SSL enabled, no ingress)", "../../config/samples/simplekafkacluster_ssl.yaml") testProduceConsumeInternalSSL(defaultTLSSecretName) testJmxExporter() testUninstallKafkaCluster() - testInstallKafkaCluster("../../config/samples/simplekafkacluster_4disk.yaml") + testInstallNoIngressKafkaCluster("Installing Kafka cluster (Zookeeper-based, 4 disks, no ingress)", "../../config/samples/simplekafkacluster_4disk.yaml") testMultiDiskRemoval() testUninstallKafkaCluster() testInstallKafkaCluster("../../config/samples/simplekafkacluster_5broker.yaml") @@ -74,10 +74,30 @@ var _ = ginkgo.When("Testing e2e test altogether", ginkgo.Ordered, func() { testUninstallKafkaCluster() testUninstallZookeeperCluster() // kraft tests - testInstallKafkaCluster("../../config/samples/kraft/simplekafkacluster_kraft.yaml") + testInstallNoIngressKafkaCluster("Installing Kafka cluster (KRaft mode, plaintext, no ingress)", "../../config/samples/kraft/simplekafkacluster_kraft.yaml") testProduceConsumeInternal() testJmxExporter() testUninstallKafkaCluster() + testInstallZookeeperCluster() + testInstallEnvoyKafkaCluster("Installing Kafka cluster (Zookeeper-based, Envoy ingress)", "../../config/samples/simplekafkacluster_with_envoy.yaml") + testProduceConsumeInternal() + testJmxExporter() + testUninstallKafkaCluster() + testUninstallZookeeperCluster() + testInstallEnvoyKafkaCluster("Installing Kafka cluster (KRaft mode, Envoy ingress)", "../../config/samples/kraft/simplekafkacluster_kraft_with_envoy.yaml") + testProduceConsumeInternal() + testJmxExporter() + testUninstallKafkaCluster() + testInstallZookeeperCluster() + testInstallEnvoyGatewayKafkaCluster("Installing Kafka cluster (Zookeeper-based, Envoy Gateway ingress)", "../../config/samples/simplekafkacluster_with_envoygateway.yaml") + testProduceConsumeInternal() + testJmxExporter() + testUninstallEnvoyGatewayKafkaCluster("../../config/samples/simplekafkacluster_with_envoygateway.yaml") + testUninstallZookeeperCluster() + testInstallEnvoyGatewayKafkaCluster("Installing Kafka cluster (KRaft mode, Envoy Gateway ingress)", "../../config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml") + testProduceConsumeInternal() + testJmxExporter() + testUninstallEnvoyGatewayKafkaCluster("../../config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml") testUninstall() snapshotClusterAndCompare(snapshottedInfo) }) diff --git a/tests/e2e/test_install.go b/tests/e2e/test_install.go index 2542da172..f0f643f5c 100644 --- a/tests/e2e/test_install.go +++ b/tests/e2e/test_install.go @@ -19,8 +19,8 @@ import ( "sync" "github.com/gruntwork-io/terratest/modules/k8s" - ginkgo "github.com/onsi/ginkgo/v2" - gomega "github.com/onsi/gomega" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" ) func testInstall() bool { @@ -35,10 +35,10 @@ func testInstall() bool { ginkgo.It("Installing infrastructure components in parallel", func() { var wg sync.WaitGroup - errChan := make(chan error, 2) + errChan := make(chan error, 3) - // Install cert-manager and Contour in parallel - independent charts, no install-order dependency - wg.Add(2) + // Install cert-manager, Contour, and Envoy Gateway in parallel - independent charts, no install-order dependency + wg.Add(3) go func() { defer wg.Done() @@ -56,6 +56,14 @@ func testInstall() bool { } }() + go func() { + defer wg.Done() + ginkgo.By("Installing Envoy Gateway Helm chart") + if installErr := envoyGatewayHelmDescriptor.installHelmChart(kubectlOptions); installErr != nil { + errChan <- installErr + } + }() + wg.Wait() close(errChan) @@ -64,6 +72,17 @@ func testInstall() bool { } }) + ginkgo.It("Creating Envoy Gateway GatewayClass", func() { + gatewayClassManifest := `apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller` + err = applyK8sResourceManifestFromString(kubectlOptions, gatewayClassManifest) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("Installing dependency operators in parallel", func() { var wg sync.WaitGroup errChan := make(chan error, 2) diff --git a/tests/e2e/test_install_cluster.go b/tests/e2e/test_install_cluster.go deleted file mode 100644 index d5ca9c624..000000000 --- a/tests/e2e/test_install_cluster.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates -// Copyright 2025 Adobe. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "github.com/gruntwork-io/terratest/modules/k8s" - ginkgo "github.com/onsi/ginkgo/v2" - gomega "github.com/onsi/gomega" -) - -func testInstallZookeeperCluster() bool { - return ginkgo.When("Installing Zookeeper cluster", func() { - var kubectlOptions k8s.KubectlOptions - var err error - - ginkgo.It("Acquiring K8s config and context", func() { - kubectlOptions, err = kubectlOptionsForCurrentContext() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - kubectlOptions.Namespace = zookeeperOperatorHelmDescriptor.Namespace - requireCreatingZookeeperCluster(kubectlOptions) - }) -} - -func testInstallKafkaCluster(kafkaClusterManifestPath string) bool { //nolint:unparam // Note: respecting Ginkgo testing interface by returning bool. - return ginkgo.When("Installing Kafka cluster", func() { - var kubectlOptions k8s.KubectlOptions - var err error - - ginkgo.It("Acquiring K8s config and context", func() { - kubectlOptions, err = kubectlOptionsForCurrentContext() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - kubectlOptions.Namespace = koperatorLocalHelmDescriptor.Namespace - requireCreatingKafkaCluster(kubectlOptions, kafkaClusterManifestPath) - }) -} diff --git a/tests/e2e/test_install_kafka_cluster.go b/tests/e2e/test_install_kafka_cluster.go new file mode 100644 index 000000000..1c0834e5a --- /dev/null +++ b/tests/e2e/test_install_kafka_cluster.go @@ -0,0 +1,82 @@ +// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +func testInstallZookeeperCluster() bool { + return ginkgo.When("Installing Zookeeper cluster (required for Zookeeper-based Kafka)", func() { + var kubectlOptions k8s.KubectlOptions + var err error + + ginkgo.It("Acquiring K8s config and context", func() { + kubectlOptions, err = kubectlOptionsForCurrentContext() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + kubectlOptions.Namespace = zookeeperOperatorHelmDescriptor.Namespace + requireCreatingZookeeperCluster(kubectlOptions) + }) +} + +func testInstallNoIngressKafkaCluster(clusterDescription, kafkaClusterManifestPath string) bool { //nolint:unparam // Note: respecting Ginkgo testing interface by returning bool. + return ginkgo.When(clusterDescription, func() { + var kubectlOptions k8s.KubectlOptions + var err error + + ginkgo.It("Acquiring K8s config and context", func() { + kubectlOptions, err = kubectlOptionsForCurrentContext() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + kubectlOptions.Namespace = koperatorLocalHelmDescriptor.Namespace + requireCreatingKafkaCluster(kubectlOptions, kafkaClusterManifestPath) + }) +} + +func testInstallEnvoyKafkaCluster(clusterDescription, kafkaClusterManifestPath string) bool { //nolint:unparam // Note: respecting Ginkgo testing interface by returning bool. + return ginkgo.When(clusterDescription, func() { + var kubectlOptions k8s.KubectlOptions + var err error + + ginkgo.It("Acquiring K8s config and context", func() { + kubectlOptions, err = kubectlOptionsForCurrentContext() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + kubectlOptions.Namespace = koperatorLocalHelmDescriptor.Namespace + requireCreatingKafkaCluster(kubectlOptions, kafkaClusterManifestPath) + }) +} + +func testInstallEnvoyGatewayKafkaCluster(clusterDescription, kafkaClusterManifestPath string) bool { //nolint:unparam // Note: respecting Ginkgo testing interface by returning bool. + return ginkgo.When(clusterDescription, func() { + var kubectlOptions k8s.KubectlOptions + var err error + + ginkgo.It("Acquiring K8s config and context", func() { + kubectlOptions, err = kubectlOptionsForCurrentContext() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + kubectlOptions.Namespace = koperatorLocalHelmDescriptor.Namespace + requireCreatingKafkaCluster(kubectlOptions, kafkaClusterManifestPath) + }) +} diff --git a/tests/e2e/test_snapshot.go b/tests/e2e/test_snapshot.go index 92d0667a7..4ac3c7601 100644 --- a/tests/e2e/test_snapshot.go +++ b/tests/e2e/test_snapshot.go @@ -21,9 +21,8 @@ import ( "strings" "github.com/gruntwork-io/terratest/modules/k8s" - ginkgo "github.com/onsi/ginkgo/v2" - gomega "github.com/onsi/gomega" - "github.com/onsi/gomega/format" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -41,12 +40,23 @@ func (s *clusterSnapshot) Resources() []metav1.PartialObjectMetadata { func (s *clusterSnapshot) ResourcesAsComparisonType() []localComparisonPartialObjectMetadataType { var localList []localComparisonPartialObjectMetadataType for _, r := range s.resources { - // Filter out cert-manager related resources to avoid comparison failures - // when cert-manager is not fully cleaned up during uninstall + // Filter out cert-manager and envoy-gateway related resources to avoid comparison failures + // when these components are not fully cleaned up during uninstall resourceName := r.GetName() if strings.Contains(resourceName, "cert-manager") || strings.Contains(resourceName, "acme.cert-manager") { continue } + // Filter out Envoy Gateway and Gateway API resources (CRDs, RBAC, APIServices) + if strings.Contains(resourceName, "gateway.envoyproxy.io") || + strings.Contains(resourceName, "gateway.networking.x-k8s.io") || + strings.Contains(resourceName, "gateway.networking.k8s.io") || + strings.Contains(resourceName, "eg-gateway-helm-certgen") { + continue + } + // Filter out Kind cluster infrastructure resources + if resourceName == "cloud-provider-kind" { + continue + } localList = append(localList, localComparisonPartialObjectMetadataType{ GVK: r.GroupVersionKind(), @@ -162,13 +172,57 @@ func snapshotClusterAndCompare(snapshottedInitialInfo *clusterSnapshot) bool { snapshotCluster(snapshottedCurrentInfo) ginkgo.It("Checking resources list", func() { - // Temporarily increase maximum output length (default 4000) to fit more objects in the printed diff. - // Only doing this here because other assertions typically don't run against objects with this many elements. - initialMaxLength := format.MaxLength - defer func() { format.MaxLength = initialMaxLength }() - format.MaxLength = 9000 + current := snapshottedCurrentInfo.ResourcesAsComparisonType() + initial := snapshottedInitialInfo.ResourcesAsComparisonType() + + // Calculate differences for better error reporting + var extra []localComparisonPartialObjectMetadataType + var missing []localComparisonPartialObjectMetadataType + + for _, c := range current { + found := false + for _, i := range initial { + if c.GVK == i.GVK && c.Namespace == i.Namespace && c.Name == i.Name { + found = true + break + } + } + if !found { + extra = append(extra, c) + } + } + + for _, i := range initial { + found := false + for _, c := range current { + if c.GVK == i.GVK && c.Namespace == i.Namespace && c.Name == i.Name { + found = true + break + } + } + if !found { + missing = append(missing, i) + } + } + + // If there are differences, print them clearly and fail with a simple message + if len(extra) > 0 || len(missing) > 0 { + if len(extra) > 0 { + ginkgo.GinkgoWriter.Printf("\n=== EXTRA RESOURCES (present now but not in initial snapshot) ===\n") + for _, r := range extra { + ginkgo.GinkgoWriter.Printf(" %s/%s %s (namespace: %q)\n", r.GVK.Group, r.GVK.Kind, r.Name, r.Namespace) + } + } + + if len(missing) > 0 { + ginkgo.GinkgoWriter.Printf("\n=== MISSING RESOURCES (present in initial snapshot but not now) ===\n") + for _, r := range missing { + ginkgo.GinkgoWriter.Printf(" %s/%s %s (namespace: %q)\n", r.GVK.Group, r.GVK.Kind, r.Name, r.Namespace) + } + } - gomega.Expect(snapshottedCurrentInfo.ResourcesAsComparisonType()).To(gomega.ConsistOf(snapshottedInitialInfo.ResourcesAsComparisonType())) + ginkgo.Fail(fmt.Sprintf("Cluster resources mismatch: %d extra, %d missing (see details above)", len(extra), len(missing))) + } }) }) } diff --git a/tests/e2e/test_uninstall.go b/tests/e2e/test_uninstall.go index cd026b919..f7cf1debf 100644 --- a/tests/e2e/test_uninstall.go +++ b/tests/e2e/test_uninstall.go @@ -63,5 +63,10 @@ func testUninstall() bool { ConfigPath: kubectlOptions.ConfigPath, Namespace: contourIngressControllerHelmDescriptor.Namespace, }) + requireUninstallingEnvoyGateway(k8s.KubectlOptions{ + ContextName: kubectlOptions.ContextName, + ConfigPath: kubectlOptions.ConfigPath, + Namespace: envoyGatewayHelmDescriptor.Namespace, + }) }) } diff --git a/tests/e2e/test_uninstall_cluster.go b/tests/e2e/test_uninstall_cluster.go index 46d505330..dcd447d02 100644 --- a/tests/e2e/test_uninstall_cluster.go +++ b/tests/e2e/test_uninstall_cluster.go @@ -50,3 +50,31 @@ func testUninstallKafkaCluster() bool { //nolint:unparam // Note: respecting Gin requireDeleteKafkaCluster(kubectlOptions, kafkaClusterName) }) } + +func testUninstallEnvoyGatewayKafkaCluster(manifestPath string) bool { //nolint:unparam // Note: respecting Ginkgo testing interface by returning bool. + return ginkgo.When("Uninstalling Envoy Gateway Kafka cluster and cert-manager resources", func() { + var kubectlOptions k8s.KubectlOptions + var err error + + ginkgo.It("Acquiring K8s config and context", func() { + kubectlOptions, err = kubectlOptionsForCurrentContext() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + kubectlOptions.Namespace = koperatorLocalHelmDescriptor.Namespace + + // Delete KafkaCluster CR first + requireDeleteKafkaCluster(kubectlOptions, kafkaClusterName) + + // Delete cert-manager resources (Certificate and Issuer) + ginkgo.It("Deleting cert-manager Certificate", func() { + err := deleteK8sResourceNoErrNotFound(kubectlOptions, defaultDeletionTimeout, "certificate", "envoygateway-tls-cert") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.It("Deleting cert-manager Issuer", func() { + err := deleteK8sResourceNoErrNotFound(kubectlOptions, defaultDeletionTimeout, "issuer", "envoygateway-selfsigned-issuer") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + }) +} diff --git a/tests/e2e/types.go b/tests/e2e/types.go index 78e1728ce..011cd8e9d 100644 --- a/tests/e2e/types.go +++ b/tests/e2e/types.go @@ -22,10 +22,11 @@ import ( ) type dependencyCRDsType struct { - zookeeper []string - prometheus []string - certManager []string - contour []string + zookeeper []string + prometheus []string + certManager []string + contour []string + envoyGateway []string } func (c *dependencyCRDsType) Zookeeper() []string { @@ -40,6 +41,9 @@ func (c *dependencyCRDsType) CertManager() []string { func (c *dependencyCRDsType) Contour() []string { return c.contour } +func (c *dependencyCRDsType) EnvoyGateway() []string { + return c.envoyGateway +} func (c *dependencyCRDsType) Initialize(kubectlOptions k8s.KubectlOptions) error { var err error @@ -51,6 +55,10 @@ func (c *dependencyCRDsType) Initialize(kubectlOptions k8s.KubectlOptions) error if err != nil { return fmt.Errorf("initialize Contour Ingress Controller CRDs error: %w", err) } + c.envoyGateway, err = listK8sResourceKinds(kubectlOptions, apiGroupKoperatorDependencies()["envoy-gateway"]) + if err != nil { + return fmt.Errorf("initialize Envoy Gateway CRDs error: %w", err) + } c.prometheus, err = listK8sResourceKinds(kubectlOptions, apiGroupKoperatorDependencies()["prometheus"]) if err != nil { return fmt.Errorf("initialize Prometheus CRDs error: %w", err) diff --git a/tests/e2e/uninstall.go b/tests/e2e/uninstall.go index 5ab3f6f06..3d9b8a2b2 100644 --- a/tests/e2e/uninstall.go +++ b/tests/e2e/uninstall.go @@ -239,7 +239,7 @@ func requireRemoveCertManagerCRDs(kubectlOptions k8s.KubectlOptions) { }) } func requireUninstallingContour(kubectlOptions k8s.KubectlOptions) { - ginkgo.When("Uninstalling zookeeper-operator", func() { + ginkgo.When("Uninstalling contour", func() { requireUninstallingContourHelmChart(kubectlOptions) requireRemoveContourCRDs(kubectlOptions) requireRemoveNamespace(kubectlOptions, contourIngressControllerHelmDescriptor.Namespace) @@ -282,6 +282,70 @@ func requireRemoveContourCRDs(kubectlOptions k8s.KubectlOptions) { }) } +func requireUninstallingEnvoyGateway(kubectlOptions k8s.KubectlOptions) { + ginkgo.When("Uninstalling Envoy Gateway", func() { + requireUninstallingEnvoyGatewayHelmChart(kubectlOptions) + requireRemoveEnvoyGatewayCRDs(kubectlOptions) + requireRemoveNamespace(kubectlOptions, envoyGatewayHelmDescriptor.Namespace) + }) +} + +func requireUninstallingEnvoyGatewayHelmChart(kubectlOptions k8s.KubectlOptions) { + ginkgo.It("Uninstalling Envoy Gateway Helm chart", func() { + err := envoyGatewayHelmDescriptor.uninstallHelmChart(kubectlOptions, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Cleaning up Envoy Gateway Helm hook resources") + // Envoy Gateway Helm chart uses hooks that create resources not cleaned up by helm uninstall + // Explicitly delete known leftover resources + + // Delete ServiceAccount in envoy-gateway-system namespace + namespacedOpts := kubectlOptions + namespacedOpts.Namespace = envoyGatewayHelmDescriptor.Namespace + err = deleteK8sResourceNoErrNotFound(namespacedOpts, defaultDeletionTimeout, "serviceaccount", "eg-gateway-helm-certgen") + if err != nil && !isKubectlNotFoundError(err) { + ginkgo.By(fmt.Sprintf("Warning: Failed to delete ServiceAccount eg-gateway-helm-certgen: %v", err)) + } + + // Delete MutatingWebhookConfiguration (cluster-scoped) + // Note: The full name includes the namespace suffix + clusterOpts := kubectlOptions + clusterOpts.Namespace = "" + webhookName := fmt.Sprintf("envoy-gateway-topology-injector.%s", envoyGatewayHelmDescriptor.Namespace) + err = deleteK8sResourceNoErrNotFound(clusterOpts, defaultDeletionTimeout, "mutatingwebhookconfiguration", webhookName) + if err != nil && !isKubectlNotFoundError(err) { + ginkgo.By(fmt.Sprintf("Warning: Failed to delete MutatingWebhookConfiguration %s: %v", webhookName, err)) + } + + ginkgo.By("Verifying Envoy Gateway helm chart resources cleanup") + + k8sResourceKinds, err := listK8sResourceKinds(kubectlOptions, "") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + envoyGatewayAvailableResourceKinds := stringSlicesInstersect(dependencyCRDs.EnvoyGateway(), k8sResourceKinds) + envoyGatewayAvailableResourceKinds = append(envoyGatewayAvailableResourceKinds, basicK8sResourceKinds()...) + + remainedResources, err := getK8sResources(kubectlOptions, + envoyGatewayAvailableResourceKinds, + fmt.Sprintf(managedByHelmLabelTemplate, envoyGatewayHelmDescriptor.ReleaseName), + "", + kubectlArgGoTemplateKindNameNamespace, + "--all-namespaces") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + gomega.Expect(remainedResources).Should(gomega.BeEmpty()) + }) +} + +func requireRemoveEnvoyGatewayCRDs(kubectlOptions k8s.KubectlOptions) { + ginkgo.It("Removing Envoy Gateway CRDs", func() { + for _, crd := range dependencyCRDs.EnvoyGateway() { + err := deleteK8sResourceNoErrNotFound(kubectlOptions, defaultDeletionTimeout, crdKind, crd) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + } + }) +} + // requireRemoveNamespace deletes the indicated namespace object func requireRemoveNamespace(kubectlOptions k8s.KubectlOptions, namespace string) { ginkgo.It(fmt.Sprintf("Removing namespace %s", namespace), func() { diff --git a/tests/e2e/uninstall_cluster.go b/tests/e2e/uninstall_cluster.go index 79434f90d..3d554bbbd 100644 --- a/tests/e2e/uninstall_cluster.go +++ b/tests/e2e/uninstall_cluster.go @@ -36,7 +36,7 @@ func requireDeleteKafkaCluster(kubectlOptions k8s.KubectlOptions, name string) { gomega.Eventually(context.Background(), func() []string { ginkgo.By("Verifying the Kafka cluster resource cleanup") - // Check only those Koperator related resource types we have in K8s (istio usecase) + // Check only those Koperator related resource types we have in K8s k8sResourceKinds, err := listK8sResourceKinds(kubectlOptions, "") gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) diff --git a/tests/e2e/versions.go b/tests/e2e/versions.go index fb078b19f..ca12926d7 100644 --- a/tests/e2e/versions.go +++ b/tests/e2e/versions.go @@ -24,6 +24,9 @@ const ( // ContourVersion is the version of Contour ingress controller Helm chart ContourVersion = "0.6.0" // renovate: datasource=helm depName=contour registryUrl=https://projectcontour.github.io/helm-charts + // EnvoyGatewayVersion is the version of Envoy Gateway Helm chart + EnvoyGatewayVersion = "v1.8.2" // renovate: datasource=helm depName=gateway-helm registryUrl=https://gateway.envoyproxy.io + // PrometheusOperatorVersion is the version of kube-prometheus-stack Helm chart PrometheusOperatorVersion = "88.1.5" // renovate: datasource=helm depName=kube-prometheus-stack registryUrl=https://prometheus-community.github.io/helm-charts From 794dcde63c475ea7c9a7eccf3ce508527dab0e73 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Mon, 10 Aug 2026 10:13:52 +0200 Subject: [PATCH 2/6] update --- ...eway.networking.k8s.io_gatewayclasses.yaml | 11 +- .../gateway.networking.k8s.io_gateways.yaml | 34 +- .../gateway.networking.k8s.io_tcproutes.yaml | 744 +++++++++++++++++- go.mod | 10 +- go.sum | 20 +- tests/e2e/go.mod | 4 +- tests/e2e/go.sum | 8 +- .../go-cruise-control/integration_test/go.mod | 2 +- .../go-cruise-control/integration_test/go.sum | 4 +- .../banzaicloud/operator-tools/go.mod | 13 +- .../banzaicloud/operator-tools/go.sum | 48 +- 11 files changed, 805 insertions(+), 93 deletions(-) diff --git a/config/test/crd/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml b/config/test/crd/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml index 15412b869..ea55d0dad 100644 --- a/config/test/crd/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml +++ b/config/test/crd/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml @@ -3,7 +3,7 @@ kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.0 + gateway.networking.k8s.io/bundle-version: v1.6.1 gateway.networking.k8s.io/channel: standard name: gatewayclasses.gateway.networking.k8s.io spec: @@ -54,6 +54,9 @@ spec: Gateway is not deleted while in use. GatewayClass is a Cluster level resource. + + A GatewayClass name SHOULD be compliant with RFC 1035, consisting of a maximum of 63 lower case alphanumeric + characters or hyphens ('-'), and MUST start and end with an alphanumeric character. properties: apiVersion: description: |- @@ -507,9 +510,3 @@ spec: storage: false subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: null - storedVersions: null diff --git a/config/test/crd/gateway-api/gateway.networking.k8s.io_gateways.yaml b/config/test/crd/gateway-api/gateway.networking.k8s.io_gateways.yaml index 169e74fcb..6de4376f2 100644 --- a/config/test/crd/gateway-api/gateway.networking.k8s.io_gateways.yaml +++ b/config/test/crd/gateway-api/gateway.networking.k8s.io_gateways.yaml @@ -3,7 +3,7 @@ kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.0 + gateway.networking.k8s.io/bundle-version: v1.6.1 gateway.networking.k8s.io/channel: standard name: gateways.gateway.networking.k8s.io spec: @@ -38,6 +38,8 @@ spec: description: |- Gateway represents an instance of a service-traffic handling infrastructure by binding Listeners to a set of IP addresses. + A Gateway name SHOULD be compliant with RFC 1035, consisting of a maximum of 63 lower case alphanumeric + characters or hyphens ('-'), and MUST start and end with an alphanumeric character. properties: apiVersion: description: |- @@ -248,7 +250,7 @@ spec: An implementation may chose to add additional implementation-specific annotations as they see fit. Support: Extended - maxProperties: 8 + maxProperties: 16 type: object x-kubernetes-validations: - message: Annotation keys must be in the form of an optional @@ -470,6 +472,12 @@ spec: request to "foo.example.com" SHOULD only be routed using routes attached to the "foo.example.com" Listener (and not the "*.example.com" Listener). + If traffic to a Gateway does not match any Listener's hostname (or if + the Listener does not specify a hostname and the request does not match + any attached Route), the request MUST be rejected. The specific mechanism + for rejection depends on the protocol: HTTP returns a 404 status code, + while gRPC returns an Unimplemented status code. + This concept is known as "Listener Isolation", and it is an Extended feature of Gateway API. Implementations that do not support Listener Isolation MUST clearly document this, and MUST NOT claim support for the @@ -1110,7 +1118,7 @@ spec: - kind - name type: object - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -1275,7 +1283,7 @@ spec: - kind - name type: object - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -1876,7 +1884,7 @@ spec: An implementation may chose to add additional implementation-specific annotations as they see fit. Support: Extended - maxProperties: 8 + maxProperties: 16 type: object x-kubernetes-validations: - message: Annotation keys must be in the form of an optional @@ -2098,6 +2106,12 @@ spec: request to "foo.example.com" SHOULD only be routed using routes attached to the "foo.example.com" Listener (and not the "*.example.com" Listener). + If traffic to a Gateway does not match any Listener's hostname (or if + the Listener does not specify a hostname and the request does not match + any attached Route), the request MUST be rejected. The specific mechanism + for rejection depends on the protocol: HTTP returns a 404 status code, + while gRPC returns an Unimplemented status code. + This concept is known as "Listener Isolation", and it is an Extended feature of Gateway API. Implementations that do not support Listener Isolation MUST clearly document this, and MUST NOT claim support for the @@ -2738,7 +2752,7 @@ spec: - kind - name type: object - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -2903,7 +2917,7 @@ spec: - kind - name type: object - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -3275,9 +3289,3 @@ spec: storage: false subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: null - storedVersions: null diff --git a/config/test/crd/gateway-api/gateway.networking.k8s.io_tcproutes.yaml b/config/test/crd/gateway-api/gateway.networking.k8s.io_tcproutes.yaml index 88211eff0..23515e298 100644 --- a/config/test/crd/gateway-api/gateway.networking.k8s.io_tcproutes.yaml +++ b/config/test/crd/gateway-api/gateway.networking.k8s.io_tcproutes.yaml @@ -3,7 +3,7 @@ kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.0 + gateway.networking.k8s.io/bundle-version: v1.6.1 gateway.networking.k8s.io/channel: experimental name: tcproutes.gateway.networking.k8s.io spec: @@ -21,6 +21,728 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + TCPRoute provides a way to route TCP requests. When combined with a Gateway + listener, it can be used to forward connections on the port specified by the + listener to a set of backends specified by the TCPRoute. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of TCPRoute. + properties: + parentRefs: + description: |- + ParentRefs references the resources (usually Gateways) that a Route wants + to be attached to. Note that the referenced parent resource needs to + allow this for the attachment to be complete. For Gateways, that means + the Gateway needs to allow attachment from Routes of this kind and + namespace. For Services, that means the Service must either be in the same + namespace for a "producer" route, or the mesh implementation must support + and allow "consumer" routes for the referenced Service. ReferenceGrant is + not applicable for governing ParentRefs to Services - it is not possible to + create a "producer" route for a Service in a different namespace from the + Route. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + ParentRefs must be _distinct_. This means either that: + + * They select different objects. If this is the case, then parentRef + entries are distinct. In terms of fields, this means that the + multi-part key defined by `group`, `kind`, `namespace`, and `name` must + be unique across all parentRef entries in the Route. + * They do not select different objects, but for each optional field used, + each ParentRef that selects the same object must set the same set of + optional fields to different values. If one ParentRef sets a + combination of optional fields, all must set the same combination. + + Some examples: + + * If one ParentRef sets `sectionName`, all ParentRefs referencing the + same object must also set `sectionName`. + * If one ParentRef sets `port`, all ParentRefs referencing the same + object must also set `port`. + * If one ParentRef sets `sectionName` and `port`, all ParentRefs + referencing the same object must also set `sectionName` and `port`. + + It is possible to separately reference multiple distinct objects that may + be collapsed by an implementation. For example, some implementations may + choose to merge compatible Gateway Listeners together. If that is the + case, the list of routes attached to those resources should also be + merged. + + Note that for ParentRefs that cross namespace boundaries, there are specific + rules. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example, + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable other kinds of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + items: + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + maxItems: 32 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: sectionName or port must be specified when parentRefs includes + 2 or more references to the same parent + rule: 'self.all(p1, self.all(p2, p1.group == p2.group && p1.kind + == p2.kind && p1.name == p2.name && (((!has(p1.__namespace__) + || p1.__namespace__ == '''') && (!has(p2.__namespace__) || p2.__namespace__ + == '''')) || (has(p1.__namespace__) && has(p2.__namespace__) && + p1.__namespace__ == p2.__namespace__)) ? ((!has(p1.sectionName) + || p1.sectionName == '''') == (!has(p2.sectionName) || p2.sectionName + == '''') && (!has(p1.port) || p1.port == 0) == (!has(p2.port) + || p2.port == 0)): true))' + - message: sectionName or port must be unique when parentRefs includes + 2 or more references to the same parent + rule: self.all(p1, self.exists_one(p2, p1.group == p2.group && p1.kind + == p2.kind && p1.name == p2.name && (((!has(p1.__namespace__) + || p1.__namespace__ == '') && (!has(p2.__namespace__) || p2.__namespace__ + == '')) || (has(p1.__namespace__) && has(p2.__namespace__) && + p1.__namespace__ == p2.__namespace__ )) && (((!has(p1.sectionName) + || p1.sectionName == '') && (!has(p2.sectionName) || p2.sectionName + == '')) || ( has(p1.sectionName) && has(p2.sectionName) && p1.sectionName + == p2.sectionName)) && (((!has(p1.port) || p1.port == 0) && (!has(p2.port) + || p2.port == 0)) || (has(p1.port) && has(p2.port) && p1.port + == p2.port)))) + rules: + description: Rules are a list of TCP matchers and actions. + items: + description: TCPRouteRule is the configuration for a given rule. + properties: + backendRefs: + description: |- + BackendRefs defines the backend(s) where matching requests should be + sent. If unspecified or invalid (refers to a nonexistent resource or a + Service with no endpoints), the underlying implementation MUST actively + reject connection attempts to this backend. Connection rejections must + respect weight; if an invalid backend is requested to have 80% of + connections, then 80% of connections must be rejected instead. + + Support: Core for Kubernetes Service + items: + description: |- + BackendRef defines how a Route should forward a request to a Kubernetes + resource. + + Note that when a namespace different than the local namespace is specified, a + ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + + When the BackendRef points to a Kubernetes Service, implementations SHOULD + honor the appProtocol field if it is set for the target Service Port. + + Implementations supporting appProtocol SHOULD recognize the Kubernetes + Standard Application Protocols defined in KEP-3726. + + If a Service appProtocol isn't specified, an implementation MAY infer the + backend protocol through its own means. Implementations MAY infer the + protocol from the Route type referring to the backend Service. + + If a Route is not able to send traffic to the backend using the specified + protocol then the backend is considered invalid. Implementations MUST set the + "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. + + + Note that when the BackendTLSPolicy object is enabled by the implementation, + there are some extra rules about validity to consider here. See the fields + where this struct is used for more information about the exact behavior. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + Support: Core (Services with a type other than ExternalName) + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + weight: + default: 1 + description: |- + Weight specifies the proportion of requests forwarded to the referenced + backend. This is computed as weight/(sum of all weights in this + BackendRefs list). For non-zero values, there may be some epsilon from + the exact proportion defined here depending on the precision an + implementation supports. Weight is not a percentage and the sum of + weights does not need to equal 100. + + If only one backend is specified and it has a weight greater than 0, 100% + of the traffic is forwarded to that backend. If weight is set to 0, no + traffic should be forwarded for this entry. If unspecified, weight + defaults to 1. + + Support for this field varies based on the context where used. + format: int32 + maximum: 1000000 + minimum: 0 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind == ''Service'') + ? has(self.port) : true' + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + name: + description: Name is the name of the route rule. This name MUST + be unique within a Route if it is set. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - backendRefs + type: object + maxItems: 1 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + useDefaultGateways: + description: |- + UseDefaultGateways indicates the default Gateway scope to use for this + Route. If unset (the default) or set to None, the Route will not be + attached to any default Gateway; if set, it will be attached to any + default Gateway supporting the named scope, subject to the usual rules + about which Routes a Gateway is allowed to claim. + + Think carefully before using this functionality! The set of default + Gateways supporting the requested scope can change over time without + any notice to the Route author, and in many situations it will not be + appropriate to request a default Gateway for a given Route -- for + example, a Route with specific security requirements should almost + certainly not use a default Gateway. + enum: + - All + - None + type: string + required: + - rules + type: object + status: + description: Status defines the current state of TCPRoute. + properties: + parents: + description: |- + Parents is a list of parent resources (usually Gateways) that are + associated with the route, and the status of the route with respect to + each parent. When this route attaches to a parent, the controller that + manages the parent must add an entry to this list when the controller + first sees the route and should update the entry as appropriate when the + route or gateway is modified. + + Note that parent references that cannot be resolved by an implementation + of this API will not be added to this list. Implementations of this API + can only populate Route status for the Gateways/parent resources they are + responsible for. + + A maximum of 32 Gateways will be represented in this list. An empty list + means the route has not been attached to any Gateway. + items: + description: |- + RouteParentStatus describes the status of a route with respect to an + associated Parent. + properties: + conditions: + description: |- + Conditions describes the status of the route with respect to the Gateway. + Note that the route's availability is also subject to the Gateway's own + status conditions and listener status. + + If the Route's ParentRef specifies an existing Gateway that supports + Routes of this kind AND that Gateway's controller has sufficient access, + then that Gateway's controller MUST set the "Accepted" condition on the + Route, to indicate whether the route has been accepted or rejected by the + Gateway, and why. + + A Route MUST be considered "Accepted" if at least one of the Route's + rules is implemented by the Gateway. + + There are a number of cases where the "Accepted" condition may not be set + due to lack of controller visibility, that includes when: + + * The Route refers to a nonexistent parent. + * The Route is of a type that the controller does not support. + * The Route is in a namespace to which the controller does not have access. + items: + description: Condition contains details for one aspect of + the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controllerName: + description: |- + ControllerName is a domain/path string that indicates the name of the + controller that wrote this status. This corresponds with the + controllerName field on GatewayClass. + + Example: "example.net/gateway-controller". + + The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are + valid Kubernetes names + (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). + + Controllers MUST populate this field when writing status. Controllers should ensure that + entries to status populated with their ControllerName are cleaned up when they are no + longer necessary. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + parentRef: + description: |- + ParentRef corresponds with a ParentRef in the spec that this + RouteParentStatus struct describes the status of. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + required: + - conditions + - controllerName + - parentRef + type: object + maxItems: 32 + type: array + x-kubernetes-list-type: atomic + required: + - parents + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + deprecated: true + deprecationWarning: The v1alpha2 version of TCPRoute has been deprecated and will + be removed in a future release of the API. Please upgrade to v1. name: v1alpha2 schema: openAPIV3Schema: @@ -300,12 +1022,6 @@ spec: connections, then 80% of connections must be rejected instead. Support: Core for Kubernetes Service - - Support: Extended for Kubernetes ServiceImport - - Support: Implementation-specific for any other resource - - Support for weight: Extended items: description: |- BackendRef defines how a Route should forward a request to a Kubernetes @@ -428,10 +1144,8 @@ spec: type: array x-kubernetes-list-type: atomic name: - description: |- - Name is the name of the route rule. This name MUST be unique within a Route if it is set. - - Support: Extended + description: Name is the name of the route rule. This name MUST + be unique within a Route if it is set. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -745,12 +1459,6 @@ spec: - spec type: object served: true - storage: true + storage: false subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: null - storedVersions: null diff --git a/go.mod b/go.mod index cfb708bf8..45f76d13c 100644 --- a/go.mod +++ b/go.mod @@ -61,8 +61,8 @@ require ( golang.org/x/mod v0.38.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/tools v0.48.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260729162451-8efbd57d26e0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260729162451-8efbd57d26e0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260807164820-c8921c73eeea // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260807164820-c8921c73eeea // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect @@ -98,7 +98,7 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.19.1 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -106,7 +106,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pierrec/lz4/v4 v4.1.27 // indirect + github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.24.1 // indirect @@ -133,7 +133,7 @@ require ( k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 // indirect - sigs.k8s.io/gateway-api v1.6.0 + sigs.k8s.io/gateway-api v1.6.1 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/yaml v1.6.0 ) diff --git a/go.sum b/go.sum index 65697552b..b92f052b1 100644 --- a/go.sum +++ b/go.sum @@ -136,8 +136,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= -github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -170,8 +170,8 @@ github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0 h1:2nosf3P75OZv2/ZO/9Px5ZgZ5gbKrzA3joN1QMfOGMQ= github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0/go.mod h1:lAVhWwbNaveeJmxrxuSTxMgKpF6DjnuVpn6T8WiBwYQ= -github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= -github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= +github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -292,10 +292,10 @@ golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/api v0.0.0-20260729162451-8efbd57d26e0 h1:ybvH/ZpOcpCrjtkb7oW/fdlzbEmRVeumw19SRQmNFKU= -google.golang.org/genproto/googleapis/api v0.0.0-20260729162451-8efbd57d26e0/go.mod h1:HJ9MpJLeDSstBkx1LILTpd5f41ADSMZcTPypw02qEGw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260729162451-8efbd57d26e0 h1:mJiOtnGp0k/BcSgdu03G2NwnscCfCH+h2QKUBZr18KI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260729162451-8efbd57d26e0/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260807164820-c8921c73eeea h1:Jifw/kjs/r3B0uszvls/m3c3tmZs2YHGM9C+rvxP9gY= +google.golang.org/genproto/googleapis/api v0.0.0-20260807164820-c8921c73eeea/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260807164820-c8921c73eeea h1:kVhQEPTpKQahD5+JSBTfBB19wcgQTTjAIn45MBqnyHk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260807164820-c8921c73eeea/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -327,8 +327,8 @@ k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0x k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM= sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= sigs.k8s.io/controller-runtime v0.24.1/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= -sigs.k8s.io/gateway-api v1.6.0 h1:735YBRj5NXFrOGX0GoSjwzUIzbz8kiEOfADsqHFmHgE= -sigs.k8s.io/gateway-api v1.6.0/go.mod h1:FVfx3t389ybeXOqvDghLbdvJdSCfI/PReqCUI3lu3mY= +sigs.k8s.io/gateway-api v1.6.1 h1:mock6phZbI6rvZerwrVNk7hVNymQgHo+6sJ81Ia7ftY= +sigs.k8s.io/gateway-api v1.6.1/go.mod h1:FVfx3t389ybeXOqvDghLbdvJdSCfI/PReqCUI3lu3mY= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= diff --git a/tests/e2e/go.mod b/tests/e2e/go.mod index 6483d0f19..ea5412353 100644 --- a/tests/e2e/go.mod +++ b/tests/e2e/go.mod @@ -123,7 +123,7 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.19.1 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect github.com/mattn/go-colorable v0.1.15 // indirect @@ -138,7 +138,7 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0 // indirect - github.com/pierrec/lz4/v4 v4.1.27 // indirect + github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pquerna/otp v1.4.0 // indirect diff --git a/tests/e2e/go.sum b/tests/e2e/go.sum index e749de88e..559b57f3d 100644 --- a/tests/e2e/go.sum +++ b/tests/e2e/go.sum @@ -250,8 +250,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= -github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -301,8 +301,8 @@ github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0 h1:2nosf3P75OZv2/ZO/9Px5ZgZ5gbKrzA3joN1QMfOGMQ= github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0/go.mod h1:lAVhWwbNaveeJmxrxuSTxMgKpF6DjnuVpn6T8WiBwYQ= -github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= -github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= +github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/third_party/github.com/banzaicloud/go-cruise-control/integration_test/go.mod b/third_party/github.com/banzaicloud/go-cruise-control/integration_test/go.mod index 0a02a82a7..1480903de 100644 --- a/third_party/github.com/banzaicloud/go-cruise-control/integration_test/go.mod +++ b/third_party/github.com/banzaicloud/go-cruise-control/integration_test/go.mod @@ -5,7 +5,7 @@ go 1.25.0 require ( github.com/banzaicloud/go-cruise-control v0.6.0 github.com/compose-spec/compose-go/v2 v2.14.0 - github.com/docker/cli v29.7.1+incompatible + github.com/docker/cli v29.7.2+incompatible github.com/docker/compose/v2 v2.40.3 github.com/go-logr/logr v1.4.4 github.com/go-logr/zapr v1.3.0 diff --git a/third_party/github.com/banzaicloud/go-cruise-control/integration_test/go.sum b/third_party/github.com/banzaicloud/go-cruise-control/integration_test/go.sum index 3a515986c..41a500d00 100644 --- a/third_party/github.com/banzaicloud/go-cruise-control/integration_test/go.sum +++ b/third_party/github.com/banzaicloud/go-cruise-control/integration_test/go.sum @@ -101,8 +101,8 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/buildx v0.29.1 h1:58hxM5Z4mnNje3G5NKfULT9xCr8ooM8XFtlfUK9bKaA= github.com/docker/buildx v0.29.1/go.mod h1:J4EFv6oxlPiV1MjO0VyJx2u5tLM7ImDEl9zyB8d4wPI= -github.com/docker/cli v29.7.1+incompatible h1:ILZpP6B7fedIr6ANy824QkDp1WMJuouIq0O2SrBkB2w= -github.com/docker/cli v29.7.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v29.7.2+incompatible h1:dlkwallR8XqfeVnA2ELEhdwvb4lsSwuB4IgsG8Q9cLY= +github.com/docker/cli v29.7.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/compose/v2 v2.40.3 h1:XeYkQu1svDtyfZPv5nTwFryQ25ZJMkIlc4pz9HalMPI= github.com/docker/compose/v2 v2.40.3/go.mod h1:iNY1tvoHTyN3C3QHCuWAgj3OjR2T6mGkk/qxfbBF/4M= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= diff --git a/third_party/github.com/banzaicloud/operator-tools/go.mod b/third_party/github.com/banzaicloud/operator-tools/go.mod index 71dcea634..a9984a841 100644 --- a/third_party/github.com/banzaicloud/operator-tools/go.mod +++ b/third_party/github.com/banzaicloud/operator-tools/go.mod @@ -50,7 +50,6 @@ require ( github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/evanphx/json-patch v5.9.11+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect - github.com/felixge/httpsnoop v1.1.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.2 // indirect github.com/go-errors/errors v1.4.2 // indirect @@ -74,6 +73,7 @@ require ( github.com/google/gnostic-models v0.7.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gosuri/uitable v0.0.4 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.5.0 // indirect @@ -112,9 +112,9 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xlab/treeprint v1.2.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect - go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect @@ -129,9 +129,8 @@ require ( golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.48.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260729162451-8efbd57d26e0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260729162451-8efbd57d26e0 // indirect - google.golang.org/grpc v1.83.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260807164820-c8921c73eeea // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260807164820-c8921c73eeea // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/third_party/github.com/banzaicloud/operator-tools/go.sum b/third_party/github.com/banzaicloud/operator-tools/go.sum index d9fdca28e..d24de446b 100644 --- a/third_party/github.com/banzaicloud/operator-tools/go.sum +++ b/third_party/github.com/banzaicloud/operator-tools/go.sum @@ -206,8 +206,8 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -376,10 +376,10 @@ go.opentelemetry.io/contrib/bridges/prometheus v0.67.0 h1:dkBzNEAIKADEaFnuESzcXv go.opentelemetry.io/contrib/bridges/prometheus v0.67.0/go.mod h1:Z5RIwRkZgauOIfnG5IpidvLpERjhTninpP1dTG2jTl4= go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 h1:4fnRcNpc6YFtG3zsFw9achKn3XgmxPxuMuqIL5rE8e8= go.opentelemetry.io/contrib/exporters/autoexport v0.67.0/go.mod h1:qTvIHMFKoxW7HXg02gm6/Wofhq5p3Ib/A/NNt1EoBSQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0 h1:Dn8rkudDzY6KV9dr/D/bTUuWgqDf9xe0rr4G2elrn0Y= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0/go.mod h1:gMk9F0xDgyN9M/3Ed5Y1wKcx/9mlU91NXY2SNq7RQuU= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0 h1:HIBTQ3VO5aupLKjC90JgMqpezVXwFuq6Ryjn0/izoag= @@ -388,10 +388,10 @@ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 h1:8UQ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0/go.mod h1:2lmweYCiHYpEjQ/lSJBYhj9jP1zvCvQW4BqL9dnT7FQ= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 h1:w1K+pCJoPpQifuVpsKamUdn9U0zM3xUziVOqsGksUrY= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0/go.mod h1:HBy4BjzgVE8139ieRI75oXm3EcDN+6GhD88JT1Kjvxg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= go.opentelemetry.io/otel/exporters/prometheus v0.65.0 h1:jOveH/b4lU9HT7y+Gfamf18BqlOuz2PWEvs8yM7Q6XE= @@ -400,20 +400,20 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.19.0 h1:GJkybS+crDMdExT/B go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.19.0/go.mod h1:NuAyxRYIG2lKX3YQkB+83StTxM7s52PUUkRRiC0wnYI= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 h1:mS47AX77OtFfKG4vtp+84kuGSFZHTyxtXIN269vChY0= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0/go.mod h1:PJnsC41lAGncJlPUniSwM81gc80GkgWJWr3cu2nKEtU= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo= go.opentelemetry.io/otel/log v0.19.0 h1:KUZs/GOsw79TBBMfDWsXS+KZ4g2Ckzksd1ymzsIEbo4= go.opentelemetry.io/otel/log v0.19.0/go.mod h1:5DQYeGmxVIr4n0/BcJvF4upsraHjg6vudJJpnkL6Ipk= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= go.opentelemetry.io/otel/sdk/log v0.19.0 h1:scYVLqT22D2gqXItnWiocLUKGH9yvkkeql5dBDiXyko= go.opentelemetry.io/otel/sdk/log v0.19.0/go.mod h1:vFBowwXGLlW9AvpuF7bMgnNI95LiW10szrOdvzBHlAg= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -492,10 +492,10 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/api v0.0.0-20260729162451-8efbd57d26e0 h1:ybvH/ZpOcpCrjtkb7oW/fdlzbEmRVeumw19SRQmNFKU= -google.golang.org/genproto/googleapis/api v0.0.0-20260729162451-8efbd57d26e0/go.mod h1:HJ9MpJLeDSstBkx1LILTpd5f41ADSMZcTPypw02qEGw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260729162451-8efbd57d26e0 h1:mJiOtnGp0k/BcSgdu03G2NwnscCfCH+h2QKUBZr18KI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260729162451-8efbd57d26e0/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260807164820-c8921c73eeea h1:Jifw/kjs/r3B0uszvls/m3c3tmZs2YHGM9C+rvxP9gY= +google.golang.org/genproto/googleapis/api v0.0.0-20260807164820-c8921c73eeea/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260807164820-c8921c73eeea h1:kVhQEPTpKQahD5+JSBTfBB19wcgQTTjAIn45MBqnyHk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260807164820-c8921c73eeea/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= From ba1b69f4c982a1d44a96cf2936a0e40b248fdec5 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Mon, 10 Aug 2026 12:00:29 +0200 Subject: [PATCH 3/6] fix(e2e): use testInstallNoIngressKafkaCluster for 5-broker test case testInstallKafkaCluster no longer exists after test_install_cluster.go was renamed to test_install_kafka_cluster.go; this call site was missed. --- tests/e2e/koperator_suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/koperator_suite_test.go b/tests/e2e/koperator_suite_test.go index 2e2b9f5fe..dfa75c9a9 100644 --- a/tests/e2e/koperator_suite_test.go +++ b/tests/e2e/koperator_suite_test.go @@ -69,7 +69,7 @@ var _ = ginkgo.When("Testing e2e test altogether", ginkgo.Ordered, func() { testInstallNoIngressKafkaCluster("Installing Kafka cluster (Zookeeper-based, 4 disks, no ingress)", "../../config/samples/simplekafkacluster_4disk.yaml") testMultiDiskRemoval() testUninstallKafkaCluster() - testInstallKafkaCluster("../../config/samples/simplekafkacluster_5broker.yaml") + testInstallNoIngressKafkaCluster("Installing Kafka cluster (Zookeeper-based, 5 broker, no ingress)", "../../config/samples/simplekafkacluster_5broker.yaml") testBatchedBrokerRemoval() testUninstallKafkaCluster() testUninstallZookeeperCluster() From 6a398fe953b98dbe1d6b52a7079155f05e745abd Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Mon, 10 Aug 2026 12:03:57 +0200 Subject: [PATCH 4/6] refactor(e2e): collapse duplicate kafka-cluster install helpers testInstallNoIngressKafkaCluster, testInstallEnvoyKafkaCluster, and testInstallEnvoyGatewayKafkaCluster were byte-identical bodies; the ingress type is already encoded in the manifest path, not the Go code, so the split added no behavior and let one call site drift out of sync (the 5-broker bug). Collapsed into testInstallKafkaCluster and reverted the file rename from test_install_cluster.go. --- tests/e2e/koperator_suite_test.go | 18 +++++------ ...fka_cluster.go => test_install_cluster.go} | 32 +------------------ 2 files changed, 10 insertions(+), 40 deletions(-) rename tests/e2e/{test_install_kafka_cluster.go => test_install_cluster.go} (55%) diff --git a/tests/e2e/koperator_suite_test.go b/tests/e2e/koperator_suite_test.go index dfa75c9a9..7f024df97 100644 --- a/tests/e2e/koperator_suite_test.go +++ b/tests/e2e/koperator_suite_test.go @@ -58,43 +58,43 @@ var _ = ginkgo.When("Testing e2e test altogether", ginkgo.Ordered, func() { snapshotCluster(snapshottedInfo) testInstall() testInstallZookeeperCluster() - testInstallNoIngressKafkaCluster("Installing Kafka cluster (Zookeeper-based, plaintext, no ingress)", "../../config/samples/simplekafkacluster.yaml") + testInstallKafkaCluster("Installing Kafka cluster (Zookeeper-based, plaintext, no ingress)", "../../config/samples/simplekafkacluster.yaml") testProduceConsumeInternal() testJmxExporter() testUninstallKafkaCluster() - testInstallNoIngressKafkaCluster("Installing Kafka cluster (Zookeeper-based, SSL enabled, no ingress)", "../../config/samples/simplekafkacluster_ssl.yaml") + testInstallKafkaCluster("Installing Kafka cluster (Zookeeper-based, SSL enabled, no ingress)", "../../config/samples/simplekafkacluster_ssl.yaml") testProduceConsumeInternalSSL(defaultTLSSecretName) testJmxExporter() testUninstallKafkaCluster() - testInstallNoIngressKafkaCluster("Installing Kafka cluster (Zookeeper-based, 4 disks, no ingress)", "../../config/samples/simplekafkacluster_4disk.yaml") + testInstallKafkaCluster("Installing Kafka cluster (Zookeeper-based, 4 disks, no ingress)", "../../config/samples/simplekafkacluster_4disk.yaml") testMultiDiskRemoval() testUninstallKafkaCluster() - testInstallNoIngressKafkaCluster("Installing Kafka cluster (Zookeeper-based, 5 broker, no ingress)", "../../config/samples/simplekafkacluster_5broker.yaml") + testInstallKafkaCluster("Installing Kafka cluster (Zookeeper-based, 5 broker, no ingress)", "../../config/samples/simplekafkacluster_5broker.yaml") testBatchedBrokerRemoval() testUninstallKafkaCluster() testUninstallZookeeperCluster() // kraft tests - testInstallNoIngressKafkaCluster("Installing Kafka cluster (KRaft mode, plaintext, no ingress)", "../../config/samples/kraft/simplekafkacluster_kraft.yaml") + testInstallKafkaCluster("Installing Kafka cluster (KRaft mode, plaintext, no ingress)", "../../config/samples/kraft/simplekafkacluster_kraft.yaml") testProduceConsumeInternal() testJmxExporter() testUninstallKafkaCluster() testInstallZookeeperCluster() - testInstallEnvoyKafkaCluster("Installing Kafka cluster (Zookeeper-based, Envoy ingress)", "../../config/samples/simplekafkacluster_with_envoy.yaml") + testInstallKafkaCluster("Installing Kafka cluster (Zookeeper-based, Envoy ingress)", "../../config/samples/simplekafkacluster_with_envoy.yaml") testProduceConsumeInternal() testJmxExporter() testUninstallKafkaCluster() testUninstallZookeeperCluster() - testInstallEnvoyKafkaCluster("Installing Kafka cluster (KRaft mode, Envoy ingress)", "../../config/samples/kraft/simplekafkacluster_kraft_with_envoy.yaml") + testInstallKafkaCluster("Installing Kafka cluster (KRaft mode, Envoy ingress)", "../../config/samples/kraft/simplekafkacluster_kraft_with_envoy.yaml") testProduceConsumeInternal() testJmxExporter() testUninstallKafkaCluster() testInstallZookeeperCluster() - testInstallEnvoyGatewayKafkaCluster("Installing Kafka cluster (Zookeeper-based, Envoy Gateway ingress)", "../../config/samples/simplekafkacluster_with_envoygateway.yaml") + testInstallKafkaCluster("Installing Kafka cluster (Zookeeper-based, Envoy Gateway ingress)", "../../config/samples/simplekafkacluster_with_envoygateway.yaml") testProduceConsumeInternal() testJmxExporter() testUninstallEnvoyGatewayKafkaCluster("../../config/samples/simplekafkacluster_with_envoygateway.yaml") testUninstallZookeeperCluster() - testInstallEnvoyGatewayKafkaCluster("Installing Kafka cluster (KRaft mode, Envoy Gateway ingress)", "../../config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml") + testInstallKafkaCluster("Installing Kafka cluster (KRaft mode, Envoy Gateway ingress)", "../../config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml") testProduceConsumeInternal() testJmxExporter() testUninstallEnvoyGatewayKafkaCluster("../../config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml") diff --git a/tests/e2e/test_install_kafka_cluster.go b/tests/e2e/test_install_cluster.go similarity index 55% rename from tests/e2e/test_install_kafka_cluster.go rename to tests/e2e/test_install_cluster.go index 1c0834e5a..51e837b66 100644 --- a/tests/e2e/test_install_kafka_cluster.go +++ b/tests/e2e/test_install_cluster.go @@ -36,37 +36,7 @@ func testInstallZookeeperCluster() bool { }) } -func testInstallNoIngressKafkaCluster(clusterDescription, kafkaClusterManifestPath string) bool { //nolint:unparam // Note: respecting Ginkgo testing interface by returning bool. - return ginkgo.When(clusterDescription, func() { - var kubectlOptions k8s.KubectlOptions - var err error - - ginkgo.It("Acquiring K8s config and context", func() { - kubectlOptions, err = kubectlOptionsForCurrentContext() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - kubectlOptions.Namespace = koperatorLocalHelmDescriptor.Namespace - requireCreatingKafkaCluster(kubectlOptions, kafkaClusterManifestPath) - }) -} - -func testInstallEnvoyKafkaCluster(clusterDescription, kafkaClusterManifestPath string) bool { //nolint:unparam // Note: respecting Ginkgo testing interface by returning bool. - return ginkgo.When(clusterDescription, func() { - var kubectlOptions k8s.KubectlOptions - var err error - - ginkgo.It("Acquiring K8s config and context", func() { - kubectlOptions, err = kubectlOptionsForCurrentContext() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - kubectlOptions.Namespace = koperatorLocalHelmDescriptor.Namespace - requireCreatingKafkaCluster(kubectlOptions, kafkaClusterManifestPath) - }) -} - -func testInstallEnvoyGatewayKafkaCluster(clusterDescription, kafkaClusterManifestPath string) bool { //nolint:unparam // Note: respecting Ginkgo testing interface by returning bool. +func testInstallKafkaCluster(clusterDescription, kafkaClusterManifestPath string) bool { //nolint:unparam // Note: respecting Ginkgo testing interface by returning bool. return ginkgo.When(clusterDescription, func() { var kubectlOptions k8s.KubectlOptions var err error From d70d477e12d68a6ac142596f9ac93051a4ef7720 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Mon, 10 Aug 2026 12:17:22 +0200 Subject: [PATCH 5/6] fix(e2e): apply CruiseControl e2e tuning to envoy/envoygateway samples simplekafkacluster_with_envoy.yaml, simplekafkacluster_with_envoygateway.yaml, and their kraft counterparts were added without the CC warmup tuning that b7be337a (#291) applied to the other e2e-exercised samples (15s sampling, shrunk metric/sample-store windows, min.valid.partition.ratio, etc), so they carried the old flaky defaults. Brought all 4 in line with the reference samples, verified word-for-word identical against the fixed simplekafkacluster.yaml/simplekafkacluster_kraft.yaml. --- .../simplekafkacluster_kraft_with_envoy.yaml | 35 ++++++++++++++----- ...ekafkacluster_kraft_with_envoygateway.yaml | 35 ++++++++++++++----- .../simplekafkacluster_with_envoy.yaml | 35 ++++++++++++++----- .../simplekafkacluster_with_envoygateway.yaml | 35 ++++++++++++++----- 4 files changed, 108 insertions(+), 32 deletions(-) diff --git a/config/samples/kraft/simplekafkacluster_kraft_with_envoy.yaml b/config/samples/kraft/simplekafkacluster_kraft_with_envoy.yaml index ed8835b94..9e99d4c30 100644 --- a/config/samples/kraft/simplekafkacluster_kraft_with_envoy.yaml +++ b/config/samples/kraft/simplekafkacluster_kraft_with_envoy.yaml @@ -19,6 +19,8 @@ spec: cruise.control.metrics.topic.auto.create=true cruise.control.metrics.topic.num.partitions=1 cruise.control.metrics.topic.replication.factor=2 + # e2e: publish CruiseControl metrics every 15s (default 60s) so CC warms up quickly; not production defaults + cruise.control.metrics.reporter.metrics.reporting.interval.ms=15000 brokerConfigGroups: default: storageConfigs: @@ -71,7 +73,8 @@ spec: cruiseControlTaskSpec: RetryDurationMinutes: 5 topicConfig: - partitions: 12 + # e2e: fewer partitions on the __CruiseControlMetrics topic => faster CC monitoring coverage + partitions: 3 replicationFactor: 3 config: | # Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. @@ -80,7 +83,8 @@ spec: # Configuration for the metadata client. # ======================================= # The maximum interval in milliseconds between two metadata refreshes. - #metadata.max.age.ms=300000 + # e2e: must be <= metric.sampling.interval.ms (CC sanityCheckSamplingPeriod); low for the fast 15s sampling + metadata.max.age.ms=10000 # Client id for the Cruise Control. It is used for the metadata client. #client.id=kafka-cruise-control # The size of TCP send buffer bytes for the metadata client. @@ -109,23 +113,35 @@ spec: broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples # The replication factor of Kafka metric sample store topic sample.store.topic.replication.factor=2 + # e2e: shrink CC's sample-store topics from their 32-partitions-each default so broker/disk removal + # reshuffles only a few partitions (not 64) and the CC execution completes quickly; not production defaults. + # NOTE: CC only ever increases these partition counts (never shrinks), so this only takes effect on a fresh cluster. + partition.sample.store.topic.partition.count=3 + broker.sample.store.topic.partition.count=3 # The config for the number of Kafka sample store consumer threads num.sample.loading.threads=8 # The partition assignor class for the metric samplers metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor # The metric sampling interval in milliseconds - metric.sampling.interval.ms=120000 + # e2e: sample every 15s (default 120s) so CC accrues valid windows fast; not production defaults + metric.sampling.interval.ms=15000 + # e2e: CC reads the reporter interval for a sanity check and requires it <= the sampling interval + cruise.control.metrics.reporter.metrics.reporting.interval.ms=15000 + # e2e: must be >= metric.sampling.interval.ms (CC sanityCheckSamplingPeriod) metric.anomaly.detection.interval.ms=180000 # The partition metrics window size in milliseconds - partition.metrics.window.ms=300000 + # e2e: shrink window to the 15s sampling interval so CC accrues a valid window fast; not production defaults + partition.metrics.window.ms=15000 # The number of partition metric windows to keep in memory num.partition.metrics.windows=1 # The minimum partition metric samples required for a partition in each window min.samples.per.partition.metrics.window=1 # The broker metrics window size in milliseconds - broker.metrics.window.ms=300000 + # e2e: shrink window to the 15s sampling interval to speed up CC warmup; not production defaults + broker.metrics.window.ms=15000 # The number of broker metric windows to keep in memory - num.broker.metrics.windows=20 + # e2e: only 2 broker windows needed for warmup on a tiny static cluster (was 20); not production defaults + num.broker.metrics.windows=2 # The minimum broker metric samples required for a partition in each window min.samples.per.broker.metrics.window=1 # The configuration for the BrokerCapacityConfigFileResolver (supports JBOD and non-JBOD broker capacities) @@ -140,7 +156,9 @@ spec: # The list of supported hard goals hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal # The minimum percentage of well monitored partitions out of all the partitions - min.monitored.partition.percentage=0.95 + # e2e: lowered from CC's 0.995 default so the load model becomes valid quickly on a tiny cluster + # with few partitions/windows; not production defaults + min.valid.partition.ratio=0.5 # The balance threshold for CPU cpu.balance.threshold=1.1 # The balance threshold for disk @@ -184,7 +202,8 @@ spec: # The max number of partitions to move in/out on a given broker at a given time. num.concurrent.partition.movements.per.broker=10 # The interval between two execution progress checks. - execution.progress.check.interval.ms=10000 + # e2e: 5s is the floor (must be >= min.execution.progress.check.interval.ms, default 5000); was 10s + execution.progress.check.interval.ms=5000 # Configurations for anomaly detector # ======================================= # The goal violation notifier class diff --git a/config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml b/config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml index 9484f4b78..d17dd7cdc 100644 --- a/config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml +++ b/config/samples/kraft/simplekafkacluster_kraft_with_envoygateway.yaml @@ -67,6 +67,8 @@ spec: cruise.control.metrics.topic.auto.create=true cruise.control.metrics.topic.num.partitions=1 cruise.control.metrics.topic.replication.factor=2 + # e2e: publish CruiseControl metrics every 15s (default 60s) so CC warms up quickly; not production defaults + cruise.control.metrics.reporter.metrics.reporting.interval.ms=15000 brokerConfigGroups: default: storageConfigs: @@ -120,7 +122,8 @@ spec: cruiseControlTaskSpec: RetryDurationMinutes: 5 topicConfig: - partitions: 12 + # e2e: fewer partitions on the __CruiseControlMetrics topic => faster CC monitoring coverage + partitions: 3 replicationFactor: 3 config: | # Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. @@ -129,7 +132,8 @@ spec: # Configuration for the metadata client. # ======================================= # The maximum interval in milliseconds between two metadata refreshes. - #metadata.max.age.ms=300000 + # e2e: must be <= metric.sampling.interval.ms (CC sanityCheckSamplingPeriod); low for the fast 15s sampling + metadata.max.age.ms=10000 # Client id for the Cruise Control. It is used for the metadata client. #client.id=kafka-cruise-control # The size of TCP send buffer bytes for the metadata client. @@ -158,23 +162,35 @@ spec: broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples # The replication factor of Kafka metric sample store topic sample.store.topic.replication.factor=2 + # e2e: shrink CC's sample-store topics from their 32-partitions-each default so broker/disk removal + # reshuffles only a few partitions (not 64) and the CC execution completes quickly; not production defaults. + # NOTE: CC only ever increases these partition counts (never shrinks), so this only takes effect on a fresh cluster. + partition.sample.store.topic.partition.count=3 + broker.sample.store.topic.partition.count=3 # The config for the number of Kafka sample store consumer threads num.sample.loading.threads=8 # The partition assignor class for the metric samplers metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor # The metric sampling interval in milliseconds - metric.sampling.interval.ms=120000 + # e2e: sample every 15s (default 120s) so CC accrues valid windows fast; not production defaults + metric.sampling.interval.ms=15000 + # e2e: CC reads the reporter interval for a sanity check and requires it <= the sampling interval + cruise.control.metrics.reporter.metrics.reporting.interval.ms=15000 + # e2e: must be >= metric.sampling.interval.ms (CC sanityCheckSamplingPeriod) metric.anomaly.detection.interval.ms=180000 # The partition metrics window size in milliseconds - partition.metrics.window.ms=300000 + # e2e: shrink window to the 15s sampling interval so CC accrues a valid window fast; not production defaults + partition.metrics.window.ms=15000 # The number of partition metric windows to keep in memory num.partition.metrics.windows=1 # The minimum partition metric samples required for a partition in each window min.samples.per.partition.metrics.window=1 # The broker metrics window size in milliseconds - broker.metrics.window.ms=300000 + # e2e: shrink window to the 15s sampling interval to speed up CC warmup; not production defaults + broker.metrics.window.ms=15000 # The number of broker metric windows to keep in memory - num.broker.metrics.windows=20 + # e2e: only 2 broker windows needed for warmup on a tiny static cluster (was 20); not production defaults + num.broker.metrics.windows=2 # The minimum broker metric samples required for a partition in each window min.samples.per.broker.metrics.window=1 # The configuration for the BrokerCapacityConfigFileResolver (supports JBOD and non-JBOD broker capacities) @@ -189,7 +205,9 @@ spec: # The list of supported hard goals hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal # The minimum percentage of well monitored partitions out of all the partitions - min.monitored.partition.percentage=0.95 + # e2e: lowered from CC's 0.995 default so the load model becomes valid quickly on a tiny cluster + # with few partitions/windows; not production defaults + min.valid.partition.ratio=0.5 # The balance threshold for CPU cpu.balance.threshold=1.1 # The balance threshold for disk @@ -233,7 +251,8 @@ spec: # The max number of partitions to move in/out on a given broker at a given time. num.concurrent.partition.movements.per.broker=10 # The interval between two execution progress checks. - execution.progress.check.interval.ms=10000 + # e2e: 5s is the floor (must be >= min.execution.progress.check.interval.ms, default 5000); was 10s + execution.progress.check.interval.ms=5000 # Configurations for anomaly detector # ======================================= # The goal violation notifier class diff --git a/config/samples/simplekafkacluster_with_envoy.yaml b/config/samples/simplekafkacluster_with_envoy.yaml index 464862e0d..877c9da56 100644 --- a/config/samples/simplekafkacluster_with_envoy.yaml +++ b/config/samples/simplekafkacluster_with_envoy.yaml @@ -36,6 +36,8 @@ spec: cruise.control.metrics.topic.auto.create=true cruise.control.metrics.topic.num.partitions=1 cruise.control.metrics.topic.replication.factor=2 + # e2e: publish CruiseControl metrics every 15s (default 60s) so CC warms up quickly; not production defaults + cruise.control.metrics.reporter.metrics.reporting.interval.ms=15000 brokerConfigGroups: default: storageConfigs: @@ -62,7 +64,8 @@ spec: cruiseControlTaskSpec: RetryDurationMinutes: 5 topicConfig: - partitions: 12 + # e2e: fewer partitions on the __CruiseControlMetrics topic => faster CC monitoring coverage + partitions: 3 replicationFactor: 3 config: | # Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. @@ -71,7 +74,8 @@ spec: # Configuration for the metadata client. # ======================================= # The maximum interval in milliseconds between two metadata refreshes. - #metadata.max.age.ms=300000 + # e2e: must be <= metric.sampling.interval.ms (CC sanityCheckSamplingPeriod); low for the fast 15s sampling + metadata.max.age.ms=10000 # Client id for the Cruise Control. It is used for the metadata client. #client.id=kafka-cruise-control # The size of TCP send buffer bytes for the metadata client. @@ -100,23 +104,35 @@ spec: broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples # The replication factor of Kafka metric sample store topic sample.store.topic.replication.factor=2 + # e2e: shrink CC's sample-store topics from their 32-partitions-each default so broker/disk removal + # reshuffles only a few partitions (not 64) and the CC execution completes quickly; not production defaults. + # NOTE: CC only ever increases these partition counts (never shrinks), so this only takes effect on a fresh cluster. + partition.sample.store.topic.partition.count=3 + broker.sample.store.topic.partition.count=3 # The config for the number of Kafka sample store consumer threads num.sample.loading.threads=8 # The partition assignor class for the metric samplers metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor # The metric sampling interval in milliseconds - metric.sampling.interval.ms=120000 + # e2e: sample every 15s (default 120s) so CC accrues valid windows fast; not production defaults + metric.sampling.interval.ms=15000 + # e2e: CC reads the reporter interval for a sanity check and requires it <= the sampling interval + cruise.control.metrics.reporter.metrics.reporting.interval.ms=15000 + # e2e: must be >= metric.sampling.interval.ms (CC sanityCheckSamplingPeriod) metric.anomaly.detection.interval.ms=180000 # The partition metrics window size in milliseconds - partition.metrics.window.ms=300000 + # e2e: shrink window to the 15s sampling interval so CC accrues a valid window fast; not production defaults + partition.metrics.window.ms=15000 # The number of partition metric windows to keep in memory num.partition.metrics.windows=1 # The minimum partition metric samples required for a partition in each window min.samples.per.partition.metrics.window=1 # The broker metrics window size in milliseconds - broker.metrics.window.ms=300000 + # e2e: shrink window to the 15s sampling interval to speed up CC warmup; not production defaults + broker.metrics.window.ms=15000 # The number of broker metric windows to keep in memory - num.broker.metrics.windows=20 + # e2e: only 2 broker windows needed for warmup on a tiny static cluster (was 20); not production defaults + num.broker.metrics.windows=2 # The minimum broker metric samples required for a partition in each window min.samples.per.broker.metrics.window=1 # The configuration for the BrokerCapacityConfigFileResolver (supports JBOD and non-JBOD broker capacities) @@ -131,7 +147,9 @@ spec: # The list of supported hard goals hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal # The minimum percentage of well monitored partitions out of all the partitions - min.monitored.partition.percentage=0.95 + # e2e: lowered from CC's 0.995 default so the load model becomes valid quickly on a tiny cluster + # with few partitions/windows; not production defaults + min.valid.partition.ratio=0.5 # The balance threshold for CPU cpu.balance.threshold=1.1 # The balance threshold for disk @@ -175,7 +193,8 @@ spec: # The max number of partitions to move in/out on a given broker at a given time. num.concurrent.partition.movements.per.broker=10 # The interval between two execution progress checks. - execution.progress.check.interval.ms=10000 + # e2e: 5s is the floor (must be >= min.execution.progress.check.interval.ms, default 5000); was 10s + execution.progress.check.interval.ms=5000 # Configurations for anomaly detector # ======================================= # The goal violation notifier class diff --git a/config/samples/simplekafkacluster_with_envoygateway.yaml b/config/samples/simplekafkacluster_with_envoygateway.yaml index 52f660909..d8d334cec 100644 --- a/config/samples/simplekafkacluster_with_envoygateway.yaml +++ b/config/samples/simplekafkacluster_with_envoygateway.yaml @@ -68,6 +68,8 @@ spec: cruise.control.metrics.topic.auto.create=true cruise.control.metrics.topic.num.partitions=1 cruise.control.metrics.topic.replication.factor=2 + # e2e: publish CruiseControl metrics every 15s (default 60s) so CC warms up quickly; not production defaults + cruise.control.metrics.reporter.metrics.reporting.interval.ms=15000 brokerConfigGroups: default: storageConfigs: @@ -94,7 +96,8 @@ spec: cruiseControlTaskSpec: RetryDurationMinutes: 5 topicConfig: - partitions: 12 + # e2e: fewer partitions on the __CruiseControlMetrics topic => faster CC monitoring coverage + partitions: 3 replicationFactor: 3 config: | # Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. @@ -103,7 +106,8 @@ spec: # Configuration for the metadata client. # ======================================= # The maximum interval in milliseconds between two metadata refreshes. - #metadata.max.age.ms=300000 + # e2e: must be <= metric.sampling.interval.ms (CC sanityCheckSamplingPeriod); low for the fast 15s sampling + metadata.max.age.ms=10000 # Client id for the Cruise Control. It is used for the metadata client. #client.id=kafka-cruise-control # The size of TCP send buffer bytes for the metadata client. @@ -132,23 +136,35 @@ spec: broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples # The replication factor of Kafka metric sample store topic sample.store.topic.replication.factor=2 + # e2e: shrink CC's sample-store topics from their 32-partitions-each default so broker/disk removal + # reshuffles only a few partitions (not 64) and the CC execution completes quickly; not production defaults. + # NOTE: CC only ever increases these partition counts (never shrinks), so this only takes effect on a fresh cluster. + partition.sample.store.topic.partition.count=3 + broker.sample.store.topic.partition.count=3 # The config for the number of Kafka sample store consumer threads num.sample.loading.threads=8 # The partition assignor class for the metric samplers metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor # The metric sampling interval in milliseconds - metric.sampling.interval.ms=120000 + # e2e: sample every 15s (default 120s) so CC accrues valid windows fast; not production defaults + metric.sampling.interval.ms=15000 + # e2e: CC reads the reporter interval for a sanity check and requires it <= the sampling interval + cruise.control.metrics.reporter.metrics.reporting.interval.ms=15000 + # e2e: must be >= metric.sampling.interval.ms (CC sanityCheckSamplingPeriod) metric.anomaly.detection.interval.ms=180000 # The partition metrics window size in milliseconds - partition.metrics.window.ms=300000 + # e2e: shrink window to the 15s sampling interval so CC accrues a valid window fast; not production defaults + partition.metrics.window.ms=15000 # The number of partition metric windows to keep in memory num.partition.metrics.windows=1 # The minimum partition metric samples required for a partition in each window min.samples.per.partition.metrics.window=1 # The broker metrics window size in milliseconds - broker.metrics.window.ms=300000 + # e2e: shrink window to the 15s sampling interval to speed up CC warmup; not production defaults + broker.metrics.window.ms=15000 # The number of broker metric windows to keep in memory - num.broker.metrics.windows=20 + # e2e: only 2 broker windows needed for warmup on a tiny static cluster (was 20); not production defaults + num.broker.metrics.windows=2 # The minimum broker metric samples required for a partition in each window min.samples.per.broker.metrics.window=1 # The configuration for the BrokerCapacityConfigFileResolver (supports JBOD and non-JBOD broker capacities) @@ -163,7 +179,9 @@ spec: # The list of supported hard goals hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal # The minimum percentage of well monitored partitions out of all the partitions - min.monitored.partition.percentage=0.95 + # e2e: lowered from CC's 0.995 default so the load model becomes valid quickly on a tiny cluster + # with few partitions/windows; not production defaults + min.valid.partition.ratio=0.5 # The balance threshold for CPU cpu.balance.threshold=1.1 # The balance threshold for disk @@ -207,7 +225,8 @@ spec: # The max number of partitions to move in/out on a given broker at a given time. num.concurrent.partition.movements.per.broker=10 # The interval between two execution progress checks. - execution.progress.check.interval.ms=10000 + # e2e: 5s is the floor (must be >= min.execution.progress.check.interval.ms, default 5000); was 10s + execution.progress.check.interval.ms=5000 # Configurations for anomaly detector # ======================================= # The goal violation notifier class From a51bec7bfdafca6a35f12f9bc39f338d5f2bbdfc Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Mon, 10 Aug 2026 15:04:36 +0200 Subject: [PATCH 6/6] fix(ci): disable cloud-provider-kind's Gateway API CRD install cloud-provider-kind installs the standard-channel Gateway API CRDs by default (gateway-channel=standard). Those collide via server-side-apply with the CRDs Envoy Gateway's Helm chart installs in the same cluster (different field manager, differing bundle-version/channel annotations and .spec.versions), failing "helm install eg oci://.../gateway-helm" with an SSA conflict during the e2e "Installing infrastructure components in parallel" step. cloud-provider-kind is only needed here for LoadBalancer Service IP allocation, not its own Gateway controller. --- .github/actions/kind-create/action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/kind-create/action.yaml b/.github/actions/kind-create/action.yaml index 12e3034ef..8f7880a59 100644 --- a/.github/actions/kind-create/action.yaml +++ b/.github/actions/kind-create/action.yaml @@ -83,5 +83,6 @@ runs: echo "Install cloud-provider-kind" go install sigs.k8s.io/cloud-provider-kind@latest kubectl label node e2e-kind-control-plane node.kubernetes.io/exclude-from-external-load-balancers- - ~/go/bin/cloud-provider-kind & + # gateway-channel disabled: its own Gateway API CRDs conflict via SSA with Envoy Gateway's Helm-installed ones + ~/go/bin/cloud-provider-kind --gateway-channel disabled & shell: bash