From cd3005317f96d14595ee2eccd5e4e81bdd8bf000 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Tue, 14 Jul 2026 10:58:02 +0200 Subject: [PATCH 01/16] feat: mutex-protected in-process map for scheduling safety --- cmd/manager/main.go | 95 ++-- helm/bundles/cortex-nova/values.yaml | 3 + .../nova/hypervisor_overcommit_controller.go | 8 +- .../hypervisor_overcommit_controller_test.go | 2 +- pkg/clientcache/cache.go | 262 ++++++++++ pkg/clientcache/cache_test.go | 436 ++++++++++++++++ pkg/clientcache/client.go | 265 ++++++++++ pkg/clientcache/client_test.go | 470 ++++++++++++++++++ pkg/clientcache/config.go | 24 + pkg/clientcache/interfaces.go | 22 + pkg/clientcache/runnable.go | 87 ++++ pkg/multicluster/client.go | 25 + 12 files changed, 1652 insertions(+), 47 deletions(-) create mode 100644 pkg/clientcache/cache.go create mode 100644 pkg/clientcache/cache_test.go create mode 100644 pkg/clientcache/client.go create mode 100644 pkg/clientcache/client_test.go create mode 100644 pkg/clientcache/config.go create mode 100644 pkg/clientcache/interfaces.go create mode 100644 pkg/clientcache/runnable.go diff --git a/cmd/manager/main.go b/cmd/manager/main.go index af86e3d84..8976816ba 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -64,6 +64,7 @@ import ( commitmentsapi "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments/api" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/quota" + "github.com/cobaltcore-dev/cortex/pkg/clientcache" "github.com/cobaltcore-dev/cortex/pkg/conf" "github.com/cobaltcore-dev/cortex/pkg/monitoring" "github.com/cobaltcore-dev/cortex/pkg/multicluster" @@ -372,6 +373,22 @@ func main() { os.Exit(1) } + // Transparent in-process overlay cache for CRDs that are eventually + // consistent across in-pod clients (e.g. Reservations). Writes populate an + // overlay; reads merge it with the informer result until the real object is + // observed. *multicluster.Client serves as both the inner client.Client and + // the InformerSource; the cache itself has no multicluster dependency. + clientCacheConfig := conf.GetConfigOrDie[clientcache.RootConfig]() + cachingClient, err := clientcache.New(multiclusterClient, multiclusterClient, scheme, clientCacheConfig.ClientCache) + if err != nil { + setupLog.Error(err, "unable to create client cache") + os.Exit(1) + } + if err := mgr.Add(cachingClient); err != nil { + setupLog.Error(err, "unable to add client cache to manager") + os.Exit(1) + } + // Our custom monitoring registry can add prometheus labels to all metrics. // This is useful to distinguish metrics from different deployments. metricsConfig := conf.GetConfigOrDie[monitoring.Config]() @@ -403,10 +420,10 @@ func main() { commitmentsConfig := conf.GetConfigOrDie[commitments.Config]() var commitmentsVMSource reservations.VMSource if commitmentsConfig.DatasourceName != "" { - commitmentsVMSource = reservations.NewPostgresVMSource(multiclusterClient, commitmentsConfig.DatasourceName) + commitmentsVMSource = reservations.NewPostgresVMSource(cachingClient, commitmentsConfig.DatasourceName) } if slices.Contains(mainConfig.EnabledControllers, "committed-resource-reservations-controller") { - commitmentsAPI := commitmentsapi.NewAPIWithConfig(multiclusterClient, commitmentsConfig.API, commitmentsVMSource) + commitmentsAPI := commitmentsapi.NewAPIWithConfig(cachingClient, commitmentsConfig.API, commitmentsVMSource) commitmentsAPI.Init(mux, metrics.Registry, ctrl.Log.WithName("commitments-api")) } @@ -426,8 +443,8 @@ func main() { metrics.Registry.MustRegister(noHostFoundCounter) metrics.Registry.MustRegister(placementCounter) // Inferred through the base controller. - filterWeigherController.Client = multiclusterClient - filterWeigherController.CRRecorder.Client = multiclusterClient + filterWeigherController.Client = cachingClient + filterWeigherController.CRRecorder.Client = cachingClient if err := filterWeigherController.SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "nova FilterWeigherPipelineController") os.Exit(1) @@ -442,7 +459,7 @@ func main() { novaClient := nova.NewNovaClient() novaClientConfig := conf.GetConfigOrDie[nova.NovaClientConfig]() if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - return novaClient.Init(ctx, multiclusterClient, novaClientConfig) + return novaClient.Init(ctx, cachingClient, novaClientConfig) })); err != nil { setupLog.Error(err, "unable to initialize nova client") os.Exit(1) @@ -453,7 +470,7 @@ func main() { Breaker: &nova.DetectorCycleBreaker{NovaClient: novaClient}, } // Inferred through the base controller. - deschedulingsController.Client = multiclusterClient + deschedulingsController.Client = cachingClient if err := (deschedulingsController).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "nova DetectorPipelineController") os.Exit(1) @@ -461,7 +478,7 @@ func main() { go deschedulingsController.CreateDeschedulingsPeriodically(ctx) // Deschedulings cleanup on startup if err := (&nova.DeschedulingsCleanup{ - Client: multiclusterClient, + Client: cachingClient, Scheme: mgr.GetScheme(), }).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Cleanup") @@ -481,13 +498,13 @@ func main() { novaClient := nova.NewNovaClient() novaClientConfig := conf.GetConfigOrDie[nova.NovaClientConfig]() if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - return novaClient.Init(ctx, multiclusterClient, novaClientConfig) + return novaClient.Init(ctx, cachingClient, novaClientConfig) })); err != nil { setupLog.Error(err, "unable to initialize nova client") os.Exit(1) } if err := (&nova.DeschedulingsExecutor{ - Client: multiclusterClient, + Client: cachingClient, Scheme: mgr.GetScheme(), Conf: executorConfig, NovaClient: novaClient, @@ -498,8 +515,8 @@ func main() { } if slices.Contains(mainConfig.EnabledControllers, "hypervisor-overcommit-controller") { hypervisorOvercommitController := &nova.HypervisorOvercommitController{} - hypervisorOvercommitController.Client = multiclusterClient - if err := hypervisorOvercommitController.SetupWithManager(mgr); err != nil { + hypervisorOvercommitController.Client = cachingClient + if err := hypervisorOvercommitController.SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "HypervisorOvercommitController") os.Exit(1) @@ -511,7 +528,7 @@ func main() { Monitor: filterWeigherPipelineMonitor, } // Inferred through the base controller. - controller.Client = multiclusterClient + controller.Client = cachingClient if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler") os.Exit(1) @@ -531,7 +548,7 @@ func main() { Monitor: filterWeigherPipelineMonitor, } // Inferred through the base controller. - controller.Client = multiclusterClient + controller.Client = cachingClient if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler") os.Exit(1) @@ -551,7 +568,7 @@ func main() { Monitor: filterWeigherPipelineMonitor, } // Inferred through the base controller. - controller.Client = multiclusterClient + controller.Client = cachingClient if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler") os.Exit(1) @@ -570,7 +587,7 @@ func main() { Monitor: filterWeigherPipelineMonitor, } // Inferred through the base controller. - controller.Client = multiclusterClient + controller.Client = cachingClient if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler") os.Exit(1) @@ -586,11 +603,11 @@ func main() { if slices.Contains(mainConfig.EnabledControllers, "committed-resource-reservations-controller") { setupLog.Info("enabling controller", "controller", "committed-resource-reservations-controller") - monitor := reservations.NewMonitor(multiclusterClient) + monitor := reservations.NewMonitor(cachingClient) metrics.Registry.MustRegister(&monitor) if err := (&commitments.CommitmentReservationController{ - Client: multiclusterClient, + Client: cachingClient, Scheme: mgr.GetScheme(), Conf: commitmentsConfig.ReservationController, }).SetupWithManager(mgr, multiclusterClient); err != nil { @@ -600,11 +617,11 @@ func main() { crControllerConf := commitmentsConfig.CommittedResourceController - crControllerMonitor := commitments.NewCRControllerMonitor(multiclusterClient) + crControllerMonitor := commitments.NewCRControllerMonitor(cachingClient) metrics.Registry.MustRegister(&crControllerMonitor) if err := (&commitments.CommittedResourceController{ - Client: multiclusterClient, + Client: cachingClient, Scheme: mgr.GetScheme(), Conf: crControllerConf, Monitor: &crControllerMonitor, @@ -622,7 +639,7 @@ func main() { usageReconcilerConf := commitmentsConfig.UsageReconciler usageReconcilerConf.ApplyDefaults() if err := (&commitments.UsageReconciler{ - Client: multiclusterClient, + Client: cachingClient, Conf: usageReconcilerConf, VMSource: commitmentsVMSource, Monitor: usageReconcilerMonitor, @@ -637,7 +654,7 @@ func main() { monitor := datasources.NewMonitor() metrics.Registry.MustRegister(&monitor) if err := (&openstack.OpenStackDatasourceReconciler{ - Client: multiclusterClient, + Client: cachingClient, Scheme: mgr.GetScheme(), Monitor: monitor, }).SetupWithManager(mgr, multiclusterClient); err != nil { @@ -645,7 +662,7 @@ func main() { os.Exit(1) } if err := (&prometheus.PrometheusDatasourceReconciler{ - Client: multiclusterClient, + Client: cachingClient, Scheme: mgr.GetScheme(), Monitor: monitor, }).SetupWithManager(mgr, multiclusterClient); err != nil { @@ -658,7 +675,7 @@ func main() { monitor := extractor.NewMonitor() metrics.Registry.MustRegister(&monitor) if err := (&extractor.KnowledgeReconciler{ - Client: multiclusterClient, + Client: cachingClient, Scheme: mgr.GetScheme(), Monitor: monitor, Conf: conf.GetConfigOrDie[extractor.KnowledgeReconcilerConfig](), @@ -667,7 +684,7 @@ func main() { os.Exit(1) } if err := (&extractor.TriggerReconciler{ - Client: multiclusterClient, + Client: cachingClient, Scheme: mgr.GetScheme(), Conf: conf.GetConfigOrDie[extractor.TriggerReconcilerConfig](), }).SetupWithManager(mgr, multiclusterClient); err != nil { @@ -679,7 +696,7 @@ func main() { setupLog.Info("enabling controller", "controller", "kpis-controller") kpisControllerConfig := conf.GetConfigOrDie[kpis.ControllerConfig]() if err := (&kpis.Controller{ - Client: multiclusterClient, + Client: cachingClient, Config: kpisControllerConfig, }).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "KPIController") @@ -711,7 +728,7 @@ func main() { if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { // Create PostgresReader from the configured Datasource CRD // This runs after the cache is started - postgresReader, err := external.NewPostgresReader(ctx, multiclusterClient, failoverConfig.DatasourceName) + postgresReader, err := external.NewPostgresReader(ctx, cachingClient, failoverConfig.DatasourceName) if err != nil { setupLog.Error(err, "unable to create postgres reader for failover controller", "datasourceName", failoverConfig.DatasourceName) @@ -727,7 +744,7 @@ func main() { // 1. Watch-based per-reservation reconciliation (acknowledgment, validation) // 2. Periodic bulk VM processing (creating/assigning reservations) failoverController := failover.NewFailoverReservationController( - multiclusterClient, + cachingClient, vmSource, failoverConfig, schedulerClient, @@ -766,12 +783,12 @@ func main() { capacityConfig := conf.GetConfigOrDie[capacity.Config]() capacityConfig.ApplyDefaults() - capacityMonitor := capacity.NewMonitor(multiclusterClient) + capacityMonitor := capacity.NewMonitor(cachingClient) if err := metrics.Registry.Register(&capacityMonitor); err != nil { setupLog.Error(err, "failed to register capacity monitor metrics, continuing without metrics") } - capacityController := capacity.NewController(multiclusterClient, capacityConfig, commitmentsVMSource) + capacityController := capacity.NewController(cachingClient, capacityConfig, commitmentsVMSource) if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { return capacityController.Start(ctx) })); err != nil { @@ -804,7 +821,7 @@ func main() { // Defer initialization until the manager starts (cache must be ready for postgres reader) if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { // Create PostgresReader from the configured Datasource CRD - postgresReader, err := external.NewPostgresReader(ctx, multiclusterClient, datasourceName) + postgresReader, err := external.NewPostgresReader(ctx, cachingClient, datasourceName) if err != nil { setupLog.Error(err, "unable to create postgres reader for quota controller", "datasourceName", datasourceName) @@ -817,7 +834,7 @@ func main() { // Create the quota controller quotaController := quota.NewQuotaController( - multiclusterClient, + cachingClient, vmSource, quotaConfig, quotaMetrics, @@ -879,11 +896,11 @@ func main() { setupLog.Info("starting commitments syncer") syncerMonitor := commitments.NewSyncerMonitor() must.Succeed(metrics.Registry.Register(syncerMonitor)) - syncer := commitments.NewSyncer(multiclusterClient, syncerMonitor) + syncer := commitments.NewSyncer(cachingClient, syncerMonitor) syncerConfig := conf.GetConfigOrDie[commitments.SyncerConfig]() syncerConfig.FlavorGroupResourceConfig = commitmentsConfig.API.FlavorGroupResourceConfig if err := (&task.Runner{ - Client: multiclusterClient, + Client: cachingClient, Interval: syncerConfig.SyncInterval.Duration, Name: "commitments-sync-task", Run: func(ctx context.Context) error { return syncer.SyncReservations(ctx) }, @@ -897,11 +914,11 @@ func main() { setupLog.Info("starting nova history cleanup task") historyCleanupConfig := conf.GetConfigOrDie[nova.HistoryCleanupConfig]() if err := (&task.Runner{ - Client: multiclusterClient, + Client: cachingClient, Interval: time.Hour, Name: "nova-history-cleanup-task", Run: func(ctx context.Context) error { - return nova.HistoryCleanup(ctx, multiclusterClient, historyCleanupConfig) + return nova.HistoryCleanup(ctx, cachingClient, historyCleanupConfig) }, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to add nova history cleanup task to manager") @@ -912,11 +929,11 @@ func main() { setupLog.Info("starting manila history cleanup task") historyCleanupConfig := conf.GetConfigOrDie[manila.HistoryCleanupConfig]() if err := (&task.Runner{ - Client: multiclusterClient, + Client: cachingClient, Interval: time.Hour, Name: "manila-history-cleanup-task", Run: func(ctx context.Context) error { - return manila.HistoryCleanup(ctx, multiclusterClient, historyCleanupConfig) + return manila.HistoryCleanup(ctx, cachingClient, historyCleanupConfig) }, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to add manila history cleanup task to manager") @@ -927,11 +944,11 @@ func main() { setupLog.Info("starting cinder history cleanup task") historyCleanupConfig := conf.GetConfigOrDie[cinder.HistoryCleanupConfig]() if err := (&task.Runner{ - Client: multiclusterClient, + Client: cachingClient, Interval: time.Hour, Name: "cinder-history-cleanup-task", Run: func(ctx context.Context) error { - return cinder.HistoryCleanup(ctx, multiclusterClient, historyCleanupConfig) + return cinder.HistoryCleanup(ctx, cachingClient, historyCleanupConfig) }, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to add cinder history cleanup task to manager") diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 1171f3b01..fb0eee4c1 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -110,6 +110,9 @@ cortex: &cortex - kvm.cloud.sap/v1/Hypervisor - kvm.cloud.sap/v1/HypervisorList - v1/Secret + clientcache: + gvks: + - cortex.cloud/v1alpha1/Reservation keystoneSecretRef: name: cortex-nova-openstack-keystone namespace: default diff --git a/internal/scheduling/nova/hypervisor_overcommit_controller.go b/internal/scheduling/nova/hypervisor_overcommit_controller.go index 72e4507fc..7df253849 100644 --- a/internal/scheduling/nova/hypervisor_overcommit_controller.go +++ b/internal/scheduling/nova/hypervisor_overcommit_controller.go @@ -217,7 +217,7 @@ func (c *HypervisorOvercommitController) predicateRemoteHypervisor() predicate.P // SetupWithManager sets up the controller with the Manager and a multicluster // client. The multicluster client is used to watch for changes in the // Hypervisor CRD across all clusters and trigger reconciliations accordingly. -func (c *HypervisorOvercommitController) SetupWithManager(mgr ctrl.Manager) (err error) { +func (c *HypervisorOvercommitController) SetupWithManager(mgr ctrl.Manager, mcl *multicluster.Client) (err error) { // This will load the config in a safe way and gracefully handle errors. c.config, err = conf.GetConfig[HypervisorOvercommitConfig]() if err != nil { @@ -227,12 +227,6 @@ func (c *HypervisorOvercommitController) SetupWithManager(mgr ctrl.Manager) (err if err := c.config.Validate(); err != nil { return err } - // Check that the provided client is a multicluster client, since we need - // that to watch for hypervisors across clusters. - mcl, ok := c.Client.(*multicluster.Client) - if !ok { - return errors.New("provided client must be a multicluster client") - } bldr := multicluster.BuildController(mcl, mgr) // The hypervisor crd may be distributed across multiple remote clusters. bldr, err = bldr.WatchesMulticluster(&hv1.Hypervisor{}, diff --git a/internal/scheduling/nova/hypervisor_overcommit_controller_test.go b/internal/scheduling/nova/hypervisor_overcommit_controller_test.go index e52669c3a..f122831eb 100644 --- a/internal/scheduling/nova/hypervisor_overcommit_controller_test.go +++ b/internal/scheduling/nova/hypervisor_overcommit_controller_test.go @@ -725,7 +725,7 @@ func TestHypervisorOvercommitController_SetupWithManager_InvalidClient(t *testin // SetupWithManager should fail - either because config loading fails // (in test environment without config files) or because the client // is not a multicluster client. - err := controller.SetupWithManager(mgr) + err := controller.SetupWithManager(mgr, nil) if err == nil { t.Error("expected error when calling SetupWithManager, got nil") } diff --git a/pkg/clientcache/cache.go b/pkg/clientcache/cache.go new file mode 100644 index 000000000..639fd7663 --- /dev/null +++ b/pkg/clientcache/cache.go @@ -0,0 +1,262 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package clientcache + +import ( + "strconv" + "sync" + "time" + + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// entry is a single overlaid object with the bookkeeping needed for eviction. +type entry struct { + obj client.Object + uid types.UID + resourceVersion string + deleted bool // tombstone: object was deleted through the caching client + expiresAt time.Time +} + +// objectKey identifies an object within a GVK by namespace and name. +type objectKey struct { + namespace string + name string +} + +// overlay is the generic, client-independent core of the cache. It stores +// pending writes keyed by GVK and objectKey and merges them into informer +// read results until the real object is observed in an informer (eviction). +type overlay struct { + mu sync.RWMutex + byGVK map[schema.GroupVersionKind]map[objectKey]*entry + ttl time.Duration + // indexers holds the IndexerFunc per field per GVK, captured from + // IndexField calls, so overlay entries can be matched against FieldSelectors. + indexers map[schema.GroupVersionKind]map[string]client.IndexerFunc +} + +func newOverlay(ttl time.Duration) *overlay { + return &overlay{ + byGVK: make(map[schema.GroupVersionKind]map[objectKey]*entry), + ttl: ttl, + indexers: make(map[schema.GroupVersionKind]map[string]client.IndexerFunc), + } +} + +func keyForObject(obj client.Object) objectKey { + return objectKey{namespace: obj.GetNamespace(), name: obj.GetName()} +} + +// upsert stores a live (non-tombstone) entry for the object. +func (o *overlay) upsert(gvk schema.GroupVersionKind, obj client.Object) { + o.mu.Lock() + defer o.mu.Unlock() + o.ensureGVK(gvk) + o.byGVK[gvk][keyForObject(obj)] = &entry{ + obj: obj.DeepCopyObject().(client.Object), + uid: obj.GetUID(), + resourceVersion: obj.GetResourceVersion(), + deleted: false, + expiresAt: time.Now().Add(o.ttl), + } +} + +// remove stores a tombstone so the object is filtered out of reads until the +// deletion is observed in an informer. +func (o *overlay) remove(gvk schema.GroupVersionKind, obj client.Object) { + o.mu.Lock() + defer o.mu.Unlock() + o.ensureGVK(gvk) + o.byGVK[gvk][keyForObject(obj)] = &entry{ + obj: obj.DeepCopyObject().(client.Object), + uid: obj.GetUID(), + resourceVersion: obj.GetResourceVersion(), + deleted: true, + expiresAt: time.Now().Add(o.ttl), + } +} + +// evictIfSeen removes the overlay entry for obj if the informer-observed object +// matches by UID and its ResourceVersion is at least as new as the cached one. +func (o *overlay) evictIfSeen(gvk schema.GroupVersionKind, obj client.Object) { + o.mu.Lock() + defer o.mu.Unlock() + entries, ok := o.byGVK[gvk] + if !ok { + return + } + key := keyForObject(obj) + e, ok := entries[key] + if !ok { + return + } + // Only evict when the informer sees the same object generation (by UID) at + // a ResourceVersion >= the one we cached. Otherwise the informer might be + // showing an older revision than our pending write. + if e.uid != "" && obj.GetUID() != "" && e.uid != obj.GetUID() { + return + } + if !resourceVersionAtLeast(obj.GetResourceVersion(), e.resourceVersion) { + return + } + delete(entries, key) +} + +// get returns the overlay entry for the key, if present. +func (o *overlay) get(gvk schema.GroupVersionKind, key objectKey) (*entry, bool) { + o.mu.RLock() + defer o.mu.RUnlock() + entries, ok := o.byGVK[gvk] + if !ok { + return nil, false + } + e, ok := entries[key] + return e, ok +} + +// overlayList merges the overlay entries for the GVK into the informer result, +// deduplicating by objectKey (overlay wins), dropping tombstones, and filtering +// overlay-only entries against the list options' label and field selectors. +func (o *overlay) overlayList(gvk schema.GroupVersionKind, existing []runtime.Object, lo *client.ListOptions) []runtime.Object { + o.mu.RLock() + defer o.mu.RUnlock() + entries := o.byGVK[gvk] + if len(entries) == 0 { + return existing + } + + result := make([]runtime.Object, 0, len(existing)+len(entries)) + // Track which overlay keys are handled so overlay-only entries can be added. + handled := make(map[objectKey]bool, len(entries)) + + for _, item := range existing { + obj, ok := item.(client.Object) + if !ok { + result = append(result, item) + continue + } + key := keyForObject(obj) + e, present := entries[key] + if !present { + result = append(result, item) + continue + } + handled[key] = true + // Overlay wins over the informer result for the same key. + if e.deleted { + // Tombstone: drop the object entirely. + continue + } + result = append(result, e.obj.DeepCopyObject()) + } + + // Add overlay-only entries (not present in the informer result) that match + // the list options. + for key, e := range entries { + if handled[key] { + continue + } + if e.deleted { + continue + } + if !o.matchesLocked(gvk, e.obj, lo) { + continue + } + result = append(result, e.obj.DeepCopyObject()) + } + return result +} + +// matchesLocked reports whether obj satisfies the list options' namespace, +// label and field selectors. Callers must hold at least the read lock. +func (o *overlay) matchesLocked(gvk schema.GroupVersionKind, obj client.Object, lo *client.ListOptions) bool { + if lo == nil { + return true + } + if lo.Namespace != "" && obj.GetNamespace() != lo.Namespace { + return false + } + if lo.LabelSelector != nil && !lo.LabelSelector.Matches(labels.Set(obj.GetLabels())) { + return false + } + if lo.FieldSelector != nil && !lo.FieldSelector.Empty() { + set := o.fieldSetLocked(gvk, obj) + if !lo.FieldSelector.Matches(set) { + return false + } + } + return true +} + +// fieldSetLocked builds a fields.Set for obj using the registered IndexerFuncs +// for the GVK. Callers must hold at least the read lock. +func (o *overlay) fieldSetLocked(gvk schema.GroupVersionKind, obj client.Object) fields.Set { + set := fields.Set{} + for field, fn := range o.indexers[gvk] { + for _, v := range fn(obj) { + // A field selector matches a single value; take the first indexed + // value for the field (mirrors controller-runtime cache behaviour). + set[field] = v + break + } + } + return set +} + +// registerIndex captures an IndexerFunc for a field so overlay entries can be +// matched against MatchingFields queries. +func (o *overlay) registerIndex(gvk schema.GroupVersionKind, field string, fn client.IndexerFunc) { + o.mu.Lock() + defer o.mu.Unlock() + if o.indexers[gvk] == nil { + o.indexers[gvk] = make(map[string]client.IndexerFunc) + } + o.indexers[gvk][field] = fn +} + +// cleanupExpired removes entries whose TTL has passed. +func (o *overlay) cleanupExpired(now time.Time) { + o.mu.Lock() + defer o.mu.Unlock() + for _, entries := range o.byGVK { + for key, e := range entries { + if now.After(e.expiresAt) { + delete(entries, key) + } + } + } +} + +func (o *overlay) ensureGVK(gvk schema.GroupVersionKind) { + if o.byGVK[gvk] == nil { + o.byGVK[gvk] = make(map[objectKey]*entry) + } +} + +// resourceVersionAtLeast reports whether observed >= cached, treating +// ResourceVersions as opaque monotonically increasing integers (as the +// kubernetes apiserver guarantees per resource). Unparsable or empty values +// are treated conservatively: an empty cached RV means "evict on any sighting". +func resourceVersionAtLeast(observed, cached string) bool { + if cached == "" { + return true + } + if observed == "" { + return false + } + oi, oerr := strconv.ParseUint(observed, 10, 64) + ci, cerr := strconv.ParseUint(cached, 10, 64) + if oerr != nil || cerr != nil { + // Fall back to string comparison if not integers. + return observed >= cached + } + return oi >= ci +} diff --git a/pkg/clientcache/cache_test.go b/pkg/clientcache/cache_test.go new file mode 100644 index 000000000..5d3cceead --- /dev/null +++ b/pkg/clientcache/cache_test.go @@ -0,0 +1,436 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package clientcache + +import ( + "context" + "sync" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + toolscachek8s "k8s.io/client-go/tools/cache" + ccache "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +const azIndexField = "spec.availabilityZone" + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add to scheme: %v", err) + } + return scheme +} + +func newReservation(name, az, rv string) *v1alpha1.Reservation { + r := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID("uid-" + name), + ResourceVersion: rv, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + AvailabilityZone: az, + }, + } + return r +} + +func newTestClient(t *testing.T, objs ...client.Object) client.Client { + t.Helper() + return fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(objs...). + WithStatusSubresource(&v1alpha1.Reservation{}). + WithIndex(&v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok || res.Spec.AvailabilityZone == "" { + return nil + } + return []string{res.Spec.AvailabilityZone} + }). + Build() +} + +// fakeInformer is a controllable informer that records handlers and lets tests +// fire Add/Update events to trigger eviction. +type fakeInformer struct { + ccache.Informer + mu sync.Mutex + handlers []toolscachek8s.ResourceEventHandler +} + +func (f *fakeInformer) AddEventHandler(h toolscachek8s.ResourceEventHandler) (toolscachek8s.ResourceEventHandlerRegistration, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.handlers = append(f.handlers, h) + return nil, nil +} + +func (f *fakeInformer) fireAdd(obj any) { + f.mu.Lock() + handlers := append([]toolscachek8s.ResourceEventHandler(nil), f.handlers...) + f.mu.Unlock() + for _, h := range handlers { + h.OnAdd(obj, false) + } +} + +func (f *fakeInformer) fireUpdate(oldObj, newObj any) { + f.mu.Lock() + handlers := append([]toolscachek8s.ResourceEventHandler(nil), f.handlers...) + f.mu.Unlock() + for _, h := range handlers { + h.OnUpdate(oldObj, newObj) + } +} + +// fakeInformerSource returns a single shared fakeInformer for all kinds. +type fakeInformerSource struct { + inf *fakeInformer +} + +func (s *fakeInformerSource) GetInformersForKind(ctx context.Context, obj client.Object) ([]ccache.Informer, error) { + return []ccache.Informer{s.inf}, nil +} + +func reservationConfig() Config { + return Config{ + GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, + TTL: metav1.Duration{Duration: 2 * time.Minute}, + } +} + +func newCaching(t *testing.T, inner client.Client, src InformerSource) *CachingClient { + t.Helper() + c, err := New(inner, src, testScheme(t), reservationConfig()) + if err != nil { + t.Fatalf("New: %v", err) + } + return c +} + +func listReservations(t *testing.T, c client.Client, opts ...client.ListOption) []v1alpha1.Reservation { + t.Helper() + var list v1alpha1.ReservationList + if err := c.List(context.Background(), &list, opts...); err != nil { + t.Fatalf("List: %v", err) + } + return list.Items +} + +// 1. Write/Read: Create then immediate List/Get shows the object despite an +// empty informer (fake inner client without the object pre-loaded... but the +// fake client persists creates, so we simulate informer lag by deleting from +// inner after caching — instead we verify overlay independently below). +func TestCreateThenGetVisible(t *testing.T) { + inner := newTestClient(t) + src := &fakeInformerSource{inf: &fakeInformer{}} + c := newCaching(t, inner, src) + + r := newReservation("res-1", "az-1", "") + if err := c.Create(context.Background(), r); err != nil { + t.Fatalf("Create: %v", err) + } + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err != nil { + t.Fatalf("Get after create: %v", err) + } + if got.Spec.AvailabilityZone != "az-1" { + t.Fatalf("expected az-1, got %q", got.Spec.AvailabilityZone) + } +} + +// 2. Overlay with empty informer: inner List returns [], overlay entry still in +// the result. +func TestOverlayWhenInnerEmpty(t *testing.T) { + inner := newTestClient(t) // no objects + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + // Directly seed the overlay to simulate a write whose object is not yet in + // the (empty) inner client. + c.overlay.upsert(reservationGVK(), newReservation("res-2", "az-1", "5")) + + items := listReservations(t, c) + if len(items) != 1 || items[0].Name != "res-2" { + t.Fatalf("expected overlay entry res-2, got %+v", items) + } +} + +// 3. Eviction: an informer sighting (Add or Update) evicts the overlay entry +// only when it matches by UID and carries a ResourceVersion >= the cached one. +func TestEviction(t *testing.T) { + // Cached entry is always uid-res-3 @ RV 10. + const cachedRV = "10" + cases := []struct { + name string + useUpdate bool // fire OnUpdate instead of OnAdd + observedUID string // "" => reuse the cached object's UID + observedRV string + wantEvicted bool + }{ + {name: "add older RV keeps", observedRV: "9", wantEvicted: false}, + {name: "add equal RV evicts", observedRV: "10", wantEvicted: true}, + {name: "add newer RV evicts", observedRV: "11", wantEvicted: true}, + {name: "update newer RV evicts", useUpdate: true, observedRV: "11", wantEvicted: true}, + {name: "update older RV keeps", useUpdate: true, observedRV: "9", wantEvicted: false}, + {name: "uid mismatch keeps", observedUID: "uid-other", observedRV: "11", wantEvicted: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + inner := newTestClient(t) + inf := &fakeInformer{} + c := newCaching(t, inner, &fakeInformerSource{inf: inf}) + + ctx := t.Context() + go func() { + if err := c.Start(ctx); err != nil && ctx.Err() == nil { + t.Errorf("c.Start: %v", err) + } + }() + waitFor(t, func() bool { + inf.mu.Lock() + defer inf.mu.Unlock() + return len(inf.handlers) > 0 + }) + + c.overlay.upsert(reservationGVK(), newReservation("res-3", "az-1", cachedRV)) + + observed := newReservation("res-3", "az-1", tc.observedRV) + if tc.observedUID != "" { + observed.UID = types.UID(tc.observedUID) + } + if tc.useUpdate { + inf.fireUpdate(nil, observed) + } else { + inf.fireAdd(observed) + } + + _, present := c.overlay.get(reservationGVK(), objectKey{name: "res-3"}) + if present == tc.wantEvicted { + t.Fatalf("evicted=%v, want evicted=%v", !present, tc.wantEvicted) + } + }) + } +} + +// TestEvictionIgnoresNonObject: an informer event carrying a non-client.Object +// payload is ignored and does not panic or evict. +func TestEvictionIgnoresNonObject(t *testing.T) { + inner := newTestClient(t) + inf := &fakeInformer{} + c := newCaching(t, inner, &fakeInformerSource{inf: inf}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + if err := c.Start(ctx); err != nil && ctx.Err() == nil { + t.Errorf("c.Start: %v", err) + } + }() + waitFor(t, func() bool { + inf.mu.Lock() + defer inf.mu.Unlock() + return len(inf.handlers) > 0 + }) + + c.overlay.upsert(reservationGVK(), newReservation("res-x", "az-1", "1")) + inf.fireAdd("not-an-object") + if _, ok := c.overlay.get(reservationGVK(), objectKey{name: "res-x"}); !ok { + t.Fatalf("non-object event must not evict the entry") + } +} + +// 4. TTL: cleanupExpired removes expired entries. +func TestTTLCleanup(t *testing.T) { + o := newOverlay(time.Minute) + o.upsert(reservationGVK(), newReservation("res-4", "az-1", "1")) + // Force expiry. + o.mu.Lock() + for _, entries := range o.byGVK { + for _, e := range entries { + e.expiresAt = time.Now().Add(-time.Second) + } + } + o.mu.Unlock() + o.cleanupExpired(time.Now()) + if _, ok := o.get(reservationGVK(), objectKey{name: "res-4"}); ok { + t.Fatalf("expired entry should be removed") + } +} + +// 5. Tombstone: Delete filters the object from List/Get even though inner still +// has it. +func TestTombstone(t *testing.T) { + r := newReservation("res-5", "az-1", "") + inner := newTestClient(t, r) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + if err := c.Delete(context.Background(), r); err != nil { + t.Fatalf("Delete: %v", err) + } + // Re-add to inner to simulate informer lag: fake client already removed it, + // so re-create via inner directly (bypassing overlay). + if err := inner.Create(context.Background(), newReservation("res-5", "az-1", "")); err != nil { + t.Fatalf("re-create inner: %v", err) + } + + var got v1alpha1.Reservation + err := c.Get(context.Background(), types.NamespacedName{Name: "res-5"}, &got) + if !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound for tombstoned object, got %v", err) + } + items := listReservations(t, c) + if len(items) != 0 { + t.Fatalf("expected tombstone to filter from list, got %+v", items) + } +} + +// 6. Update/Patch: newer overlay version overrides stale inner read. +func TestUpdateOverridesInner(t *testing.T) { + r := newReservation("res-6", "az-old", "1") + inner := newTestClient(t, r) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + // Read current to obtain the up-to-date ResourceVersion for update. + var cur v1alpha1.Reservation + if err := inner.Get(context.Background(), types.NamespacedName{Name: "res-6"}, &cur); err != nil { + t.Fatalf("inner get: %v", err) + } + cur.Spec.AvailabilityZone = "az-new" + if err := c.Update(context.Background(), &cur); err != nil { + t.Fatalf("Update: %v", err) + } + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-6"}, &got); err != nil { + t.Fatalf("Get: %v", err) + } + if got.Spec.AvailabilityZone != "az-new" { + t.Fatalf("expected az-new from overlay, got %q", got.Spec.AvailabilityZone) + } +} + +// 7. Label-matching: overlay-only entry appears only for matching MatchingLabels. +func TestLabelMatching(t *testing.T) { + inner := newTestClient(t) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + r := newReservation("res-7", "az-1", "1") + r.Labels = map[string]string{"team": "a"} + c.overlay.upsert(reservationGVK(), r) + + match := listReservations(t, c, client.MatchingLabels{"team": "a"}) + if len(match) != 1 { + t.Fatalf("expected match for team=a, got %+v", match) + } + noMatch := listReservations(t, c, client.MatchingLabels{"team": "b"}) + if len(noMatch) != 0 { + t.Fatalf("expected no match for team=b, got %+v", noMatch) + } +} + +// 8. Field-matching: after IndexField registration, overlay-only entry appears +// only for matching MatchingFields. +func TestFieldMatching(t *testing.T) { + inner := newTestClient(t) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + if err := c.IndexField(context.Background(), &v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { + res := obj.(*v1alpha1.Reservation) + if res.Spec.AvailabilityZone == "" { + return nil + } + return []string{res.Spec.AvailabilityZone} + }); err != nil { + t.Fatalf("IndexField: %v", err) + } + + c.overlay.upsert(reservationGVK(), newReservation("res-8", "az-1", "1")) + + match := listReservations(t, c, client.MatchingFields{azIndexField: "az-1"}) + if len(match) != 1 { + t.Fatalf("expected field match az-1, got %+v", match) + } + noMatch := listReservations(t, c, client.MatchingFields{azIndexField: "az-2"}) + if len(noMatch) != 0 { + t.Fatalf("expected no field match az-2, got %+v", noMatch) + } +} + +// 9. Non-cached GVK: calls pass through unchanged (no overlay effect). +func TestNonCachedGVKPassthrough(t *testing.T) { + inner := newTestClient(t) + // Config with no GVKs → Reservation is not cached. + c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{}) + if err != nil { + t.Fatalf("New: %v", err) + } + r := newReservation("res-9", "az-1", "") + if err := c.Create(context.Background(), r); err != nil { + t.Fatalf("Create: %v", err) + } + // Overlay must be empty for non-cached GVK. + if _, ok := c.overlay.get(reservationGVK(), objectKey{name: "res-9"}); ok { + t.Fatalf("non-cached GVK should not populate overlay") + } + // Delete it in inner, then Get should be NotFound (no overlay resurrection). + if err := c.Delete(context.Background(), r); err != nil { + t.Fatalf("Delete: %v", err) + } + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-9"}, &got); !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound, got %v", err) + } +} + +// 10. Dedup: object present in both informer (inner) and overlay appears once. +func TestDedup(t *testing.T) { + r := newReservation("res-10", "az-1", "1") + inner := newTestClient(t, r) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + // Overlay holds a newer version of the same object. + newer := newReservation("res-10", "az-1", "2") + newer.Spec.TargetHost = "host-x" + c.overlay.upsert(reservationGVK(), newer) + + items := listReservations(t, c) + if len(items) != 1 { + t.Fatalf("expected exactly one item after dedup, got %d: %+v", len(items), items) + } + if items[0].Spec.TargetHost != "host-x" { + t.Fatalf("expected overlay version to win, got %+v", items[0]) + } +} + +// helpers + +func reservationGVK() schema.GroupVersionKind { + return v1alpha1.GroupVersion.WithKind("Reservation") +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("condition not met within timeout") +} diff --git a/pkg/clientcache/client.go b/pkg/clientcache/client.go new file mode 100644 index 000000000..e9a4b01f6 --- /dev/null +++ b/pkg/clientcache/client.go @@ -0,0 +1,265 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package clientcache + +import ( + "context" + "errors" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// defaultTTL is used when Config.TTL is zero. +const defaultTTL = 2 * time.Minute + +// CachingClient wraps an inner client.Client with a transparent in-process +// overlay. Writes populate the overlay; reads merge the overlay with the inner +// (informer-backed) result; entries are evicted once the real object appears in +// an informer (see runnable.go) or after TTL expiry. +// +// It embeds client.Client so all methods not overridden below are delegated to +// the inner client unchanged. +type CachingClient struct { + client.Client // inner client, used for delegation + + informers InformerSource + scheme *runtime.Scheme + overlay *overlay + ttl time.Duration + gvks map[schema.GroupVersionKind]bool +} + +// New builds a CachingClient wrapping inner. informers supplies the informers +// used for eviction, scheme resolves object GVKs, and conf lists the GVKs to +// overlay and the TTL. GVK strings are formatted as "//" +// and are resolved against scheme. +func New(inner client.Client, informers InformerSource, scheme *runtime.Scheme, conf Config) (*CachingClient, error) { + gvks, err := resolveGVKs(scheme, conf.GVKs) + if err != nil { + return nil, err + } + ttl := conf.TTL.Duration + if ttl <= 0 { + ttl = defaultTTL + } + return &CachingClient{ + Client: inner, + informers: informers, + scheme: scheme, + overlay: newOverlay(ttl), + ttl: ttl, + gvks: gvks, + }, nil +} + +// resolveGVKs maps "//" strings to GVKs via the scheme's +// known types. Mirrors the resolution logic of multicluster.InitFromConf, but +// stays local to this package. +func resolveGVKs(scheme *runtime.Scheme, gvkStrs []string) (map[schema.GroupVersionKind]bool, error) { + byStr := make(map[string]schema.GroupVersionKind) + for gvk := range scheme.AllKnownTypes() { + byStr[gvk.GroupVersion().String()+"/"+gvk.Kind] = gvk + } + out := make(map[schema.GroupVersionKind]bool, len(gvkStrs)) + for _, s := range gvkStrs { + gvk, ok := byStr[s] + if !ok { + return nil, errors.New("clientcache: no gvk registered in scheme for " + s) + } + out[gvk] = true + } + return out, nil +} + +// Inner returns the wrapped client, e.g. for use with a controller Builder that +// needs the raw client rather than the caching wrapper. +func (c *CachingClient) Inner() client.Client { return c.Client } + +// gvkFor resolves the GVK of obj and reports whether it is cached. +func (c *CachingClient) gvkFor(obj runtime.Object) (schema.GroupVersionKind, bool) { + gvks, _, err := c.scheme.ObjectKinds(obj) + if err != nil || len(gvks) != 1 { + return schema.GroupVersionKind{}, false + } + gvk := gvks[0] + return gvk, c.gvks[gvk] +} + +// itemGVKForList resolves the singular item GVK for a list object and reports +// whether that item GVK is cached. The list GVK's Kind ends with "List". +func (c *CachingClient) itemGVKForList(list client.ObjectList) (schema.GroupVersionKind, bool) { + gvks, _, err := c.scheme.ObjectKinds(list) + if err != nil || len(gvks) != 1 { + return schema.GroupVersionKind{}, false + } + gvk := gvks[0] + if kind, ok := trimListSuffix(gvk.Kind); ok { + gvk.Kind = kind + } + return gvk, c.gvks[gvk] +} + +// trimListSuffix strips a trailing "List" from a Kind, reporting whether it did. +func trimListSuffix(kind string) (string, bool) { + const suffix = "List" + if len(kind) > len(suffix) && kind[len(kind)-len(suffix):] == suffix { + return kind[:len(kind)-len(suffix)], true + } + return kind, false +} + +// Create delegates to the inner client and, on success for a cached GVK, adds +// the object to the overlay. +func (c *CachingClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if err := c.Client.Create(ctx, obj, opts...); err != nil { + return err + } + if gvk, cached := c.gvkFor(obj); cached { + c.overlay.upsert(gvk, obj) + } + return nil +} + +// Update delegates to the inner client and, on success for a cached GVK, +// refreshes the overlay entry. +func (c *CachingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if err := c.Client.Update(ctx, obj, opts...); err != nil { + return err + } + if gvk, cached := c.gvkFor(obj); cached { + c.overlay.upsert(gvk, obj) + } + return nil +} + +// Patch delegates to the inner client and, on success for a cached GVK, +// refreshes the overlay entry with the patched object. +func (c *CachingClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if err := c.Client.Patch(ctx, obj, patch, opts...); err != nil { + return err + } + if gvk, cached := c.gvkFor(obj); cached { + c.overlay.upsert(gvk, obj) + } + return nil +} + +// Delete delegates to the inner client and, on success for a cached GVK, stores +// a tombstone in the overlay. +func (c *CachingClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if err := c.Client.Delete(ctx, obj, opts...); err != nil { + return err + } + if gvk, cached := c.gvkFor(obj); cached { + c.overlay.remove(gvk, obj) + } + return nil +} + +// Get delegates to the inner client, then applies the overlay: a tombstone +// yields NotFound; a live overlay entry overrides the inner result; and an +// overlay entry can satisfy a Get that the inner client reports as NotFound. +func (c *CachingClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.Get(ctx, key, obj, opts...) + } + err := c.Client.Get(ctx, key, obj, opts...) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + e, ok := c.overlay.get(gvk, objectKey{namespace: key.Namespace, name: key.Name}) + if !ok { + // No overlay entry: return the inner result (value or NotFound) as-is. + return err + } + if e.deleted { + return apierrors.NewNotFound(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, key.Name) + } + // Live overlay entry: copy it into obj, overriding the inner result. + if cpErr := c.scheme.Convert(e.obj, obj, nil); cpErr != nil { + return cpErr + } + return nil +} + +// List delegates to the inner client, then merges the overlay into the result. +func (c *CachingClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + itemGVK, cached := c.itemGVKForList(list) + if !cached { + return c.Client.List(ctx, list, opts...) + } + if err := c.Client.List(ctx, list, opts...); err != nil { + return err + } + items, err := meta.ExtractList(list) + if err != nil { + return err + } + lo := &client.ListOptions{} + lo.ApplyOptions(opts) + merged := c.overlay.overlayList(itemGVK, items, lo) + return meta.SetList(list, merged) +} + +// IndexField delegates to the inner client (if it is a FieldIndexer) and also +// registers the IndexerFunc with the overlay so overlay entries can be matched +// against MatchingFields. +func (c *CachingClient) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { + if indexer, ok := c.Client.(client.FieldIndexer); ok { + if err := indexer.IndexField(ctx, obj, field, extractValue); err != nil { + return err + } + } + if gvk, cached := c.gvkFor(obj); cached { + c.overlay.registerIndex(gvk, field, extractValue) + } + return nil +} + +// Status returns a status writer that mirrors status Update/Patch writes for +// cached GVKs into the overlay. +func (c *CachingClient) Status() client.StatusWriter { + return &statusWriter{c: c, inner: c.Client.Status()} +} + +// statusWriter wraps the inner status writer and reflects status writes into +// the overlay for cached GVKs. +type statusWriter struct { + c *CachingClient + inner client.StatusWriter +} + +func (s *statusWriter) Create(ctx context.Context, obj, subResource client.Object, opts ...client.SubResourceCreateOption) error { + return s.inner.Create(ctx, obj, subResource, opts...) +} + +func (s *statusWriter) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if err := s.inner.Update(ctx, obj, opts...); err != nil { + return err + } + if gvk, cached := s.c.gvkFor(obj); cached { + s.c.overlay.upsert(gvk, obj) + } + return nil +} + +func (s *statusWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + if err := s.inner.Patch(ctx, obj, patch, opts...); err != nil { + return err + } + if gvk, cached := s.c.gvkFor(obj); cached { + s.c.overlay.upsert(gvk, obj) + } + return nil +} + +func (s *statusWriter) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + return s.inner.Apply(ctx, obj, opts...) +} diff --git a/pkg/clientcache/client_test.go b/pkg/clientcache/client_test.go new file mode 100644 index 000000000..08d043f26 --- /dev/null +++ b/pkg/clientcache/client_test.go @@ -0,0 +1,470 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package clientcache + +import ( + "context" + "errors" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +// errClient wraps an inner client.Client and injects a configurable error into +// each mutating/read operation, so the error-propagation paths of +// CachingClient (which must not touch the overlay on failure) can be exercised. +type errClient struct { + client.Client + createErr error + updateErr error + patchErr error + deleteErr error + getErr error + listErr error +} + +func (e *errClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if e.createErr != nil { + return e.createErr + } + return e.Client.Create(ctx, obj, opts...) +} + +func (e *errClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if e.updateErr != nil { + return e.updateErr + } + return e.Client.Update(ctx, obj, opts...) +} + +func (e *errClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if e.patchErr != nil { + return e.patchErr + } + return e.Client.Patch(ctx, obj, patch, opts...) +} + +func (e *errClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if e.deleteErr != nil { + return e.deleteErr + } + return e.Client.Delete(ctx, obj, opts...) +} + +func (e *errClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if e.getErr != nil { + return e.getErr + } + return e.Client.Get(ctx, key, obj, opts...) +} + +func (e *errClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if e.listErr != nil { + return e.listErr + } + return e.Client.List(ctx, list, opts...) +} + +// forceInnerAZ writes a divergent AvailabilityZone directly to the inner client, +// bypassing the caching wrapper (and its overlay). Used to prove that reads +// through the caching client are served from the overlay, not the inner client. +func forceInnerAZ(t *testing.T, inner client.Client, name, az string) { + t.Helper() + var cur v1alpha1.Reservation + if err := inner.Get(context.Background(), types.NamespacedName{Name: name}, &cur); err != nil { + t.Fatalf("forceInnerAZ get: %v", err) + } + cur.Spec.AvailabilityZone = az + if err := inner.Update(context.Background(), &cur); err != nil { + t.Fatalf("forceInnerAZ update: %v", err) + } +} + +// forceInnerStatusHost writes a divergent status Host directly to the inner +// client, bypassing the caching wrapper. +func forceInnerStatusHost(t *testing.T, inner client.Client, name, host string) { + t.Helper() + var cur v1alpha1.Reservation + if err := inner.Get(context.Background(), types.NamespacedName{Name: name}, &cur); err != nil { + t.Fatalf("forceInnerStatusHost get: %v", err) + } + cur.Status.Host = host + if err := inner.Status().Update(context.Background(), &cur); err != nil { + t.Fatalf("forceInnerStatusHost update: %v", err) + } +} + +// TestNewUnknownGVKError: New fails when a configured GVK string is not +// registered in the scheme. +func TestNewUnknownGVKError(t *testing.T) { + inner := newTestClient(t) + _, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{ + GVKs: []string{"cortex.cloud/v1alpha1/DoesNotExist"}, + }) + if err == nil { + t.Fatalf("expected error for unknown GVK, got nil") + } +} + +// TestNewDefaultTTL: a zero TTL in the config falls back to defaultTTL. +func TestNewDefaultTTL(t *testing.T) { + inner := newTestClient(t) + c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{ + GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + if c.ttl != defaultTTL { + t.Fatalf("expected ttl %v, got %v", defaultTTL, c.ttl) + } + if c.overlay.ttl != defaultTTL { + t.Fatalf("expected overlay ttl %v, got %v", defaultTTL, c.overlay.ttl) + } +} + +// TestNewExplicitTTL: a non-zero TTL is honoured verbatim. +func TestNewExplicitTTL(t *testing.T) { + inner := newTestClient(t) + c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{ + GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, + TTL: metav1.Duration{Duration: 90 * time.Second}, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + if c.ttl != 90*time.Second { + t.Fatalf("expected ttl 90s, got %v", c.ttl) + } +} + +// TestInnerReturnsWrappedClient: Inner returns the exact client passed to New. +func TestInnerReturnsWrappedClient(t *testing.T) { + inner := newTestClient(t) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + if c.Inner() != inner { + t.Fatalf("Inner did not return the wrapped client") + } +} + +// TestWriteErrorLeavesOverlayUntouched: for every mutating method, a failing +// inner call surfaces the error and leaves the overlay untouched (no live entry +// and no tombstone). +func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { + sentinel := errors.New("boom") + cases := []struct { + name string + rv string // ResourceVersion for the object passed to the op + seed bool // pre-seed the object in the inner client (delete needs it) + // mkClient wraps an inner client (which may already contain r) with the + // relevant injected error. + mkClient func(inner client.Client) client.Client + op func(c *CachingClient, r *v1alpha1.Reservation) error + }{ + { + name: "create", + mkClient: func(inner client.Client) client.Client { return &errClient{Client: inner, createErr: sentinel} }, + op: func(c *CachingClient, r *v1alpha1.Reservation) error { return c.Create(context.Background(), r) }, + }, + { + name: "update", + rv: "1", + mkClient: func(inner client.Client) client.Client { return &errClient{Client: inner, updateErr: sentinel} }, + op: func(c *CachingClient, r *v1alpha1.Reservation) error { return c.Update(context.Background(), r) }, + }, + { + name: "patch", + rv: "1", + mkClient: func(inner client.Client) client.Client { return &errClient{Client: inner, patchErr: sentinel} }, + op: func(c *CachingClient, r *v1alpha1.Reservation) error { + p := r.DeepCopy() + p.Spec.AvailabilityZone = "az-new" + return c.Patch(context.Background(), p, client.MergeFrom(r)) + }, + }, + { + name: "delete", + seed: true, + mkClient: func(inner client.Client) client.Client { return &errClient{Client: inner, deleteErr: sentinel} }, + op: func(c *CachingClient, r *v1alpha1.Reservation) error { return c.Delete(context.Background(), r) }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := newReservation("res-"+tc.name, "az-1", tc.rv) + var base client.Client + if tc.seed { + base = newTestClient(t, r) + } else { + base = newTestClient(t) + } + c := newCaching(t, tc.mkClient(base), &fakeInformerSource{inf: &fakeInformer{}}) + + if err := tc.op(c, r); !errors.Is(err, sentinel) { + t.Fatalf("expected sentinel error, got %v", err) + } + if _, ok := c.overlay.get(reservationGVK(), objectKey{name: r.Name}); ok { + t.Fatalf("overlay must not be touched on %s failure", tc.name) + } + }) + } +} + +// TestWriteServedFromOverlay: after a write through the caching client, a Get +// returns the written value even though the inner client has been forced to a +// divergent (stale) value behind the cache's back. This proves the read path is +// actually served from the overlay, not merely that the overlay was written. +func TestWriteServedFromOverlay(t *testing.T) { + cases := []struct { + name string + // write performs the write under test through c, given the current + // object cur fetched from inner, and returns the value it wrote. + write func(t *testing.T, c *CachingClient, cur *v1alpha1.Reservation) string + // diverge forces the inner client to a stale value behind the cache. + diverge func(t *testing.T, inner client.Client, name string) + // read extracts the field under test from a Get result. + read func(*v1alpha1.Reservation) string + }{ + { + name: "patch spec", + write: func(t *testing.T, c *CachingClient, cur *v1alpha1.Reservation) string { + base := cur.DeepCopy() + cur.Spec.AvailabilityZone = "az-new" + if err := c.Patch(context.Background(), cur, client.MergeFrom(base)); err != nil { + t.Fatalf("Patch: %v", err) + } + return "az-new" + }, + diverge: func(t *testing.T, inner client.Client, name string) { forceInnerAZ(t, inner, name, "az-stale") }, + read: func(r *v1alpha1.Reservation) string { return r.Spec.AvailabilityZone }, + }, + { + name: "status update", + write: func(t *testing.T, c *CachingClient, cur *v1alpha1.Reservation) string { + cur.Status.Host = "host-active" + if err := c.Status().Update(context.Background(), cur); err != nil { + t.Fatalf("Status().Update: %v", err) + } + return "host-active" + }, + diverge: func(t *testing.T, inner client.Client, name string) { + forceInnerStatusHost(t, inner, name, "host-stale") + }, + read: func(r *v1alpha1.Reservation) string { return r.Status.Host }, + }, + { + name: "status patch", + write: func(t *testing.T, c *CachingClient, cur *v1alpha1.Reservation) string { + base := cur.DeepCopy() + cur.Status.Host = "host-patched" + if err := c.Status().Patch(context.Background(), cur, client.MergeFrom(base)); err != nil { + t.Fatalf("Status().Patch: %v", err) + } + return "host-patched" + }, + diverge: func(t *testing.T, inner client.Client, name string) { + forceInnerStatusHost(t, inner, name, "host-stale") + }, + read: func(r *v1alpha1.Reservation) string { return r.Status.Host }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := newReservation("res-served", "az-1", "") + inner := newTestClient(t, r) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + var cur v1alpha1.Reservation + if err := inner.Get(context.Background(), types.NamespacedName{Name: r.Name}, &cur); err != nil { + t.Fatalf("inner get: %v", err) + } + want := tc.write(t, c, &cur) + tc.diverge(t, inner, r.Name) + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: r.Name}, &got); err != nil { + t.Fatalf("Get: %v", err) + } + if tc.read(&got) != want { + t.Fatalf("expected overlay value %q to be served, got %q", want, tc.read(&got)) + } + }) + } +} + +// TestGetPropagatesNonNotFoundError: a cached-GVK Get surfaces inner errors +// other than NotFound without consulting the overlay. +func TestGetPropagatesNonNotFoundError(t *testing.T) { + sentinel := errors.New("get boom") + inner := &errClient{Client: newTestClient(t), getErr: sentinel} + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + // Seed a live overlay entry that must NOT mask the underlying error. + c.overlay.upsert(reservationGVK(), newReservation("res-ge", "az-1", "1")) + + var got v1alpha1.Reservation + err := c.Get(context.Background(), types.NamespacedName{Name: "res-ge"}, &got) + if !errors.Is(err, sentinel) { + t.Fatalf("expected sentinel error, got %v", err) + } +} + +// TestGetOverlayResurrectsNotFound: a live overlay entry satisfies a Get that +// the inner client reports as NotFound (write not yet visible in the informer). +func TestGetOverlayResurrectsNotFound(t *testing.T) { + inner := newTestClient(t) // empty: inner Get returns NotFound + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + c.overlay.upsert(reservationGVK(), newReservation("res-gr", "az-z", "1")) + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-gr"}, &got); err != nil { + t.Fatalf("expected overlay to satisfy Get, got %v", err) + } + if got.Spec.AvailabilityZone != "az-z" { + t.Fatalf("expected az-z from overlay, got %q", got.Spec.AvailabilityZone) + } +} + +// TestGetNotFoundWithNoOverlay: inner NotFound with no overlay entry propagates +// NotFound unchanged for a cached GVK. +func TestGetNotFoundWithNoOverlay(t *testing.T) { + inner := newTestClient(t) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + var got v1alpha1.Reservation + err := c.Get(context.Background(), types.NamespacedName{Name: "missing"}, &got) + if !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound, got %v", err) + } +} + +// TestGetNonCachedPropagatesError: for a non-cached GVK, Get is a pure +// passthrough and surfaces the inner error verbatim. +func TestGetNonCachedPropagatesError(t *testing.T) { + sentinel := errors.New("get boom") + inner := &errClient{Client: newTestClient(t), getErr: sentinel} + c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{}) + if err != nil { + t.Fatalf("New: %v", err) + } + var got v1alpha1.Reservation + if gerr := c.Get(context.Background(), types.NamespacedName{Name: "x"}, &got); !errors.Is(gerr, sentinel) { + t.Fatalf("expected sentinel error, got %v", gerr) + } +} + +// TestListPropagatesError: for a cached GVK, a failing inner List surfaces the +// error rather than returning a partial overlay merge. +func TestListPropagatesError(t *testing.T) { + sentinel := errors.New("list boom") + inner := &errClient{Client: newTestClient(t), listErr: sentinel} + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + c.overlay.upsert(reservationGVK(), newReservation("res-le", "az-1", "1")) + + var list v1alpha1.ReservationList + if err := c.List(context.Background(), &list); !errors.Is(err, sentinel) { + t.Fatalf("expected sentinel error, got %v", err) + } +} + +// TestStatusUpdateErrorLeavesOverlayUntouched: a failed status update does not +// populate the overlay. +func TestStatusUpdateErrorLeavesOverlayUntouched(t *testing.T) { + inner := newTestClient(t) // object absent → status update fails + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + r := newReservation("res-se", "az-1", "1") + if err := c.Status().Update(context.Background(), r); err == nil { + t.Fatalf("expected status update to fail for missing object") + } + if _, ok := c.overlay.get(reservationGVK(), objectKey{name: "res-se"}); ok { + t.Fatalf("overlay must not be populated on status update failure") + } +} + +// TestStatusCreateDelegates: Status().Create delegates to the inner status +// writer (fake client reports it unsupported) and never touches the overlay. +func TestStatusCreateDelegates(t *testing.T) { + r := newReservation("res-sc", "az-1", "") + inner := newTestClient(t, r) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + // The fake client does not support subresource Create; we only assert the + // call is delegated (returns an error) and the overlay stays empty. + if err := c.Status().Create(context.Background(), r, r); err == nil { + t.Fatalf("expected Status().Create to fail on fake client") + } + if _, ok := c.overlay.get(reservationGVK(), objectKey{name: "res-sc"}); ok { + t.Fatalf("Status().Create must not populate the overlay") + } +} + +// TestStatusUpdateNonCachedNoOverlay: Status().Update for a non-cached GVK does +// not touch the overlay. +func TestStatusUpdateNonCachedNoOverlay(t *testing.T) { + r := newReservation("res-sn", "az-1", "") + inner := newTestClient(t, r) + c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{}) + if err != nil { + t.Fatalf("New: %v", err) + } + var cur v1alpha1.Reservation + if err := inner.Get(context.Background(), types.NamespacedName{Name: "res-sn"}, &cur); err != nil { + t.Fatalf("inner get: %v", err) + } + cur.Status.Host = "host-active" + if err := c.Status().Update(context.Background(), &cur); err != nil { + t.Fatalf("Status().Update: %v", err) + } + if _, ok := c.overlay.get(reservationGVK(), objectKey{name: "res-sn"}); ok { + t.Fatalf("non-cached GVK status update should not populate overlay") + } +} + +// TestGVKForUnknownType: gvkFor reports not-cached for a type not registered in +// the scheme. +func TestGVKForUnknownType(t *testing.T) { + inner := newTestClient(t) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + if _, cached := c.gvkFor(&unknownObject{}); cached { + t.Fatalf("unknown type must not be reported as cached") + } +} + +// TestTrimListSuffix exercises the list-kind suffix trimming helper. +func TestTrimListSuffix(t *testing.T) { + cases := []struct { + in string + wantKind string + wantOK bool + }{ + {"ReservationList", "Reservation", true}, + {"List", "List", false}, // len(kind) not > len("List") + {"Reservation", "Reservation", false}, + {"", "", false}, + } + for _, tc := range cases { + gotKind, gotOK := trimListSuffix(tc.in) + if gotKind != tc.wantKind || gotOK != tc.wantOK { + t.Errorf("trimListSuffix(%q) = (%q, %v), want (%q, %v)", tc.in, gotKind, gotOK, tc.wantKind, tc.wantOK) + } + } +} + +// unknownObject is a client.Object whose type is not registered in the test +// scheme, used to exercise the "unresolvable GVK" branches. +type unknownObject struct { + metav1.TypeMeta + metav1.ObjectMeta +} + +func (u *unknownObject) DeepCopyObject() runtime.Object { return u } diff --git a/pkg/clientcache/config.go b/pkg/clientcache/config.go new file mode 100644 index 000000000..57d5d7a70 --- /dev/null +++ b/pkg/clientcache/config.go @@ -0,0 +1,24 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package clientcache + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Config configures the transparent in-process overlay cache. +type Config struct { + // GVKs the cache should overlay, formatted as "//". + // Calls for GVKs not listed here are passed through unchanged. + GVKs []string `json:"gvks"` + // TTL is the maximum lifetime of an overlay entry before it is evicted by + // the background cleanup goroutine, guarding against entries that never + // appear in the informer (e.g. after a crash). Defaults to 2m when zero. + TTL metav1.Duration `json:"ttl,omitempty"` +} + +// RootConfig is the top-level config key for the client cache. +type RootConfig struct { + ClientCache Config `json:"clientcache"` +} diff --git a/pkg/clientcache/interfaces.go b/pkg/clientcache/interfaces.go new file mode 100644 index 000000000..8c2e87599 --- /dev/null +++ b/pkg/clientcache/interfaces.go @@ -0,0 +1,22 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package clientcache + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// InformerSource provides, per object type, the informers the cache attaches +// to for eviction purposes. It is satisfied structurally e.g. by +// *multicluster.Client (via ClustersForGVK + cluster.GetCache().GetInformer), +// so that this package does not need to import pkg/multicluster. +type InformerSource interface { + // GetInformersForKind returns all informers serving the GVK of the given + // object. The cache attaches Add/Update event handlers to each informer to + // evict overlay entries once the real object appears in the informer cache. + GetInformersForKind(ctx context.Context, obj client.Object) ([]cache.Informer, error) +} diff --git a/pkg/clientcache/runnable.go b/pkg/clientcache/runnable.go new file mode 100644 index 000000000..048fd163c --- /dev/null +++ b/pkg/clientcache/runnable.go @@ -0,0 +1,87 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package clientcache + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/runtime/schema" + toolscachek8s "k8s.io/client-go/tools/cache" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// minCleanupInterval bounds the TTL cleanup ticker from below. +const minCleanupInterval = 30 * time.Second + +// Start implements manager.Runnable. It attaches informer event handlers for +// eviction and runs the TTL cleanup loop until ctx is done. +func (c *CachingClient) Start(ctx context.Context) error { + log := ctrl.LoggerFrom(ctx).WithName("clientcache") + + for gvk := range c.gvks { + obj, err := c.newObjectForGVK(gvk) + if err != nil { + log.Error(err, "failed to build object for gvk; eviction disabled for it", "gvk", gvk) + continue + } + informers, err := c.informers.GetInformersForKind(ctx, obj) + if err != nil { + log.Error(err, "failed to get informers for gvk; eviction disabled for it", "gvk", gvk) + continue + } + handler := c.evictionHandler(gvk) + for _, inf := range informers { + if _, err := inf.AddEventHandler(handler); err != nil { + log.Error(err, "failed to add eviction event handler", "gvk", gvk) + } + } + } + + interval := c.ttl / 4 + if interval < minCleanupInterval { + interval = minCleanupInterval + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case now := <-ticker.C: + c.overlay.cleanupExpired(now) + } + } +} + +// evictionHandler returns an informer event handler that evicts overlay entries +// for the GVK when the real object is observed at a >= ResourceVersion. +func (c *CachingClient) evictionHandler(gvk schema.GroupVersionKind) toolscachek8s.ResourceEventHandler { + evict := func(o any) { + obj, ok := o.(client.Object) + if !ok { + return + } + c.overlay.evictIfSeen(gvk, obj) + } + return toolscachek8s.ResourceEventHandlerFuncs{ + AddFunc: func(o any) { evict(o) }, + UpdateFunc: func(_, o any) { evict(o) }, + } +} + +// newObjectForGVK builds an empty typed object for the GVK using the scheme. +func (c *CachingClient) newObjectForGVK(gvk schema.GroupVersionKind) (client.Object, error) { + ro, err := c.scheme.New(gvk) + if err != nil { + return nil, err + } + obj, ok := ro.(client.Object) + if !ok { + return nil, fmt.Errorf("clientcache: object for gvk %s does not implement client.Object", gvk) + } + return obj, nil +} diff --git a/pkg/multicluster/client.go b/pkg/multicluster/client.go index a8e597c8d..afae980af 100644 --- a/pkg/multicluster/client.go +++ b/pkg/multicluster/client.go @@ -17,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" ) @@ -219,6 +220,30 @@ func (c *Client) ClustersForGVK(gvk schema.GroupVersionKind) ([]cluster.Cluster, return clusters, nil } +// GetInformersForKind returns the informers of all clusters serving the GVK of +// the given object. It is used by the in-process client cache to attach +// eviction event handlers. The GVK is resolved against the home scheme and must +// be explicitly configured in home or a remote cluster. +func (c *Client) GetInformersForKind(ctx context.Context, obj client.Object) ([]cache.Informer, error) { + gvk, err := c.GVKFromHomeScheme(obj) + if err != nil { + return nil, err + } + clusters, err := c.ClustersForGVK(gvk) + if err != nil { + return nil, err + } + informers := make([]cache.Informer, 0, len(clusters)) + for _, cl := range clusters { + inf, err := cl.GetCache().GetInformer(ctx, obj) + if err != nil { + return nil, err + } + informers = append(informers, inf) + } + return informers, nil +} + // clusterForWrite uses a ResourceRouter to determine which remote cluster // a resource should be written to based on the resource content and cluster labels. // From c201d3d5faf2726e279ff7ed54abbe31e78b84a2 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Tue, 14 Jul 2026 12:41:47 +0200 Subject: [PATCH 02/16] fix: simplify interval calculation for cleanup ticker --- pkg/clientcache/runnable.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/clientcache/runnable.go b/pkg/clientcache/runnable.go index 048fd163c..836bee0e6 100644 --- a/pkg/clientcache/runnable.go +++ b/pkg/clientcache/runnable.go @@ -41,10 +41,7 @@ func (c *CachingClient) Start(ctx context.Context) error { } } - interval := c.ttl / 4 - if interval < minCleanupInterval { - interval = minCleanupInterval - } + interval := max(c.ttl/4, minCleanupInterval) ticker := time.NewTicker(interval) defer ticker.Stop() for { From d9cf87ee23463d665ff47f7f0d57af3a2752a52e Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Thu, 6 Aug 2026 10:08:07 +0200 Subject: [PATCH 03/16] refactor: remove Inner method from CachingClient and associated test Signed-off-by: Markus Wieland --- pkg/clientcache/client.go | 4 ---- pkg/clientcache/client_test.go | 9 --------- 2 files changed, 13 deletions(-) diff --git a/pkg/clientcache/client.go b/pkg/clientcache/client.go index e9a4b01f6..d4560a936 100644 --- a/pkg/clientcache/client.go +++ b/pkg/clientcache/client.go @@ -77,10 +77,6 @@ func resolveGVKs(scheme *runtime.Scheme, gvkStrs []string) (map[schema.GroupVers return out, nil } -// Inner returns the wrapped client, e.g. for use with a controller Builder that -// needs the raw client rather than the caching wrapper. -func (c *CachingClient) Inner() client.Client { return c.Client } - // gvkFor resolves the GVK of obj and reports whether it is cached. func (c *CachingClient) gvkFor(obj runtime.Object) (schema.GroupVersionKind, bool) { gvks, _, err := c.scheme.ObjectKinds(obj) diff --git a/pkg/clientcache/client_test.go b/pkg/clientcache/client_test.go index 08d043f26..fd51a0ab5 100644 --- a/pkg/clientcache/client_test.go +++ b/pkg/clientcache/client_test.go @@ -146,15 +146,6 @@ func TestNewExplicitTTL(t *testing.T) { } } -// TestInnerReturnsWrappedClient: Inner returns the exact client passed to New. -func TestInnerReturnsWrappedClient(t *testing.T) { - inner := newTestClient(t) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) - if c.Inner() != inner { - t.Fatalf("Inner did not return the wrapped client") - } -} - // TestWriteErrorLeavesOverlayUntouched: for every mutating method, a failing // inner call surfaces the error and leaves the overlay untouched (no live entry // and no tombstone). From bd82974f677fa7ef34ba83b5318a3937d69d2018 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Thu, 6 Aug 2026 11:00:06 +0200 Subject: [PATCH 04/16] feedback: merge cache and client --- pkg/clientcache/cache.go | 262 ---------------- pkg/clientcache/cache_test.go | 436 --------------------------- pkg/clientcache/client.go | 269 ++++++++++++++++- pkg/clientcache/client_test.go | 527 ++++++++++++++++++++++++++++----- pkg/clientcache/runnable.go | 4 +- 5 files changed, 712 insertions(+), 786 deletions(-) delete mode 100644 pkg/clientcache/cache.go delete mode 100644 pkg/clientcache/cache_test.go diff --git a/pkg/clientcache/cache.go b/pkg/clientcache/cache.go deleted file mode 100644 index 639fd7663..000000000 --- a/pkg/clientcache/cache.go +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package clientcache - -import ( - "strconv" - "sync" - "time" - - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// entry is a single overlaid object with the bookkeeping needed for eviction. -type entry struct { - obj client.Object - uid types.UID - resourceVersion string - deleted bool // tombstone: object was deleted through the caching client - expiresAt time.Time -} - -// objectKey identifies an object within a GVK by namespace and name. -type objectKey struct { - namespace string - name string -} - -// overlay is the generic, client-independent core of the cache. It stores -// pending writes keyed by GVK and objectKey and merges them into informer -// read results until the real object is observed in an informer (eviction). -type overlay struct { - mu sync.RWMutex - byGVK map[schema.GroupVersionKind]map[objectKey]*entry - ttl time.Duration - // indexers holds the IndexerFunc per field per GVK, captured from - // IndexField calls, so overlay entries can be matched against FieldSelectors. - indexers map[schema.GroupVersionKind]map[string]client.IndexerFunc -} - -func newOverlay(ttl time.Duration) *overlay { - return &overlay{ - byGVK: make(map[schema.GroupVersionKind]map[objectKey]*entry), - ttl: ttl, - indexers: make(map[schema.GroupVersionKind]map[string]client.IndexerFunc), - } -} - -func keyForObject(obj client.Object) objectKey { - return objectKey{namespace: obj.GetNamespace(), name: obj.GetName()} -} - -// upsert stores a live (non-tombstone) entry for the object. -func (o *overlay) upsert(gvk schema.GroupVersionKind, obj client.Object) { - o.mu.Lock() - defer o.mu.Unlock() - o.ensureGVK(gvk) - o.byGVK[gvk][keyForObject(obj)] = &entry{ - obj: obj.DeepCopyObject().(client.Object), - uid: obj.GetUID(), - resourceVersion: obj.GetResourceVersion(), - deleted: false, - expiresAt: time.Now().Add(o.ttl), - } -} - -// remove stores a tombstone so the object is filtered out of reads until the -// deletion is observed in an informer. -func (o *overlay) remove(gvk schema.GroupVersionKind, obj client.Object) { - o.mu.Lock() - defer o.mu.Unlock() - o.ensureGVK(gvk) - o.byGVK[gvk][keyForObject(obj)] = &entry{ - obj: obj.DeepCopyObject().(client.Object), - uid: obj.GetUID(), - resourceVersion: obj.GetResourceVersion(), - deleted: true, - expiresAt: time.Now().Add(o.ttl), - } -} - -// evictIfSeen removes the overlay entry for obj if the informer-observed object -// matches by UID and its ResourceVersion is at least as new as the cached one. -func (o *overlay) evictIfSeen(gvk schema.GroupVersionKind, obj client.Object) { - o.mu.Lock() - defer o.mu.Unlock() - entries, ok := o.byGVK[gvk] - if !ok { - return - } - key := keyForObject(obj) - e, ok := entries[key] - if !ok { - return - } - // Only evict when the informer sees the same object generation (by UID) at - // a ResourceVersion >= the one we cached. Otherwise the informer might be - // showing an older revision than our pending write. - if e.uid != "" && obj.GetUID() != "" && e.uid != obj.GetUID() { - return - } - if !resourceVersionAtLeast(obj.GetResourceVersion(), e.resourceVersion) { - return - } - delete(entries, key) -} - -// get returns the overlay entry for the key, if present. -func (o *overlay) get(gvk schema.GroupVersionKind, key objectKey) (*entry, bool) { - o.mu.RLock() - defer o.mu.RUnlock() - entries, ok := o.byGVK[gvk] - if !ok { - return nil, false - } - e, ok := entries[key] - return e, ok -} - -// overlayList merges the overlay entries for the GVK into the informer result, -// deduplicating by objectKey (overlay wins), dropping tombstones, and filtering -// overlay-only entries against the list options' label and field selectors. -func (o *overlay) overlayList(gvk schema.GroupVersionKind, existing []runtime.Object, lo *client.ListOptions) []runtime.Object { - o.mu.RLock() - defer o.mu.RUnlock() - entries := o.byGVK[gvk] - if len(entries) == 0 { - return existing - } - - result := make([]runtime.Object, 0, len(existing)+len(entries)) - // Track which overlay keys are handled so overlay-only entries can be added. - handled := make(map[objectKey]bool, len(entries)) - - for _, item := range existing { - obj, ok := item.(client.Object) - if !ok { - result = append(result, item) - continue - } - key := keyForObject(obj) - e, present := entries[key] - if !present { - result = append(result, item) - continue - } - handled[key] = true - // Overlay wins over the informer result for the same key. - if e.deleted { - // Tombstone: drop the object entirely. - continue - } - result = append(result, e.obj.DeepCopyObject()) - } - - // Add overlay-only entries (not present in the informer result) that match - // the list options. - for key, e := range entries { - if handled[key] { - continue - } - if e.deleted { - continue - } - if !o.matchesLocked(gvk, e.obj, lo) { - continue - } - result = append(result, e.obj.DeepCopyObject()) - } - return result -} - -// matchesLocked reports whether obj satisfies the list options' namespace, -// label and field selectors. Callers must hold at least the read lock. -func (o *overlay) matchesLocked(gvk schema.GroupVersionKind, obj client.Object, lo *client.ListOptions) bool { - if lo == nil { - return true - } - if lo.Namespace != "" && obj.GetNamespace() != lo.Namespace { - return false - } - if lo.LabelSelector != nil && !lo.LabelSelector.Matches(labels.Set(obj.GetLabels())) { - return false - } - if lo.FieldSelector != nil && !lo.FieldSelector.Empty() { - set := o.fieldSetLocked(gvk, obj) - if !lo.FieldSelector.Matches(set) { - return false - } - } - return true -} - -// fieldSetLocked builds a fields.Set for obj using the registered IndexerFuncs -// for the GVK. Callers must hold at least the read lock. -func (o *overlay) fieldSetLocked(gvk schema.GroupVersionKind, obj client.Object) fields.Set { - set := fields.Set{} - for field, fn := range o.indexers[gvk] { - for _, v := range fn(obj) { - // A field selector matches a single value; take the first indexed - // value for the field (mirrors controller-runtime cache behaviour). - set[field] = v - break - } - } - return set -} - -// registerIndex captures an IndexerFunc for a field so overlay entries can be -// matched against MatchingFields queries. -func (o *overlay) registerIndex(gvk schema.GroupVersionKind, field string, fn client.IndexerFunc) { - o.mu.Lock() - defer o.mu.Unlock() - if o.indexers[gvk] == nil { - o.indexers[gvk] = make(map[string]client.IndexerFunc) - } - o.indexers[gvk][field] = fn -} - -// cleanupExpired removes entries whose TTL has passed. -func (o *overlay) cleanupExpired(now time.Time) { - o.mu.Lock() - defer o.mu.Unlock() - for _, entries := range o.byGVK { - for key, e := range entries { - if now.After(e.expiresAt) { - delete(entries, key) - } - } - } -} - -func (o *overlay) ensureGVK(gvk schema.GroupVersionKind) { - if o.byGVK[gvk] == nil { - o.byGVK[gvk] = make(map[objectKey]*entry) - } -} - -// resourceVersionAtLeast reports whether observed >= cached, treating -// ResourceVersions as opaque monotonically increasing integers (as the -// kubernetes apiserver guarantees per resource). Unparsable or empty values -// are treated conservatively: an empty cached RV means "evict on any sighting". -func resourceVersionAtLeast(observed, cached string) bool { - if cached == "" { - return true - } - if observed == "" { - return false - } - oi, oerr := strconv.ParseUint(observed, 10, 64) - ci, cerr := strconv.ParseUint(cached, 10, 64) - if oerr != nil || cerr != nil { - // Fall back to string comparison if not integers. - return observed >= cached - } - return oi >= ci -} diff --git a/pkg/clientcache/cache_test.go b/pkg/clientcache/cache_test.go deleted file mode 100644 index 5d3cceead..000000000 --- a/pkg/clientcache/cache_test.go +++ /dev/null @@ -1,436 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package clientcache - -import ( - "context" - "sync" - "testing" - "time" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - toolscachek8s "k8s.io/client-go/tools/cache" - ccache "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" -) - -const azIndexField = "spec.availabilityZone" - -func testScheme(t *testing.T) *runtime.Scheme { - t.Helper() - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("add to scheme: %v", err) - } - return scheme -} - -func newReservation(name, az, rv string) *v1alpha1.Reservation { - r := &v1alpha1.Reservation{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID("uid-" + name), - ResourceVersion: rv, - }, - Spec: v1alpha1.ReservationSpec{ - Type: v1alpha1.ReservationTypeCommittedResource, - AvailabilityZone: az, - }, - } - return r -} - -func newTestClient(t *testing.T, objs ...client.Object) client.Client { - t.Helper() - return fake.NewClientBuilder(). - WithScheme(testScheme(t)). - WithObjects(objs...). - WithStatusSubresource(&v1alpha1.Reservation{}). - WithIndex(&v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { - res, ok := obj.(*v1alpha1.Reservation) - if !ok || res.Spec.AvailabilityZone == "" { - return nil - } - return []string{res.Spec.AvailabilityZone} - }). - Build() -} - -// fakeInformer is a controllable informer that records handlers and lets tests -// fire Add/Update events to trigger eviction. -type fakeInformer struct { - ccache.Informer - mu sync.Mutex - handlers []toolscachek8s.ResourceEventHandler -} - -func (f *fakeInformer) AddEventHandler(h toolscachek8s.ResourceEventHandler) (toolscachek8s.ResourceEventHandlerRegistration, error) { - f.mu.Lock() - defer f.mu.Unlock() - f.handlers = append(f.handlers, h) - return nil, nil -} - -func (f *fakeInformer) fireAdd(obj any) { - f.mu.Lock() - handlers := append([]toolscachek8s.ResourceEventHandler(nil), f.handlers...) - f.mu.Unlock() - for _, h := range handlers { - h.OnAdd(obj, false) - } -} - -func (f *fakeInformer) fireUpdate(oldObj, newObj any) { - f.mu.Lock() - handlers := append([]toolscachek8s.ResourceEventHandler(nil), f.handlers...) - f.mu.Unlock() - for _, h := range handlers { - h.OnUpdate(oldObj, newObj) - } -} - -// fakeInformerSource returns a single shared fakeInformer for all kinds. -type fakeInformerSource struct { - inf *fakeInformer -} - -func (s *fakeInformerSource) GetInformersForKind(ctx context.Context, obj client.Object) ([]ccache.Informer, error) { - return []ccache.Informer{s.inf}, nil -} - -func reservationConfig() Config { - return Config{ - GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, - TTL: metav1.Duration{Duration: 2 * time.Minute}, - } -} - -func newCaching(t *testing.T, inner client.Client, src InformerSource) *CachingClient { - t.Helper() - c, err := New(inner, src, testScheme(t), reservationConfig()) - if err != nil { - t.Fatalf("New: %v", err) - } - return c -} - -func listReservations(t *testing.T, c client.Client, opts ...client.ListOption) []v1alpha1.Reservation { - t.Helper() - var list v1alpha1.ReservationList - if err := c.List(context.Background(), &list, opts...); err != nil { - t.Fatalf("List: %v", err) - } - return list.Items -} - -// 1. Write/Read: Create then immediate List/Get shows the object despite an -// empty informer (fake inner client without the object pre-loaded... but the -// fake client persists creates, so we simulate informer lag by deleting from -// inner after caching — instead we verify overlay independently below). -func TestCreateThenGetVisible(t *testing.T) { - inner := newTestClient(t) - src := &fakeInformerSource{inf: &fakeInformer{}} - c := newCaching(t, inner, src) - - r := newReservation("res-1", "az-1", "") - if err := c.Create(context.Background(), r); err != nil { - t.Fatalf("Create: %v", err) - } - - var got v1alpha1.Reservation - if err := c.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err != nil { - t.Fatalf("Get after create: %v", err) - } - if got.Spec.AvailabilityZone != "az-1" { - t.Fatalf("expected az-1, got %q", got.Spec.AvailabilityZone) - } -} - -// 2. Overlay with empty informer: inner List returns [], overlay entry still in -// the result. -func TestOverlayWhenInnerEmpty(t *testing.T) { - inner := newTestClient(t) // no objects - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) - - // Directly seed the overlay to simulate a write whose object is not yet in - // the (empty) inner client. - c.overlay.upsert(reservationGVK(), newReservation("res-2", "az-1", "5")) - - items := listReservations(t, c) - if len(items) != 1 || items[0].Name != "res-2" { - t.Fatalf("expected overlay entry res-2, got %+v", items) - } -} - -// 3. Eviction: an informer sighting (Add or Update) evicts the overlay entry -// only when it matches by UID and carries a ResourceVersion >= the cached one. -func TestEviction(t *testing.T) { - // Cached entry is always uid-res-3 @ RV 10. - const cachedRV = "10" - cases := []struct { - name string - useUpdate bool // fire OnUpdate instead of OnAdd - observedUID string // "" => reuse the cached object's UID - observedRV string - wantEvicted bool - }{ - {name: "add older RV keeps", observedRV: "9", wantEvicted: false}, - {name: "add equal RV evicts", observedRV: "10", wantEvicted: true}, - {name: "add newer RV evicts", observedRV: "11", wantEvicted: true}, - {name: "update newer RV evicts", useUpdate: true, observedRV: "11", wantEvicted: true}, - {name: "update older RV keeps", useUpdate: true, observedRV: "9", wantEvicted: false}, - {name: "uid mismatch keeps", observedUID: "uid-other", observedRV: "11", wantEvicted: false}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - inner := newTestClient(t) - inf := &fakeInformer{} - c := newCaching(t, inner, &fakeInformerSource{inf: inf}) - - ctx := t.Context() - go func() { - if err := c.Start(ctx); err != nil && ctx.Err() == nil { - t.Errorf("c.Start: %v", err) - } - }() - waitFor(t, func() bool { - inf.mu.Lock() - defer inf.mu.Unlock() - return len(inf.handlers) > 0 - }) - - c.overlay.upsert(reservationGVK(), newReservation("res-3", "az-1", cachedRV)) - - observed := newReservation("res-3", "az-1", tc.observedRV) - if tc.observedUID != "" { - observed.UID = types.UID(tc.observedUID) - } - if tc.useUpdate { - inf.fireUpdate(nil, observed) - } else { - inf.fireAdd(observed) - } - - _, present := c.overlay.get(reservationGVK(), objectKey{name: "res-3"}) - if present == tc.wantEvicted { - t.Fatalf("evicted=%v, want evicted=%v", !present, tc.wantEvicted) - } - }) - } -} - -// TestEvictionIgnoresNonObject: an informer event carrying a non-client.Object -// payload is ignored and does not panic or evict. -func TestEvictionIgnoresNonObject(t *testing.T) { - inner := newTestClient(t) - inf := &fakeInformer{} - c := newCaching(t, inner, &fakeInformerSource{inf: inf}) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go func() { - if err := c.Start(ctx); err != nil && ctx.Err() == nil { - t.Errorf("c.Start: %v", err) - } - }() - waitFor(t, func() bool { - inf.mu.Lock() - defer inf.mu.Unlock() - return len(inf.handlers) > 0 - }) - - c.overlay.upsert(reservationGVK(), newReservation("res-x", "az-1", "1")) - inf.fireAdd("not-an-object") - if _, ok := c.overlay.get(reservationGVK(), objectKey{name: "res-x"}); !ok { - t.Fatalf("non-object event must not evict the entry") - } -} - -// 4. TTL: cleanupExpired removes expired entries. -func TestTTLCleanup(t *testing.T) { - o := newOverlay(time.Minute) - o.upsert(reservationGVK(), newReservation("res-4", "az-1", "1")) - // Force expiry. - o.mu.Lock() - for _, entries := range o.byGVK { - for _, e := range entries { - e.expiresAt = time.Now().Add(-time.Second) - } - } - o.mu.Unlock() - o.cleanupExpired(time.Now()) - if _, ok := o.get(reservationGVK(), objectKey{name: "res-4"}); ok { - t.Fatalf("expired entry should be removed") - } -} - -// 5. Tombstone: Delete filters the object from List/Get even though inner still -// has it. -func TestTombstone(t *testing.T) { - r := newReservation("res-5", "az-1", "") - inner := newTestClient(t, r) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) - - if err := c.Delete(context.Background(), r); err != nil { - t.Fatalf("Delete: %v", err) - } - // Re-add to inner to simulate informer lag: fake client already removed it, - // so re-create via inner directly (bypassing overlay). - if err := inner.Create(context.Background(), newReservation("res-5", "az-1", "")); err != nil { - t.Fatalf("re-create inner: %v", err) - } - - var got v1alpha1.Reservation - err := c.Get(context.Background(), types.NamespacedName{Name: "res-5"}, &got) - if !apierrors.IsNotFound(err) { - t.Fatalf("expected NotFound for tombstoned object, got %v", err) - } - items := listReservations(t, c) - if len(items) != 0 { - t.Fatalf("expected tombstone to filter from list, got %+v", items) - } -} - -// 6. Update/Patch: newer overlay version overrides stale inner read. -func TestUpdateOverridesInner(t *testing.T) { - r := newReservation("res-6", "az-old", "1") - inner := newTestClient(t, r) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) - - // Read current to obtain the up-to-date ResourceVersion for update. - var cur v1alpha1.Reservation - if err := inner.Get(context.Background(), types.NamespacedName{Name: "res-6"}, &cur); err != nil { - t.Fatalf("inner get: %v", err) - } - cur.Spec.AvailabilityZone = "az-new" - if err := c.Update(context.Background(), &cur); err != nil { - t.Fatalf("Update: %v", err) - } - - var got v1alpha1.Reservation - if err := c.Get(context.Background(), types.NamespacedName{Name: "res-6"}, &got); err != nil { - t.Fatalf("Get: %v", err) - } - if got.Spec.AvailabilityZone != "az-new" { - t.Fatalf("expected az-new from overlay, got %q", got.Spec.AvailabilityZone) - } -} - -// 7. Label-matching: overlay-only entry appears only for matching MatchingLabels. -func TestLabelMatching(t *testing.T) { - inner := newTestClient(t) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) - - r := newReservation("res-7", "az-1", "1") - r.Labels = map[string]string{"team": "a"} - c.overlay.upsert(reservationGVK(), r) - - match := listReservations(t, c, client.MatchingLabels{"team": "a"}) - if len(match) != 1 { - t.Fatalf("expected match for team=a, got %+v", match) - } - noMatch := listReservations(t, c, client.MatchingLabels{"team": "b"}) - if len(noMatch) != 0 { - t.Fatalf("expected no match for team=b, got %+v", noMatch) - } -} - -// 8. Field-matching: after IndexField registration, overlay-only entry appears -// only for matching MatchingFields. -func TestFieldMatching(t *testing.T) { - inner := newTestClient(t) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) - - if err := c.IndexField(context.Background(), &v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { - res := obj.(*v1alpha1.Reservation) - if res.Spec.AvailabilityZone == "" { - return nil - } - return []string{res.Spec.AvailabilityZone} - }); err != nil { - t.Fatalf("IndexField: %v", err) - } - - c.overlay.upsert(reservationGVK(), newReservation("res-8", "az-1", "1")) - - match := listReservations(t, c, client.MatchingFields{azIndexField: "az-1"}) - if len(match) != 1 { - t.Fatalf("expected field match az-1, got %+v", match) - } - noMatch := listReservations(t, c, client.MatchingFields{azIndexField: "az-2"}) - if len(noMatch) != 0 { - t.Fatalf("expected no field match az-2, got %+v", noMatch) - } -} - -// 9. Non-cached GVK: calls pass through unchanged (no overlay effect). -func TestNonCachedGVKPassthrough(t *testing.T) { - inner := newTestClient(t) - // Config with no GVKs → Reservation is not cached. - c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{}) - if err != nil { - t.Fatalf("New: %v", err) - } - r := newReservation("res-9", "az-1", "") - if err := c.Create(context.Background(), r); err != nil { - t.Fatalf("Create: %v", err) - } - // Overlay must be empty for non-cached GVK. - if _, ok := c.overlay.get(reservationGVK(), objectKey{name: "res-9"}); ok { - t.Fatalf("non-cached GVK should not populate overlay") - } - // Delete it in inner, then Get should be NotFound (no overlay resurrection). - if err := c.Delete(context.Background(), r); err != nil { - t.Fatalf("Delete: %v", err) - } - var got v1alpha1.Reservation - if err := c.Get(context.Background(), types.NamespacedName{Name: "res-9"}, &got); !apierrors.IsNotFound(err) { - t.Fatalf("expected NotFound, got %v", err) - } -} - -// 10. Dedup: object present in both informer (inner) and overlay appears once. -func TestDedup(t *testing.T) { - r := newReservation("res-10", "az-1", "1") - inner := newTestClient(t, r) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) - - // Overlay holds a newer version of the same object. - newer := newReservation("res-10", "az-1", "2") - newer.Spec.TargetHost = "host-x" - c.overlay.upsert(reservationGVK(), newer) - - items := listReservations(t, c) - if len(items) != 1 { - t.Fatalf("expected exactly one item after dedup, got %d: %+v", len(items), items) - } - if items[0].Spec.TargetHost != "host-x" { - t.Fatalf("expected overlay version to win, got %+v", items[0]) - } -} - -// helpers - -func reservationGVK() schema.GroupVersionKind { - return v1alpha1.GroupVersion.WithKind("Reservation") -} - -func waitFor(t *testing.T, cond func() bool) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for time.Now().Before(deadline) { - if cond() { - return - } - time.Sleep(5 * time.Millisecond) - } - t.Fatalf("condition not met within timeout") -} diff --git a/pkg/clientcache/client.go b/pkg/clientcache/client.go index d4560a936..f89fddb62 100644 --- a/pkg/clientcache/client.go +++ b/pkg/clientcache/client.go @@ -6,15 +6,59 @@ package clientcache import ( "context" "errors" + "strconv" + "sync" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) +// entry is a single overlaid object with the bookkeeping needed for eviction. +type entry struct { + obj client.Object + uid types.UID + resourceVersion string + deleted bool // tombstone: object was deleted through the caching client + expiresAt time.Time +} + +// objectKey identifies an object within a GVK by namespace and name. +type objectKey struct { + namespace string + name string +} + +func keyForObject(obj client.Object) objectKey { + return objectKey{namespace: obj.GetNamespace(), name: obj.GetName()} +} + +// resourceVersionAtLeast reports whether observed >= cached, treating +// ResourceVersions as opaque monotonically increasing integers (as the +// kubernetes apiserver guarantees per resource). Unparsable or empty values +// are treated conservatively: an empty cached RV means "evict on any sighting". +func resourceVersionAtLeast(observed, cached string) bool { + if cached == "" { + return true + } + if observed == "" { + return false + } + oi, oerr := strconv.ParseUint(observed, 10, 64) + ci, cerr := strconv.ParseUint(cached, 10, 64) + if oerr != nil || cerr != nil { + // Fall back to string comparison if not integers. + return observed >= cached + } + return oi >= ci +} + // defaultTTL is used when Config.TTL is zero. const defaultTTL = 2 * time.Minute @@ -25,14 +69,23 @@ const defaultTTL = 2 * time.Minute // // It embeds client.Client so all methods not overridden below are delegated to // the inner client unchanged. +// +// The local overlay maps each cached GVK to a set of entries keyed by +// namespace/name. An entry is either live (a pending write not yet visible in +// the informer) or a tombstone (a Delete that has not yet propagated). Reads +// merge the inner result with these entries: tombstones suppress objects; +// live entries override or supplement the inner result. type CachingClient struct { client.Client // inner client, used for delegation informers InformerSource scheme *runtime.Scheme - overlay *overlay ttl time.Duration gvks map[schema.GroupVersionKind]bool + + mu sync.RWMutex + byGVK map[schema.GroupVersionKind]map[objectKey]*entry + indexers map[schema.GroupVersionKind]map[string]client.IndexerFunc } // New builds a CachingClient wrapping inner. informers supplies the informers @@ -52,9 +105,10 @@ func New(inner client.Client, informers InformerSource, scheme *runtime.Scheme, Client: inner, informers: informers, scheme: scheme, - overlay: newOverlay(ttl), ttl: ttl, gvks: gvks, + byGVK: make(map[schema.GroupVersionKind]map[objectKey]*entry), + indexers: make(map[schema.GroupVersionKind]map[string]client.IndexerFunc), }, nil } @@ -110,6 +164,193 @@ func trimListSuffix(kind string) (string, bool) { return kind, false } +// upsert stores a live (non-tombstone) entry for the object. +func (c *CachingClient) upsert(gvk schema.GroupVersionKind, obj client.Object) { + c.mu.Lock() + defer c.mu.Unlock() + c.ensureGVK(gvk) + c.byGVK[gvk][keyForObject(obj)] = &entry{ + obj: obj.DeepCopyObject().(client.Object), + uid: obj.GetUID(), + resourceVersion: obj.GetResourceVersion(), + deleted: false, + expiresAt: time.Now().Add(c.ttl), + } +} + +// tombstone marks the object as deleted in the overlay so it is filtered out +// of reads until the deletion is observed in an informer. +func (c *CachingClient) tombstone(gvk schema.GroupVersionKind, obj client.Object) { + c.mu.Lock() + defer c.mu.Unlock() + c.ensureGVK(gvk) + c.byGVK[gvk][keyForObject(obj)] = &entry{ + obj: obj.DeepCopyObject().(client.Object), + uid: obj.GetUID(), + resourceVersion: obj.GetResourceVersion(), + deleted: true, + expiresAt: time.Now().Add(c.ttl), + } +} + +// evictIfSeen removes the overlay entry for obj if the informer-observed object +// matches by UID and its ResourceVersion is at least as new as the cached one. +func (c *CachingClient) evictIfSeen(gvk schema.GroupVersionKind, obj client.Object) { + c.mu.Lock() + defer c.mu.Unlock() + entries, ok := c.byGVK[gvk] + if !ok { + return + } + key := keyForObject(obj) + e, ok := entries[key] + if !ok { + return + } + // Only evict when the informer sees the same object generation (by UID) at + // a ResourceVersion >= the one we cached. Otherwise the informer might be + // showing an older revision than our pending write. + if e.uid != "" && obj.GetUID() != "" && e.uid != obj.GetUID() { + return + } + if !resourceVersionAtLeast(obj.GetResourceVersion(), e.resourceVersion) { + return + } + delete(entries, key) +} + +// getEntry returns the overlay entry for the key, if present. +func (c *CachingClient) getEntry(gvk schema.GroupVersionKind, key objectKey) (*entry, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + entries, ok := c.byGVK[gvk] + if !ok { + return nil, false + } + e, ok := entries[key] + return e, ok +} + +// cleanupExpired removes entries whose TTL has passed. +func (c *CachingClient) cleanupExpired(now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + for _, entries := range c.byGVK { + for key, e := range entries { + if now.After(e.expiresAt) { + delete(entries, key) + } + } + } +} + +// registerIndex captures an IndexerFunc for a field so overlay entries can be +// matched against MatchingFields queries. +func (c *CachingClient) registerIndex(gvk schema.GroupVersionKind, field string, fn client.IndexerFunc) { + c.mu.Lock() + defer c.mu.Unlock() + if c.indexers[gvk] == nil { + c.indexers[gvk] = make(map[string]client.IndexerFunc) + } + c.indexers[gvk][field] = fn +} + +// ensureGVK initialises the per-GVK entry map if absent. Callers must hold the write lock. +func (c *CachingClient) ensureGVK(gvk schema.GroupVersionKind) { + if c.byGVK[gvk] == nil { + c.byGVK[gvk] = make(map[objectKey]*entry) + } +} + +// overlayList merges the overlay entries for the GVK into the informer result, +// deduplicating by objectKey (overlay wins), dropping tombstones, and filtering +// overlay-only entries against the list options' label and field selectors. +func (c *CachingClient) overlayList(gvk schema.GroupVersionKind, existing []runtime.Object, lo *client.ListOptions) []runtime.Object { + c.mu.RLock() + defer c.mu.RUnlock() + entries := c.byGVK[gvk] + if len(entries) == 0 { + return existing + } + + result := make([]runtime.Object, 0, len(existing)+len(entries)) + // Track which overlay keys are handled so overlay-only entries can be added. + handled := make(map[objectKey]bool, len(entries)) + + for _, item := range existing { + obj, ok := item.(client.Object) + if !ok { + result = append(result, item) + continue + } + key := keyForObject(obj) + e, present := entries[key] + if !present { + result = append(result, item) + continue + } + handled[key] = true + // Overlay wins over the informer result for the same key. + if e.deleted { + // Tombstone: drop the object entirely. + continue + } + result = append(result, e.obj.DeepCopyObject()) + } + + // Add overlay-only entries (not present in the informer result) that match + // the list options. + for key, e := range entries { + if handled[key] { + continue + } + if e.deleted { + continue + } + if !c.matchesLocked(gvk, e.obj, lo) { + continue + } + result = append(result, e.obj.DeepCopyObject()) + } + return result +} + +// matchesLocked reports whether obj satisfies the list options' namespace, +// label and field selectors. Callers must hold at least the read lock. +func (c *CachingClient) matchesLocked(gvk schema.GroupVersionKind, obj client.Object, lo *client.ListOptions) bool { + if lo == nil { + return true + } + if lo.Namespace != "" && obj.GetNamespace() != lo.Namespace { + return false + } + if lo.LabelSelector != nil && !lo.LabelSelector.Matches(labels.Set(obj.GetLabels())) { + return false + } + if lo.FieldSelector != nil && !lo.FieldSelector.Empty() { + set := c.fieldSetLocked(gvk, obj) + if !lo.FieldSelector.Matches(set) { + return false + } + } + return true +} + +// fieldSetLocked builds a fields.Set for obj using the registered IndexerFuncs +// for the GVK. Callers must hold at least the read lock. +func (c *CachingClient) fieldSetLocked(gvk schema.GroupVersionKind, obj client.Object) fields.Set { + set := fields.Set{} + for field, fn := range c.indexers[gvk] { + for _, v := range fn(obj) { + // A field selector matches a single value; take the first indexed + // value for the field (mirrors controller-runtime cache behaviour). + set[field] = v + break + } + } + return set +} + // Create delegates to the inner client and, on success for a cached GVK, adds // the object to the overlay. func (c *CachingClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { @@ -117,7 +358,7 @@ func (c *CachingClient) Create(ctx context.Context, obj client.Object, opts ...c return err } if gvk, cached := c.gvkFor(obj); cached { - c.overlay.upsert(gvk, obj) + c.upsert(gvk, obj) } return nil } @@ -129,7 +370,7 @@ func (c *CachingClient) Update(ctx context.Context, obj client.Object, opts ...c return err } if gvk, cached := c.gvkFor(obj); cached { - c.overlay.upsert(gvk, obj) + c.upsert(gvk, obj) } return nil } @@ -141,19 +382,23 @@ func (c *CachingClient) Patch(ctx context.Context, obj client.Object, patch clie return err } if gvk, cached := c.gvkFor(obj); cached { - c.overlay.upsert(gvk, obj) + c.upsert(gvk, obj) } return nil } // Delete delegates to the inner client and, on success for a cached GVK, stores -// a tombstone in the overlay. +// a tombstone in the overlay. The object is NOT immediately removed from the +// local map; it stays as a deleted=true entry until the deletion propagates +// through the informer (which triggers eviction) or the TTL expires. This +// ensures that reads between the Delete call and the informer event correctly +// return NotFound rather than serving a stale object from the informer cache. func (c *CachingClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { if err := c.Client.Delete(ctx, obj, opts...); err != nil { return err } if gvk, cached := c.gvkFor(obj); cached { - c.overlay.remove(gvk, obj) + c.tombstone(gvk, obj) } return nil } @@ -170,7 +415,7 @@ func (c *CachingClient) Get(ctx context.Context, key client.ObjectKey, obj clien if err != nil && !apierrors.IsNotFound(err) { return err } - e, ok := c.overlay.get(gvk, objectKey{namespace: key.Namespace, name: key.Name}) + e, ok := c.getEntry(gvk, objectKey{namespace: key.Namespace, name: key.Name}) if !ok { // No overlay entry: return the inner result (value or NotFound) as-is. return err @@ -200,7 +445,7 @@ func (c *CachingClient) List(ctx context.Context, list client.ObjectList, opts . } lo := &client.ListOptions{} lo.ApplyOptions(opts) - merged := c.overlay.overlayList(itemGVK, items, lo) + merged := c.overlayList(itemGVK, items, lo) return meta.SetList(list, merged) } @@ -214,7 +459,7 @@ func (c *CachingClient) IndexField(ctx context.Context, obj client.Object, field } } if gvk, cached := c.gvkFor(obj); cached { - c.overlay.registerIndex(gvk, field, extractValue) + c.registerIndex(gvk, field, extractValue) } return nil } @@ -241,7 +486,7 @@ func (s *statusWriter) Update(ctx context.Context, obj client.Object, opts ...cl return err } if gvk, cached := s.c.gvkFor(obj); cached { - s.c.overlay.upsert(gvk, obj) + s.c.upsert(gvk, obj) } return nil } @@ -251,7 +496,7 @@ func (s *statusWriter) Patch(ctx context.Context, obj client.Object, patch clien return err } if gvk, cached := s.c.gvkFor(obj); cached { - s.c.overlay.upsert(gvk, obj) + s.c.upsert(gvk, obj) } return nil } diff --git a/pkg/clientcache/client_test.go b/pkg/clientcache/client_test.go index fd51a0ab5..8ad0452de 100644 --- a/pkg/clientcache/client_test.go +++ b/pkg/clientcache/client_test.go @@ -6,18 +6,149 @@ package clientcache import ( "context" "errors" + "sync" "testing" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + toolscachek8s "k8s.io/client-go/tools/cache" + ccache "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) +const azIndexField = "spec.availabilityZone" + +// --- shared test infrastructure --- + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add to scheme: %v", err) + } + return scheme +} + +func newReservation(name, az, rv string) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID("uid-" + name), + ResourceVersion: rv, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + AvailabilityZone: az, + }, + } +} + +func newTestClient(t *testing.T, objs ...client.Object) client.Client { + t.Helper() + return fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(objs...). + WithStatusSubresource(&v1alpha1.Reservation{}). + WithIndex(&v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok || res.Spec.AvailabilityZone == "" { + return nil + } + return []string{res.Spec.AvailabilityZone} + }). + Build() +} + +// fakeInformer is a controllable informer that records handlers and lets tests +// fire Add/Update events to trigger eviction. +type fakeInformer struct { + ccache.Informer + mu sync.Mutex + handlers []toolscachek8s.ResourceEventHandler +} + +func (f *fakeInformer) AddEventHandler(h toolscachek8s.ResourceEventHandler) (toolscachek8s.ResourceEventHandlerRegistration, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.handlers = append(f.handlers, h) + return nil, nil +} + +func (f *fakeInformer) fireAdd(obj any) { + f.mu.Lock() + handlers := append([]toolscachek8s.ResourceEventHandler(nil), f.handlers...) + f.mu.Unlock() + for _, h := range handlers { + h.OnAdd(obj, false) + } +} + +func (f *fakeInformer) fireUpdate(oldObj, newObj any) { + f.mu.Lock() + handlers := append([]toolscachek8s.ResourceEventHandler(nil), f.handlers...) + f.mu.Unlock() + for _, h := range handlers { + h.OnUpdate(oldObj, newObj) + } +} + +// fakeInformerSource returns a single shared fakeInformer for all kinds. +type fakeInformerSource struct { + inf *fakeInformer +} + +func (s *fakeInformerSource) GetInformersForKind(_ context.Context, _ client.Object) ([]ccache.Informer, error) { + return []ccache.Informer{s.inf}, nil +} + +func reservationConfig() Config { + return Config{ + GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, + TTL: metav1.Duration{Duration: 2 * time.Minute}, + } +} + +func newCaching(t *testing.T, inner client.Client, src InformerSource) *CachingClient { + t.Helper() + c, err := New(inner, src, testScheme(t), reservationConfig()) + if err != nil { + t.Fatalf("New: %v", err) + } + return c +} + +func listReservations(t *testing.T, c client.Client, opts ...client.ListOption) []v1alpha1.Reservation { + t.Helper() + var list v1alpha1.ReservationList + if err := c.List(context.Background(), &list, opts...); err != nil { + t.Fatalf("List: %v", err) + } + return list.Items +} + +func reservationGVK() schema.GroupVersionKind { + return v1alpha1.GroupVersion.WithKind("Reservation") +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("condition not met within timeout") +} + // errClient wraps an inner client.Client and injects a configurable error into // each mutating/read operation, so the error-propagation paths of // CachingClient (which must not touch the overlay on failure) can be exercised. @@ -74,8 +205,8 @@ func (e *errClient) List(ctx context.Context, list client.ObjectList, opts ...cl } // forceInnerAZ writes a divergent AvailabilityZone directly to the inner client, -// bypassing the caching wrapper (and its overlay). Used to prove that reads -// through the caching client are served from the overlay, not the inner client. +// bypassing the caching wrapper. Used to prove that reads through the caching +// client are served from the overlay, not the inner client. func forceInnerAZ(t *testing.T, inner client.Client, name, az string) { t.Helper() var cur v1alpha1.Reservation @@ -88,8 +219,8 @@ func forceInnerAZ(t *testing.T, inner client.Client, name, az string) { } } -// forceInnerStatusHost writes a divergent status Host directly to the inner -// client, bypassing the caching wrapper. +// forceInnerStatusHost writes a divergent status Host directly to the inner client, +// bypassing the caching wrapper. func forceInnerStatusHost(t *testing.T, inner client.Client, name, host string) { t.Helper() var cur v1alpha1.Reservation @@ -102,6 +233,17 @@ func forceInnerStatusHost(t *testing.T, inner client.Client, name, host string) } } +// unknownObject is a client.Object whose type is not registered in the test +// scheme, used to exercise the "unresolvable GVK" branches. +type unknownObject struct { + metav1.TypeMeta + metav1.ObjectMeta +} + +func (u *unknownObject) DeepCopyObject() runtime.Object { return u } + +// --- constructor tests --- + // TestNewUnknownGVKError: New fails when a configured GVK string is not // registered in the scheme. func TestNewUnknownGVKError(t *testing.T) { @@ -126,9 +268,6 @@ func TestNewDefaultTTL(t *testing.T) { if c.ttl != defaultTTL { t.Fatalf("expected ttl %v, got %v", defaultTTL, c.ttl) } - if c.overlay.ttl != defaultTTL { - t.Fatalf("expected overlay ttl %v, got %v", defaultTTL, c.overlay.ttl) - } } // TestNewExplicitTTL: a non-zero TTL is honoured verbatim. @@ -146,17 +285,284 @@ func TestNewExplicitTTL(t *testing.T) { } } -// TestWriteErrorLeavesOverlayUntouched: for every mutating method, a failing -// inner call surfaces the error and leaves the overlay untouched (no live entry -// and no tombstone). +// --- overlay behaviour tests --- + +// TestCreateThenGetVisible: Create then immediate Get shows the object via the +// overlay even before the informer has caught up. +func TestCreateThenGetVisible(t *testing.T) { + inner := newTestClient(t) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + r := newReservation("res-1", "az-1", "") + if err := c.Create(context.Background(), r); err != nil { + t.Fatalf("Create: %v", err) + } + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err != nil { + t.Fatalf("Get after create: %v", err) + } + if got.Spec.AvailabilityZone != "az-1" { + t.Fatalf("expected az-1, got %q", got.Spec.AvailabilityZone) + } +} + +// TestOverlayWhenInnerEmpty: a seeded overlay entry appears in List even when +// the inner client returns nothing. +func TestOverlayWhenInnerEmpty(t *testing.T) { + inner := newTestClient(t) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + c.upsert(reservationGVK(), newReservation("res-2", "az-1", "5")) + + items := listReservations(t, c) + if len(items) != 1 || items[0].Name != "res-2" { + t.Fatalf("expected overlay entry res-2, got %+v", items) + } +} + +// TestEviction: an informer sighting evicts the overlay entry only when it +// matches by UID and carries a ResourceVersion >= the cached one. +func TestEviction(t *testing.T) { + const cachedRV = "10" + cases := []struct { + name string + useUpdate bool + observedUID string + observedRV string + wantEvicted bool + }{ + {name: "add older RV keeps", observedRV: "9", wantEvicted: false}, + {name: "add equal RV evicts", observedRV: "10", wantEvicted: true}, + {name: "add newer RV evicts", observedRV: "11", wantEvicted: true}, + {name: "update newer RV evicts", useUpdate: true, observedRV: "11", wantEvicted: true}, + {name: "update older RV keeps", useUpdate: true, observedRV: "9", wantEvicted: false}, + {name: "uid mismatch keeps", observedUID: "uid-other", observedRV: "11", wantEvicted: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + inf := &fakeInformer{} + c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: inf}) + + ctx := t.Context() + go func() { + if err := c.Start(ctx); err != nil && ctx.Err() == nil { + t.Errorf("c.Start: %v", err) + } + }() + waitFor(t, func() bool { + inf.mu.Lock() + defer inf.mu.Unlock() + return len(inf.handlers) > 0 + }) + + c.upsert(reservationGVK(), newReservation("res-3", "az-1", cachedRV)) + + observed := newReservation("res-3", "az-1", tc.observedRV) + if tc.observedUID != "" { + observed.UID = types.UID(tc.observedUID) + } + if tc.useUpdate { + inf.fireUpdate(nil, observed) + } else { + inf.fireAdd(observed) + } + + _, present := c.getEntry(reservationGVK(), objectKey{name: "res-3"}) + if present == tc.wantEvicted { + t.Fatalf("evicted=%v, want evicted=%v", !present, tc.wantEvicted) + } + }) + } +} + +// TestEvictionIgnoresNonObject: a non-client.Object informer payload does not +// panic or evict. +func TestEvictionIgnoresNonObject(t *testing.T) { + inf := &fakeInformer{} + c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: inf}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + if err := c.Start(ctx); err != nil && ctx.Err() == nil { + t.Errorf("c.Start: %v", err) + } + }() + waitFor(t, func() bool { + inf.mu.Lock() + defer inf.mu.Unlock() + return len(inf.handlers) > 0 + }) + + c.upsert(reservationGVK(), newReservation("res-x", "az-1", "1")) + inf.fireAdd("not-an-object") + if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-x"}); !ok { + t.Fatalf("non-object event must not evict the entry") + } +} + +// TestTTLCleanup: cleanupExpired removes entries whose TTL has passed. +func TestTTLCleanup(t *testing.T) { + c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) + c.upsert(reservationGVK(), newReservation("res-4", "az-1", "1")) + c.mu.Lock() + for _, entries := range c.byGVK { + for _, e := range entries { + e.expiresAt = time.Now().Add(-time.Second) + } + } + c.mu.Unlock() + c.cleanupExpired(time.Now()) + if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-4"}); ok { + t.Fatalf("expired entry should be removed") + } +} + +// TestTombstone: Delete stores a tombstone so subsequent Get/List return +// NotFound even while the inner client still has the object. +func TestTombstone(t *testing.T) { + r := newReservation("res-5", "az-1", "") + inner := newTestClient(t, r) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + if err := c.Delete(context.Background(), r); err != nil { + t.Fatalf("Delete: %v", err) + } + // Re-create directly in inner to simulate informer lag. + if err := inner.Create(context.Background(), newReservation("res-5", "az-1", "")); err != nil { + t.Fatalf("re-create inner: %v", err) + } + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-5"}, &got); !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound for tombstoned object, got %v", err) + } + if items := listReservations(t, c); len(items) != 0 { + t.Fatalf("expected tombstone to filter from list, got %+v", items) + } +} + +// TestUpdateOverridesInner: after Update the overlay version wins over a stale +// inner read. +func TestUpdateOverridesInner(t *testing.T) { + r := newReservation("res-6", "az-old", "1") + inner := newTestClient(t, r) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + var cur v1alpha1.Reservation + if err := inner.Get(context.Background(), types.NamespacedName{Name: "res-6"}, &cur); err != nil { + t.Fatalf("inner get: %v", err) + } + cur.Spec.AvailabilityZone = "az-new" + if err := c.Update(context.Background(), &cur); err != nil { + t.Fatalf("Update: %v", err) + } + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-6"}, &got); err != nil { + t.Fatalf("Get: %v", err) + } + if got.Spec.AvailabilityZone != "az-new" { + t.Fatalf("expected az-new from overlay, got %q", got.Spec.AvailabilityZone) + } +} + +// TestLabelMatching: an overlay-only entry appears only for matching labels. +func TestLabelMatching(t *testing.T) { + c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) + + r := newReservation("res-7", "az-1", "1") + r.Labels = map[string]string{"team": "a"} + c.upsert(reservationGVK(), r) + + if match := listReservations(t, c, client.MatchingLabels{"team": "a"}); len(match) != 1 { + t.Fatalf("expected match for team=a, got %+v", match) + } + if noMatch := listReservations(t, c, client.MatchingLabels{"team": "b"}); len(noMatch) != 0 { + t.Fatalf("expected no match for team=b, got %+v", noMatch) + } +} + +// TestFieldMatching: after IndexField registration, an overlay-only entry +// appears only for matching field selectors. +func TestFieldMatching(t *testing.T) { + c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) + + if err := c.IndexField(context.Background(), &v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { + res := obj.(*v1alpha1.Reservation) + if res.Spec.AvailabilityZone == "" { + return nil + } + return []string{res.Spec.AvailabilityZone} + }); err != nil { + t.Fatalf("IndexField: %v", err) + } + + c.upsert(reservationGVK(), newReservation("res-8", "az-1", "1")) + + if match := listReservations(t, c, client.MatchingFields{azIndexField: "az-1"}); len(match) != 1 { + t.Fatalf("expected field match az-1, got %+v", match) + } + if noMatch := listReservations(t, c, client.MatchingFields{azIndexField: "az-2"}); len(noMatch) != 0 { + t.Fatalf("expected no field match az-2, got %+v", noMatch) + } +} + +// TestNonCachedGVKPassthrough: calls for unconfigured GVKs pass through to the +// inner client with no overlay involvement. +func TestNonCachedGVKPassthrough(t *testing.T) { + inner := newTestClient(t) + c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{}) + if err != nil { + t.Fatalf("New: %v", err) + } + r := newReservation("res-9", "az-1", "") + if err := c.Create(context.Background(), r); err != nil { + t.Fatalf("Create: %v", err) + } + if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-9"}); ok { + t.Fatalf("non-cached GVK should not populate overlay") + } + if err := c.Delete(context.Background(), r); err != nil { + t.Fatalf("Delete: %v", err) + } + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-9"}, &got); !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound, got %v", err) + } +} + +// TestDedup: an object present in both the inner client and the overlay appears +// exactly once in List, with the overlay version winning. +func TestDedup(t *testing.T) { + r := newReservation("res-10", "az-1", "1") + inner := newTestClient(t, r) + c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + + newer := newReservation("res-10", "az-1", "2") + newer.Spec.TargetHost = "host-x" + c.upsert(reservationGVK(), newer) + + items := listReservations(t, c) + if len(items) != 1 { + t.Fatalf("expected exactly one item after dedup, got %d: %+v", len(items), items) + } + if items[0].Spec.TargetHost != "host-x" { + t.Fatalf("expected overlay version to win, got %+v", items[0]) + } +} + +// --- error-propagation tests --- + +// TestWriteErrorLeavesOverlayUntouched: a failing inner call leaves the overlay +// untouched (no live entry and no tombstone). func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { sentinel := errors.New("boom") cases := []struct { - name string - rv string // ResourceVersion for the object passed to the op - seed bool // pre-seed the object in the inner client (delete needs it) - // mkClient wraps an inner client (which may already contain r) with the - // relevant injected error. + name string + rv string + seed bool mkClient func(inner client.Client) client.Client op func(c *CachingClient, r *v1alpha1.Reservation) error }{ @@ -202,7 +608,7 @@ func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { if err := tc.op(c, r); !errors.Is(err, sentinel) { t.Fatalf("expected sentinel error, got %v", err) } - if _, ok := c.overlay.get(reservationGVK(), objectKey{name: r.Name}); ok { + if _, ok := c.getEntry(reservationGVK(), objectKey{name: r.Name}); ok { t.Fatalf("overlay must not be touched on %s failure", tc.name) } }) @@ -211,18 +617,13 @@ func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { // TestWriteServedFromOverlay: after a write through the caching client, a Get // returns the written value even though the inner client has been forced to a -// divergent (stale) value behind the cache's back. This proves the read path is -// actually served from the overlay, not merely that the overlay was written. +// divergent (stale) value behind the cache's back. func TestWriteServedFromOverlay(t *testing.T) { cases := []struct { - name string - // write performs the write under test through c, given the current - // object cur fetched from inner, and returns the value it wrote. - write func(t *testing.T, c *CachingClient, cur *v1alpha1.Reservation) string - // diverge forces the inner client to a stale value behind the cache. + name string + write func(t *testing.T, c *CachingClient, cur *v1alpha1.Reservation) string diverge func(t *testing.T, inner client.Client, name string) - // read extracts the field under test from a Get result. - read func(*v1alpha1.Reservation) string + read func(*v1alpha1.Reservation) string }{ { name: "patch spec", @@ -291,30 +692,24 @@ func TestWriteServedFromOverlay(t *testing.T) { } } -// TestGetPropagatesNonNotFoundError: a cached-GVK Get surfaces inner errors -// other than NotFound without consulting the overlay. +// TestGetPropagatesNonNotFoundError: a non-NotFound inner error is surfaced +// without consulting the overlay. func TestGetPropagatesNonNotFoundError(t *testing.T) { sentinel := errors.New("get boom") - inner := &errClient{Client: newTestClient(t), getErr: sentinel} - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) - - // Seed a live overlay entry that must NOT mask the underlying error. - c.overlay.upsert(reservationGVK(), newReservation("res-ge", "az-1", "1")) + c := newCaching(t, &errClient{Client: newTestClient(t), getErr: sentinel}, &fakeInformerSource{inf: &fakeInformer{}}) + c.upsert(reservationGVK(), newReservation("res-ge", "az-1", "1")) var got v1alpha1.Reservation - err := c.Get(context.Background(), types.NamespacedName{Name: "res-ge"}, &got) - if !errors.Is(err, sentinel) { + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-ge"}, &got); !errors.Is(err, sentinel) { t.Fatalf("expected sentinel error, got %v", err) } } // TestGetOverlayResurrectsNotFound: a live overlay entry satisfies a Get that -// the inner client reports as NotFound (write not yet visible in the informer). +// the inner client reports as NotFound. func TestGetOverlayResurrectsNotFound(t *testing.T) { - inner := newTestClient(t) // empty: inner Get returns NotFound - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) - - c.overlay.upsert(reservationGVK(), newReservation("res-gr", "az-z", "1")) + c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) + c.upsert(reservationGVK(), newReservation("res-gr", "az-z", "1")) var got v1alpha1.Reservation if err := c.Get(context.Background(), types.NamespacedName{Name: "res-gr"}, &got); err != nil { @@ -326,24 +721,21 @@ func TestGetOverlayResurrectsNotFound(t *testing.T) { } // TestGetNotFoundWithNoOverlay: inner NotFound with no overlay entry propagates -// NotFound unchanged for a cached GVK. +// NotFound unchanged. func TestGetNotFoundWithNoOverlay(t *testing.T) { - inner := newTestClient(t) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) var got v1alpha1.Reservation - err := c.Get(context.Background(), types.NamespacedName{Name: "missing"}, &got) - if !apierrors.IsNotFound(err) { + if err := c.Get(context.Background(), types.NamespacedName{Name: "missing"}, &got); !apierrors.IsNotFound(err) { t.Fatalf("expected NotFound, got %v", err) } } // TestGetNonCachedPropagatesError: for a non-cached GVK, Get is a pure -// passthrough and surfaces the inner error verbatim. +// passthrough. func TestGetNonCachedPropagatesError(t *testing.T) { sentinel := errors.New("get boom") - inner := &errClient{Client: newTestClient(t), getErr: sentinel} - c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{}) + c, err := New(&errClient{Client: newTestClient(t), getErr: sentinel}, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{}) if err != nil { t.Fatalf("New: %v", err) } @@ -353,13 +745,12 @@ func TestGetNonCachedPropagatesError(t *testing.T) { } } -// TestListPropagatesError: for a cached GVK, a failing inner List surfaces the -// error rather than returning a partial overlay merge. +// TestListPropagatesError: a failing inner List surfaces the error rather than +// returning a partial overlay merge. func TestListPropagatesError(t *testing.T) { sentinel := errors.New("list boom") - inner := &errClient{Client: newTestClient(t), listErr: sentinel} - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) - c.overlay.upsert(reservationGVK(), newReservation("res-le", "az-1", "1")) + c := newCaching(t, &errClient{Client: newTestClient(t), listErr: sentinel}, &fakeInformerSource{inf: &fakeInformer{}}) + c.upsert(reservationGVK(), newReservation("res-le", "az-1", "1")) var list v1alpha1.ReservationList if err := c.List(context.Background(), &list); !errors.Is(err, sentinel) { @@ -370,31 +761,27 @@ func TestListPropagatesError(t *testing.T) { // TestStatusUpdateErrorLeavesOverlayUntouched: a failed status update does not // populate the overlay. func TestStatusUpdateErrorLeavesOverlayUntouched(t *testing.T) { - inner := newTestClient(t) // object absent → status update fails - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) r := newReservation("res-se", "az-1", "1") if err := c.Status().Update(context.Background(), r); err == nil { t.Fatalf("expected status update to fail for missing object") } - if _, ok := c.overlay.get(reservationGVK(), objectKey{name: "res-se"}); ok { + if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-se"}); ok { t.Fatalf("overlay must not be populated on status update failure") } } // TestStatusCreateDelegates: Status().Create delegates to the inner status -// writer (fake client reports it unsupported) and never touches the overlay. +// writer and never touches the overlay. func TestStatusCreateDelegates(t *testing.T) { r := newReservation("res-sc", "az-1", "") - inner := newTestClient(t, r) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t, r), &fakeInformerSource{inf: &fakeInformer{}}) - // The fake client does not support subresource Create; we only assert the - // call is delegated (returns an error) and the overlay stays empty. if err := c.Status().Create(context.Background(), r, r); err == nil { t.Fatalf("expected Status().Create to fail on fake client") } - if _, ok := c.overlay.get(reservationGVK(), objectKey{name: "res-sc"}); ok { + if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-sc"}); ok { t.Fatalf("Status().Create must not populate the overlay") } } @@ -416,16 +803,17 @@ func TestStatusUpdateNonCachedNoOverlay(t *testing.T) { if err := c.Status().Update(context.Background(), &cur); err != nil { t.Fatalf("Status().Update: %v", err) } - if _, ok := c.overlay.get(reservationGVK(), objectKey{name: "res-sn"}); ok { + if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-sn"}); ok { t.Fatalf("non-cached GVK status update should not populate overlay") } } +// --- helper / utility tests --- + // TestGVKForUnknownType: gvkFor reports not-cached for a type not registered in // the scheme. func TestGVKForUnknownType(t *testing.T) { - inner := newTestClient(t) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) if _, cached := c.gvkFor(&unknownObject{}); cached { t.Fatalf("unknown type must not be reported as cached") } @@ -439,7 +827,7 @@ func TestTrimListSuffix(t *testing.T) { wantOK bool }{ {"ReservationList", "Reservation", true}, - {"List", "List", false}, // len(kind) not > len("List") + {"List", "List", false}, {"Reservation", "Reservation", false}, {"", "", false}, } @@ -450,12 +838,3 @@ func TestTrimListSuffix(t *testing.T) { } } } - -// unknownObject is a client.Object whose type is not registered in the test -// scheme, used to exercise the "unresolvable GVK" branches. -type unknownObject struct { - metav1.TypeMeta - metav1.ObjectMeta -} - -func (u *unknownObject) DeepCopyObject() runtime.Object { return u } diff --git a/pkg/clientcache/runnable.go b/pkg/clientcache/runnable.go index 836bee0e6..b64500df2 100644 --- a/pkg/clientcache/runnable.go +++ b/pkg/clientcache/runnable.go @@ -49,7 +49,7 @@ func (c *CachingClient) Start(ctx context.Context) error { case <-ctx.Done(): return nil case now := <-ticker.C: - c.overlay.cleanupExpired(now) + c.cleanupExpired(now) } } } @@ -62,7 +62,7 @@ func (c *CachingClient) evictionHandler(gvk schema.GroupVersionKind) toolscachek if !ok { return } - c.overlay.evictIfSeen(gvk, obj) + c.evictIfSeen(gvk, obj) } return toolscachek8s.ResourceEventHandlerFuncs{ AddFunc: func(o any) { evict(o) }, From a3d73a1630c9cb1ccdad7830189a71f9eec3c5fc Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Thu, 6 Aug 2026 11:30:01 +0200 Subject: [PATCH 05/16] refactor: update CachingClient to use inner client interface for informers --- cmd/manager/main.go | 6 +- pkg/clientcache/client.go | 24 +-- pkg/clientcache/client_test.go | 281 +++++++++++++-------------------- pkg/clientcache/interfaces.go | 10 +- pkg/clientcache/runnable.go | 2 +- 5 files changed, 134 insertions(+), 189 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index bc8c348d4..7e09976b2 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -400,10 +400,10 @@ func main() { // Transparent in-process overlay cache for CRDs that are eventually // consistent across in-pod clients (e.g. Reservations). Writes populate an // overlay; reads merge it with the informer result until the real object is - // observed. *multicluster.Client serves as both the inner client.Client and - // the InformerSource; the cache itself has no multicluster dependency. + // observed. *multicluster.Client satisfies clientcache.Client, providing + // both the inner client.Client and informer access for eviction. clientCacheConfig := conf.GetConfigOrDie[clientcache.RootConfig]() - cachingClient, err := clientcache.New(multiclusterClient, multiclusterClient, scheme, clientCacheConfig.ClientCache) + cachingClient, err := clientcache.New(multiclusterClient, scheme, clientCacheConfig.ClientCache) if err != nil { setupLog.Error(err, "unable to create client cache") os.Exit(1) diff --git a/pkg/clientcache/client.go b/pkg/clientcache/client.go index f89fddb62..0979a0020 100644 --- a/pkg/clientcache/client.go +++ b/pkg/clientcache/client.go @@ -78,10 +78,10 @@ const defaultTTL = 2 * time.Minute type CachingClient struct { client.Client // inner client, used for delegation - informers InformerSource - scheme *runtime.Scheme - ttl time.Duration - gvks map[schema.GroupVersionKind]bool + inner Client + scheme *runtime.Scheme + ttl time.Duration + gvks map[schema.GroupVersionKind]bool mu sync.RWMutex byGVK map[schema.GroupVersionKind]map[objectKey]*entry @@ -92,7 +92,7 @@ type CachingClient struct { // used for eviction, scheme resolves object GVKs, and conf lists the GVKs to // overlay and the TTL. GVK strings are formatted as "//" // and are resolved against scheme. -func New(inner client.Client, informers InformerSource, scheme *runtime.Scheme, conf Config) (*CachingClient, error) { +func New(inner Client, scheme *runtime.Scheme, conf Config) (*CachingClient, error) { gvks, err := resolveGVKs(scheme, conf.GVKs) if err != nil { return nil, err @@ -102,13 +102,13 @@ func New(inner client.Client, informers InformerSource, scheme *runtime.Scheme, ttl = defaultTTL } return &CachingClient{ - Client: inner, - informers: informers, - scheme: scheme, - ttl: ttl, - gvks: gvks, - byGVK: make(map[schema.GroupVersionKind]map[objectKey]*entry), - indexers: make(map[schema.GroupVersionKind]map[string]client.IndexerFunc), + Client: inner, + inner: inner, + scheme: scheme, + ttl: ttl, + gvks: gvks, + byGVK: make(map[schema.GroupVersionKind]map[objectKey]*entry), + indexers: make(map[schema.GroupVersionKind]map[string]client.IndexerFunc), }, nil } diff --git a/pkg/clientcache/client_test.go b/pkg/clientcache/client_test.go index 8ad0452de..8b99d067e 100644 --- a/pkg/clientcache/client_test.go +++ b/pkg/clientcache/client_test.go @@ -50,22 +50,6 @@ func newReservation(name, az, rv string) *v1alpha1.Reservation { } } -func newTestClient(t *testing.T, objs ...client.Object) client.Client { - t.Helper() - return fake.NewClientBuilder(). - WithScheme(testScheme(t)). - WithObjects(objs...). - WithStatusSubresource(&v1alpha1.Reservation{}). - WithIndex(&v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { - res, ok := obj.(*v1alpha1.Reservation) - if !ok || res.Spec.AvailabilityZone == "" { - return nil - } - return []string{res.Spec.AvailabilityZone} - }). - Build() -} - // fakeInformer is a controllable informer that records handlers and lets tests // fire Add/Update events to trigger eviction. type fakeInformer struct { @@ -99,61 +83,41 @@ func (f *fakeInformer) fireUpdate(oldObj, newObj any) { } } -// fakeInformerSource returns a single shared fakeInformer for all kinds. -type fakeInformerSource struct { +// fakeClient composes a fake client.Client with a fakeInformer to satisfy the +// clientcache.Client interface. +type fakeClient struct { + client.Client inf *fakeInformer } -func (s *fakeInformerSource) GetInformersForKind(_ context.Context, _ client.Object) ([]ccache.Informer, error) { - return []ccache.Informer{s.inf}, nil -} - -func reservationConfig() Config { - return Config{ - GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, - TTL: metav1.Duration{Duration: 2 * time.Minute}, - } -} - -func newCaching(t *testing.T, inner client.Client, src InformerSource) *CachingClient { - t.Helper() - c, err := New(inner, src, testScheme(t), reservationConfig()) - if err != nil { - t.Fatalf("New: %v", err) - } - return c +func (f *fakeClient) GetInformersForKind(_ context.Context, _ client.Object) ([]ccache.Informer, error) { + return []ccache.Informer{f.inf}, nil } -func listReservations(t *testing.T, c client.Client, opts ...client.ListOption) []v1alpha1.Reservation { +func newTestClient(t *testing.T, objs ...client.Object) *fakeClient { t.Helper() - var list v1alpha1.ReservationList - if err := c.List(context.Background(), &list, opts...); err != nil { - t.Fatalf("List: %v", err) - } - return list.Items -} - -func reservationGVK() schema.GroupVersionKind { - return v1alpha1.GroupVersion.WithKind("Reservation") -} - -func waitFor(t *testing.T, cond func() bool) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for time.Now().Before(deadline) { - if cond() { - return - } - time.Sleep(5 * time.Millisecond) + return &fakeClient{ + Client: fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(objs...). + WithStatusSubresource(&v1alpha1.Reservation{}). + WithIndex(&v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok || res.Spec.AvailabilityZone == "" { + return nil + } + return []string{res.Spec.AvailabilityZone} + }). + Build(), + inf: &fakeInformer{}, } - t.Fatalf("condition not met within timeout") } -// errClient wraps an inner client.Client and injects a configurable error into -// each mutating/read operation, so the error-propagation paths of -// CachingClient (which must not touch the overlay on failure) can be exercised. +// errClient wraps a Client and injects configurable errors into mutating/read +// operations, so the error-propagation paths of CachingClient (which must not +// touch the overlay on failure) can be exercised. type errClient struct { - client.Client + Client createErr error updateErr error patchErr error @@ -204,9 +168,9 @@ func (e *errClient) List(ctx context.Context, list client.ObjectList, opts ...cl return e.Client.List(ctx, list, opts...) } -// forceInnerAZ writes a divergent AvailabilityZone directly to the inner client, -// bypassing the caching wrapper. Used to prove that reads through the caching -// client are served from the overlay, not the inner client. +// forceInnerAZ writes a divergent AvailabilityZone directly to the inner +// client, bypassing the caching wrapper. Used to prove reads are served from +// the overlay, not the inner client. func forceInnerAZ(t *testing.T, inner client.Client, name, az string) { t.Helper() var cur v1alpha1.Reservation @@ -219,8 +183,7 @@ func forceInnerAZ(t *testing.T, inner client.Client, name, az string) { } } -// forceInnerStatusHost writes a divergent status Host directly to the inner client, -// bypassing the caching wrapper. +// forceInnerStatusHost writes a divergent status Host directly to the inner client. func forceInnerStatusHost(t *testing.T, inner client.Client, name, host string) { t.Helper() var cur v1alpha1.Reservation @@ -242,13 +205,51 @@ type unknownObject struct { func (u *unknownObject) DeepCopyObject() runtime.Object { return u } +func reservationConfig() Config { + return Config{ + GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, + TTL: metav1.Duration{Duration: 2 * time.Minute}, + } +} + +func newCaching(t *testing.T, inner Client) *CachingClient { + t.Helper() + c, err := New(inner, testScheme(t), reservationConfig()) + if err != nil { + t.Fatalf("New: %v", err) + } + return c +} + +func listReservations(t *testing.T, c client.Client, opts ...client.ListOption) []v1alpha1.Reservation { + t.Helper() + var list v1alpha1.ReservationList + if err := c.List(context.Background(), &list, opts...); err != nil { + t.Fatalf("List: %v", err) + } + return list.Items +} + +func reservationGVK() schema.GroupVersionKind { + return v1alpha1.GroupVersion.WithKind("Reservation") +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("condition not met within timeout") +} + // --- constructor tests --- -// TestNewUnknownGVKError: New fails when a configured GVK string is not -// registered in the scheme. func TestNewUnknownGVKError(t *testing.T) { - inner := newTestClient(t) - _, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{ + _, err := New(newTestClient(t), testScheme(t), Config{ GVKs: []string{"cortex.cloud/v1alpha1/DoesNotExist"}, }) if err == nil { @@ -256,10 +257,8 @@ func TestNewUnknownGVKError(t *testing.T) { } } -// TestNewDefaultTTL: a zero TTL in the config falls back to defaultTTL. func TestNewDefaultTTL(t *testing.T) { - inner := newTestClient(t) - c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{ + c, err := New(newTestClient(t), testScheme(t), Config{ GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, }) if err != nil { @@ -270,10 +269,8 @@ func TestNewDefaultTTL(t *testing.T) { } } -// TestNewExplicitTTL: a non-zero TTL is honoured verbatim. func TestNewExplicitTTL(t *testing.T) { - inner := newTestClient(t) - c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{ + c, err := New(newTestClient(t), testScheme(t), Config{ GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, TTL: metav1.Duration{Duration: 90 * time.Second}, }) @@ -287,11 +284,8 @@ func TestNewExplicitTTL(t *testing.T) { // --- overlay behaviour tests --- -// TestCreateThenGetVisible: Create then immediate Get shows the object via the -// overlay even before the informer has caught up. func TestCreateThenGetVisible(t *testing.T) { - inner := newTestClient(t) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t)) r := newReservation("res-1", "az-1", "") if err := c.Create(context.Background(), r); err != nil { @@ -307,22 +301,15 @@ func TestCreateThenGetVisible(t *testing.T) { } } -// TestOverlayWhenInnerEmpty: a seeded overlay entry appears in List even when -// the inner client returns nothing. func TestOverlayWhenInnerEmpty(t *testing.T) { - inner := newTestClient(t) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) - + c := newCaching(t, newTestClient(t)) c.upsert(reservationGVK(), newReservation("res-2", "az-1", "5")) - items := listReservations(t, c) - if len(items) != 1 || items[0].Name != "res-2" { + if items := listReservations(t, c); len(items) != 1 || items[0].Name != "res-2" { t.Fatalf("expected overlay entry res-2, got %+v", items) } } -// TestEviction: an informer sighting evicts the overlay entry only when it -// matches by UID and carries a ResourceVersion >= the cached one. func TestEviction(t *testing.T) { const cachedRV = "10" cases := []struct { @@ -341,8 +328,8 @@ func TestEviction(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - inf := &fakeInformer{} - c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: inf}) + inner := newTestClient(t) + c := newCaching(t, inner) ctx := t.Context() go func() { @@ -351,9 +338,9 @@ func TestEviction(t *testing.T) { } }() waitFor(t, func() bool { - inf.mu.Lock() - defer inf.mu.Unlock() - return len(inf.handlers) > 0 + inner.inf.mu.Lock() + defer inner.inf.mu.Unlock() + return len(inner.inf.handlers) > 0 }) c.upsert(reservationGVK(), newReservation("res-3", "az-1", cachedRV)) @@ -363,9 +350,9 @@ func TestEviction(t *testing.T) { observed.UID = types.UID(tc.observedUID) } if tc.useUpdate { - inf.fireUpdate(nil, observed) + inner.inf.fireUpdate(nil, observed) } else { - inf.fireAdd(observed) + inner.inf.fireAdd(observed) } _, present := c.getEntry(reservationGVK(), objectKey{name: "res-3"}) @@ -376,11 +363,9 @@ func TestEviction(t *testing.T) { } } -// TestEvictionIgnoresNonObject: a non-client.Object informer payload does not -// panic or evict. func TestEvictionIgnoresNonObject(t *testing.T) { - inf := &fakeInformer{} - c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: inf}) + inner := newTestClient(t) + c := newCaching(t, inner) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -390,21 +375,20 @@ func TestEvictionIgnoresNonObject(t *testing.T) { } }() waitFor(t, func() bool { - inf.mu.Lock() - defer inf.mu.Unlock() - return len(inf.handlers) > 0 + inner.inf.mu.Lock() + defer inner.inf.mu.Unlock() + return len(inner.inf.handlers) > 0 }) c.upsert(reservationGVK(), newReservation("res-x", "az-1", "1")) - inf.fireAdd("not-an-object") + inner.inf.fireAdd("not-an-object") if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-x"}); !ok { t.Fatalf("non-object event must not evict the entry") } } -// TestTTLCleanup: cleanupExpired removes entries whose TTL has passed. func TestTTLCleanup(t *testing.T) { - c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t)) c.upsert(reservationGVK(), newReservation("res-4", "az-1", "1")) c.mu.Lock() for _, entries := range c.byGVK { @@ -419,12 +403,10 @@ func TestTTLCleanup(t *testing.T) { } } -// TestTombstone: Delete stores a tombstone so subsequent Get/List return -// NotFound even while the inner client still has the object. func TestTombstone(t *testing.T) { r := newReservation("res-5", "az-1", "") inner := newTestClient(t, r) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, inner) if err := c.Delete(context.Background(), r); err != nil { t.Fatalf("Delete: %v", err) @@ -443,12 +425,10 @@ func TestTombstone(t *testing.T) { } } -// TestUpdateOverridesInner: after Update the overlay version wins over a stale -// inner read. func TestUpdateOverridesInner(t *testing.T) { r := newReservation("res-6", "az-old", "1") inner := newTestClient(t, r) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, inner) var cur v1alpha1.Reservation if err := inner.Get(context.Background(), types.NamespacedName{Name: "res-6"}, &cur); err != nil { @@ -468,10 +448,8 @@ func TestUpdateOverridesInner(t *testing.T) { } } -// TestLabelMatching: an overlay-only entry appears only for matching labels. func TestLabelMatching(t *testing.T) { - c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) - + c := newCaching(t, newTestClient(t)) r := newReservation("res-7", "az-1", "1") r.Labels = map[string]string{"team": "a"} c.upsert(reservationGVK(), r) @@ -484,11 +462,8 @@ func TestLabelMatching(t *testing.T) { } } -// TestFieldMatching: after IndexField registration, an overlay-only entry -// appears only for matching field selectors. func TestFieldMatching(t *testing.T) { - c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) - + c := newCaching(t, newTestClient(t)) if err := c.IndexField(context.Background(), &v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { res := obj.(*v1alpha1.Reservation) if res.Spec.AvailabilityZone == "" { @@ -498,7 +473,6 @@ func TestFieldMatching(t *testing.T) { }); err != nil { t.Fatalf("IndexField: %v", err) } - c.upsert(reservationGVK(), newReservation("res-8", "az-1", "1")) if match := listReservations(t, c, client.MatchingFields{azIndexField: "az-1"}); len(match) != 1 { @@ -509,11 +483,9 @@ func TestFieldMatching(t *testing.T) { } } -// TestNonCachedGVKPassthrough: calls for unconfigured GVKs pass through to the -// inner client with no overlay involvement. func TestNonCachedGVKPassthrough(t *testing.T) { inner := newTestClient(t) - c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{}) + c, err := New(inner, testScheme(t), Config{}) if err != nil { t.Fatalf("New: %v", err) } @@ -533,12 +505,9 @@ func TestNonCachedGVKPassthrough(t *testing.T) { } } -// TestDedup: an object present in both the inner client and the overlay appears -// exactly once in List, with the overlay version winning. func TestDedup(t *testing.T) { r := newReservation("res-10", "az-1", "1") - inner := newTestClient(t, r) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t, r)) newer := newReservation("res-10", "az-1", "2") newer.Spec.TargetHost = "host-x" @@ -555,32 +524,30 @@ func TestDedup(t *testing.T) { // --- error-propagation tests --- -// TestWriteErrorLeavesOverlayUntouched: a failing inner call leaves the overlay -// untouched (no live entry and no tombstone). func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { sentinel := errors.New("boom") cases := []struct { name string rv string seed bool - mkClient func(inner client.Client) client.Client + mkClient func(inner *fakeClient) Client op func(c *CachingClient, r *v1alpha1.Reservation) error }{ { name: "create", - mkClient: func(inner client.Client) client.Client { return &errClient{Client: inner, createErr: sentinel} }, + mkClient: func(inner *fakeClient) Client { return &errClient{Client: inner, createErr: sentinel} }, op: func(c *CachingClient, r *v1alpha1.Reservation) error { return c.Create(context.Background(), r) }, }, { name: "update", rv: "1", - mkClient: func(inner client.Client) client.Client { return &errClient{Client: inner, updateErr: sentinel} }, + mkClient: func(inner *fakeClient) Client { return &errClient{Client: inner, updateErr: sentinel} }, op: func(c *CachingClient, r *v1alpha1.Reservation) error { return c.Update(context.Background(), r) }, }, { name: "patch", rv: "1", - mkClient: func(inner client.Client) client.Client { return &errClient{Client: inner, patchErr: sentinel} }, + mkClient: func(inner *fakeClient) Client { return &errClient{Client: inner, patchErr: sentinel} }, op: func(c *CachingClient, r *v1alpha1.Reservation) error { p := r.DeepCopy() p.Spec.AvailabilityZone = "az-new" @@ -590,20 +557,20 @@ func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { { name: "delete", seed: true, - mkClient: func(inner client.Client) client.Client { return &errClient{Client: inner, deleteErr: sentinel} }, + mkClient: func(inner *fakeClient) Client { return &errClient{Client: inner, deleteErr: sentinel} }, op: func(c *CachingClient, r *v1alpha1.Reservation) error { return c.Delete(context.Background(), r) }, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { r := newReservation("res-"+tc.name, "az-1", tc.rv) - var base client.Client + var base *fakeClient if tc.seed { base = newTestClient(t, r) } else { base = newTestClient(t) } - c := newCaching(t, tc.mkClient(base), &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, tc.mkClient(base)) if err := tc.op(c, r); !errors.Is(err, sentinel) { t.Fatalf("expected sentinel error, got %v", err) @@ -615,9 +582,6 @@ func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { } } -// TestWriteServedFromOverlay: after a write through the caching client, a Get -// returns the written value even though the inner client has been forced to a -// divergent (stale) value behind the cache's back. func TestWriteServedFromOverlay(t *testing.T) { cases := []struct { name string @@ -672,7 +636,7 @@ func TestWriteServedFromOverlay(t *testing.T) { t.Run(tc.name, func(t *testing.T) { r := newReservation("res-served", "az-1", "") inner := newTestClient(t, r) - c := newCaching(t, inner, &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, inner) var cur v1alpha1.Reservation if err := inner.Get(context.Background(), types.NamespacedName{Name: r.Name}, &cur); err != nil { @@ -692,11 +656,9 @@ func TestWriteServedFromOverlay(t *testing.T) { } } -// TestGetPropagatesNonNotFoundError: a non-NotFound inner error is surfaced -// without consulting the overlay. func TestGetPropagatesNonNotFoundError(t *testing.T) { sentinel := errors.New("get boom") - c := newCaching(t, &errClient{Client: newTestClient(t), getErr: sentinel}, &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, &errClient{Client: newTestClient(t), getErr: sentinel}) c.upsert(reservationGVK(), newReservation("res-ge", "az-1", "1")) var got v1alpha1.Reservation @@ -705,10 +667,8 @@ func TestGetPropagatesNonNotFoundError(t *testing.T) { } } -// TestGetOverlayResurrectsNotFound: a live overlay entry satisfies a Get that -// the inner client reports as NotFound. func TestGetOverlayResurrectsNotFound(t *testing.T) { - c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t)) c.upsert(reservationGVK(), newReservation("res-gr", "az-z", "1")) var got v1alpha1.Reservation @@ -720,10 +680,8 @@ func TestGetOverlayResurrectsNotFound(t *testing.T) { } } -// TestGetNotFoundWithNoOverlay: inner NotFound with no overlay entry propagates -// NotFound unchanged. func TestGetNotFoundWithNoOverlay(t *testing.T) { - c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t)) var got v1alpha1.Reservation if err := c.Get(context.Background(), types.NamespacedName{Name: "missing"}, &got); !apierrors.IsNotFound(err) { @@ -731,11 +689,9 @@ func TestGetNotFoundWithNoOverlay(t *testing.T) { } } -// TestGetNonCachedPropagatesError: for a non-cached GVK, Get is a pure -// passthrough. func TestGetNonCachedPropagatesError(t *testing.T) { sentinel := errors.New("get boom") - c, err := New(&errClient{Client: newTestClient(t), getErr: sentinel}, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{}) + c, err := New(&errClient{Client: newTestClient(t), getErr: sentinel}, testScheme(t), Config{}) if err != nil { t.Fatalf("New: %v", err) } @@ -745,11 +701,9 @@ func TestGetNonCachedPropagatesError(t *testing.T) { } } -// TestListPropagatesError: a failing inner List surfaces the error rather than -// returning a partial overlay merge. func TestListPropagatesError(t *testing.T) { sentinel := errors.New("list boom") - c := newCaching(t, &errClient{Client: newTestClient(t), listErr: sentinel}, &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, &errClient{Client: newTestClient(t), listErr: sentinel}) c.upsert(reservationGVK(), newReservation("res-le", "az-1", "1")) var list v1alpha1.ReservationList @@ -758,10 +712,8 @@ func TestListPropagatesError(t *testing.T) { } } -// TestStatusUpdateErrorLeavesOverlayUntouched: a failed status update does not -// populate the overlay. func TestStatusUpdateErrorLeavesOverlayUntouched(t *testing.T) { - c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t)) r := newReservation("res-se", "az-1", "1") if err := c.Status().Update(context.Background(), r); err == nil { @@ -772,11 +724,9 @@ func TestStatusUpdateErrorLeavesOverlayUntouched(t *testing.T) { } } -// TestStatusCreateDelegates: Status().Create delegates to the inner status -// writer and never touches the overlay. func TestStatusCreateDelegates(t *testing.T) { r := newReservation("res-sc", "az-1", "") - c := newCaching(t, newTestClient(t, r), &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t, r)) if err := c.Status().Create(context.Background(), r, r); err == nil { t.Fatalf("expected Status().Create to fail on fake client") @@ -786,12 +736,10 @@ func TestStatusCreateDelegates(t *testing.T) { } } -// TestStatusUpdateNonCachedNoOverlay: Status().Update for a non-cached GVK does -// not touch the overlay. func TestStatusUpdateNonCachedNoOverlay(t *testing.T) { r := newReservation("res-sn", "az-1", "") inner := newTestClient(t, r) - c, err := New(inner, &fakeInformerSource{inf: &fakeInformer{}}, testScheme(t), Config{}) + c, err := New(inner, testScheme(t), Config{}) if err != nil { t.Fatalf("New: %v", err) } @@ -810,16 +758,13 @@ func TestStatusUpdateNonCachedNoOverlay(t *testing.T) { // --- helper / utility tests --- -// TestGVKForUnknownType: gvkFor reports not-cached for a type not registered in -// the scheme. func TestGVKForUnknownType(t *testing.T) { - c := newCaching(t, newTestClient(t), &fakeInformerSource{inf: &fakeInformer{}}) + c := newCaching(t, newTestClient(t)) if _, cached := c.gvkFor(&unknownObject{}); cached { t.Fatalf("unknown type must not be reported as cached") } } -// TestTrimListSuffix exercises the list-kind suffix trimming helper. func TestTrimListSuffix(t *testing.T) { cases := []struct { in string diff --git a/pkg/clientcache/interfaces.go b/pkg/clientcache/interfaces.go index 8c2e87599..c2c3e6dd5 100644 --- a/pkg/clientcache/interfaces.go +++ b/pkg/clientcache/interfaces.go @@ -10,11 +10,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// InformerSource provides, per object type, the informers the cache attaches -// to for eviction purposes. It is satisfied structurally e.g. by -// *multicluster.Client (via ClustersForGVK + cluster.GetCache().GetInformer), -// so that this package does not need to import pkg/multicluster. -type InformerSource interface { +// Client is the interface the CachingClient requires of its inner client. +// It extends client.Client with the informer access needed for overlay eviction. +// *multicluster.Client satisfies this interface. +type Client interface { + client.Client // GetInformersForKind returns all informers serving the GVK of the given // object. The cache attaches Add/Update event handlers to each informer to // evict overlay entries once the real object appears in the informer cache. diff --git a/pkg/clientcache/runnable.go b/pkg/clientcache/runnable.go index b64500df2..f166686ca 100644 --- a/pkg/clientcache/runnable.go +++ b/pkg/clientcache/runnable.go @@ -28,7 +28,7 @@ func (c *CachingClient) Start(ctx context.Context) error { log.Error(err, "failed to build object for gvk; eviction disabled for it", "gvk", gvk) continue } - informers, err := c.informers.GetInformersForKind(ctx, obj) + informers, err := c.inner.GetInformersForKind(ctx, obj) if err != nil { log.Error(err, "failed to get informers for gvk; eviction disabled for it", "gvk", gvk) continue From 949d05fff5bf130444b2834f5f3333f9eaca3944 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Fri, 7 Aug 2026 10:00:33 +0200 Subject: [PATCH 06/16] feat: implement per-object write locks in CachingClient to prevent overlay stale reads during concurrent updates --- pkg/clientcache/client.go | 154 +++++++++++++++++++++++++++------ pkg/clientcache/client_test.go | 73 ++++++++++++++++ 2 files changed, 202 insertions(+), 25 deletions(-) diff --git a/pkg/clientcache/client.go b/pkg/clientcache/client.go index 0979a0020..7561daf28 100644 --- a/pkg/clientcache/client.go +++ b/pkg/clientcache/client.go @@ -39,6 +39,80 @@ func keyForObject(obj client.Object) objectKey { return objectKey{namespace: obj.GetNamespace(), name: obj.GetName()} } +// lockKey identifies the object whose write path a keyedMutex serializes. +type lockKey struct { + gvk schema.GroupVersionKind + key objectKey +} + +// refMutex is the actual per-object lock plus a reference count of how many +// callers currently hold or are waiting for it. The count lets keyedMutex know +// when the entry is unused so it can be deleted (see keyedMutex.lock). +type refMutex struct { + mu sync.Mutex + ref int +} + +// keyedMutex hands out one mutex per key, serializing operations that share a +// key while letting distinct keys proceed concurrently. +// +// A sync.Map (or a map we only ever insert into) would be simpler, but it would leak: +// this cache wraps a single client for the whole lifetime of the controller-manager process, +// and a controller reconciles a continuous stream of (often short-lived) objects. +// A map would therefore accumulate one mutex per distinct object ever written and +// grow without bound. To avoid that we reference count each entry and delete it +// once the last holder unlocks, so the map only ever holds locks for objects +// with writes currently in flight. +// +// The two mutexes have distinct, non-overlapping roles: +// - k.mu guards the locks map itself. It is only ever held for the tiny +// bookkeeping critical sections below (map lookup/insert/delete and the +// ref counter), never across the caller's I/O. +// - rm.mu is the real per-object lock, held by the caller across the inner +// client call and the overlay mutation. +// +// Because k.mu is never held while rm.mu is locked (we release k.mu before +// taking rm.mu), the two can never deadlock against each other. +type keyedMutex struct { + mu sync.Mutex + locks map[lockKey]*refMutex +} + +func newKeyedMutex() *keyedMutex { + return &keyedMutex{locks: make(map[lockKey]*refMutex)} +} + +// lock acquires the per-key mutex and returns a function that releases it. +func (k *keyedMutex) lock(lk lockKey) func() { + // Look up (or create) the entry for this key and register our interest by + // bumping ref, all under k.mu so the map stays consistent. We increment ref + // here, before taking rm.mu, so that a concurrent unlock cannot see ref==0 + // and delete the entry out from under us while we are blocked waiting on it. + k.mu.Lock() + rm := k.locks[lk] + if rm == nil { + rm = &refMutex{} + k.locks[lk] = rm + } + rm.ref++ + k.mu.Unlock() + + // Take the real per-object lock outside k.mu; this is where a second caller + // for the same key blocks (and where we may block across the caller's I/O). + rm.mu.Lock() + return func() { + rm.mu.Unlock() + // Drop our reference and, if we were the last holder, remove the entry + // so the map does not grow unbounded. + k.mu.Lock() + rm.ref-- + if rm.ref == 0 { + delete(k.locks, lk) + } + k.mu.Unlock() + } +} + // resourceVersionAtLeast reports whether observed >= cached, treating // ResourceVersions as opaque monotonically increasing integers (as the // kubernetes apiserver guarantees per resource). Unparsable or empty values @@ -86,6 +160,11 @@ type CachingClient struct { mu sync.RWMutex byGVK map[schema.GroupVersionKind]map[objectKey]*entry indexers map[schema.GroupVersionKind]map[string]client.IndexerFunc + + // writeLocks serializes writes to the same object so the inner call and the + // overlay mutation are atomic per object, keeping the overlay from falling + // behind the apiserver under concurrent writes. + writeLocks *keyedMutex } // New builds a CachingClient wrapping inner. informers supplies the informers @@ -102,13 +181,14 @@ func New(inner Client, scheme *runtime.Scheme, conf Config) (*CachingClient, err ttl = defaultTTL } return &CachingClient{ - Client: inner, - inner: inner, - scheme: scheme, - ttl: ttl, - gvks: gvks, - byGVK: make(map[schema.GroupVersionKind]map[objectKey]*entry), - indexers: make(map[schema.GroupVersionKind]map[string]client.IndexerFunc), + Client: inner, + inner: inner, + scheme: scheme, + ttl: ttl, + gvks: gvks, + byGVK: make(map[schema.GroupVersionKind]map[objectKey]*entry), + indexers: make(map[schema.GroupVersionKind]map[string]client.IndexerFunc), + writeLocks: newKeyedMutex(), }, nil } @@ -354,36 +434,48 @@ func (c *CachingClient) fieldSetLocked(gvk schema.GroupVersionKind, obj client.O // Create delegates to the inner client and, on success for a cached GVK, adds // the object to the overlay. func (c *CachingClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.Create(ctx, obj, opts...) + } + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + defer unlock() if err := c.Client.Create(ctx, obj, opts...); err != nil { return err } - if gvk, cached := c.gvkFor(obj); cached { - c.upsert(gvk, obj) - } + c.upsert(gvk, obj) return nil } // Update delegates to the inner client and, on success for a cached GVK, // refreshes the overlay entry. func (c *CachingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.Update(ctx, obj, opts...) + } + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + defer unlock() if err := c.Client.Update(ctx, obj, opts...); err != nil { return err } - if gvk, cached := c.gvkFor(obj); cached { - c.upsert(gvk, obj) - } + c.upsert(gvk, obj) return nil } // Patch delegates to the inner client and, on success for a cached GVK, // refreshes the overlay entry with the patched object. func (c *CachingClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.Patch(ctx, obj, patch, opts...) + } + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + defer unlock() if err := c.Client.Patch(ctx, obj, patch, opts...); err != nil { return err } - if gvk, cached := c.gvkFor(obj); cached { - c.upsert(gvk, obj) - } + c.upsert(gvk, obj) return nil } @@ -394,12 +486,16 @@ func (c *CachingClient) Patch(ctx context.Context, obj client.Object, patch clie // ensures that reads between the Delete call and the informer event correctly // return NotFound rather than serving a stale object from the informer cache. func (c *CachingClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.Delete(ctx, obj, opts...) + } + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + defer unlock() if err := c.Client.Delete(ctx, obj, opts...); err != nil { return err } - if gvk, cached := c.gvkFor(obj); cached { - c.tombstone(gvk, obj) - } + c.tombstone(gvk, obj) return nil } @@ -482,22 +578,30 @@ func (s *statusWriter) Create(ctx context.Context, obj, subResource client.Objec } func (s *statusWriter) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + gvk, cached := s.c.gvkFor(obj) + if !cached { + return s.inner.Update(ctx, obj, opts...) + } + unlock := s.c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + defer unlock() if err := s.inner.Update(ctx, obj, opts...); err != nil { return err } - if gvk, cached := s.c.gvkFor(obj); cached { - s.c.upsert(gvk, obj) - } + s.c.upsert(gvk, obj) return nil } func (s *statusWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + gvk, cached := s.c.gvkFor(obj) + if !cached { + return s.inner.Patch(ctx, obj, patch, opts...) + } + unlock := s.c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + defer unlock() if err := s.inner.Patch(ctx, obj, patch, opts...); err != nil { return err } - if gvk, cached := s.c.gvkFor(obj); cached { - s.c.upsert(gvk, obj) - } + s.c.upsert(gvk, obj) return nil } diff --git a/pkg/clientcache/client_test.go b/pkg/clientcache/client_test.go index 8b99d067e..6d7183e9a 100644 --- a/pkg/clientcache/client_test.go +++ b/pkg/clientcache/client_test.go @@ -6,6 +6,8 @@ package clientcache import ( "context" "errors" + goruntime "runtime" + "strconv" "sync" "testing" "time" @@ -756,6 +758,77 @@ func TestStatusUpdateNonCachedNoOverlay(t *testing.T) { } } +// --- concurrency regression test --- + +// orderingClient is a fake inner client whose Update records the committed +// ResourceVersion (in inner-commit order) and then yields, widening the window +// between the inner commit and the overlay upsert. This exposes the ordering +// race the per-object write lock is meant to prevent: without the lock two +// concurrent writers to the same object can commit in one order yet upsert in +// the reverse order, leaving the overlay behind the apiserver. +// +// The per-object write lock makes each inner call + upsert atomic, so the inner +// commit order and the overlay upsert order are identical: the overlay always +// reflects the last write that reached the inner client (lastRV). +type orderingClient struct { + Client + mu sync.Mutex + lastRV string // ResourceVersion of the most recent inner commit +} + +func (o *orderingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + o.mu.Lock() + o.lastRV = obj.GetResourceVersion() + o.mu.Unlock() + // Yield after the inner commit but before the caller upserts, widening the + // commit-vs-overlay reorder window. + goruntime.Gosched() + return nil +} + +// TestConcurrentUpdatesOverlayNotBehind fires many concurrent Updates to the +// SAME object with distinct ResourceVersions. The per-object write lock must +// make each inner call + overlay update atomic, so once all writes settle the +// overlay entry reflects the last write that reached the inner client (overlay +// RV == last committed RV, never a reordered/stale one). Run under -race to +// also catch data races. +func TestConcurrentUpdatesOverlayNotBehind(t *testing.T) { + const ( + rounds = 50 + n = 8 + ) + for round := range rounds { + oc := &orderingClient{Client: newTestClient(t)} + c := newCaching(t, oc) + + var wg sync.WaitGroup + for i := 1; i <= n; i++ { + wg.Add(1) + go func(rv int) { + defer wg.Done() + r := newReservation("res-conc", "az-1", strconv.Itoa(rv)) + if err := c.Update(context.Background(), r); err != nil { + t.Errorf("Update rv=%d: %v", rv, err) + } + }(i) + } + wg.Wait() + + oc.mu.Lock() + lastRV := oc.lastRV + oc.mu.Unlock() + + e, ok := c.getEntry(reservationGVK(), objectKey{name: "res-conc"}) + if !ok { + t.Fatalf("round %d: expected overlay entry for res-conc", round) + } + if e.resourceVersion != lastRV { + t.Fatalf("round %d: overlay RV %q does not match last inner commit %q", + round, e.resourceVersion, lastRV) + } + } +} + // --- helper / utility tests --- func TestGVKForUnknownType(t *testing.T) { From a783126b27cc98b794d32283198a6e6512bd2f66 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Fri, 7 Aug 2026 10:05:58 +0200 Subject: [PATCH 07/16] fix: prevent mutation of shared cache entry by deep-copying cached object in Get method --- pkg/clientcache/client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/clientcache/client.go b/pkg/clientcache/client.go index 7561daf28..1eee1ea9f 100644 --- a/pkg/clientcache/client.go +++ b/pkg/clientcache/client.go @@ -520,7 +520,10 @@ func (c *CachingClient) Get(ctx context.Context, key client.ObjectKey, obj clien return apierrors.NewNotFound(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, key.Name) } // Live overlay entry: copy it into obj, overriding the inner result. - if cpErr := c.scheme.Convert(e.obj, obj, nil); cpErr != nil { + // Deep-copy the cached object first so scheme.Convert cannot alias the + // overlay entry's maps, slices, or metadata into the caller's obj (which + // would let callers mutate the shared cache entry). + if cpErr := c.scheme.Convert(e.obj.DeepCopyObject(), obj, nil); cpErr != nil { return cpErr } return nil From a4dfed2cbc2cc7e5a44152d4f176e01f9184471e Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Fri, 7 Aug 2026 10:06:22 +0200 Subject: [PATCH 08/16] feat: add NeedLeaderElection method to CachingClient for lifecycle management across replicas --- pkg/clientcache/runnable.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/clientcache/runnable.go b/pkg/clientcache/runnable.go index f166686ca..b1749276e 100644 --- a/pkg/clientcache/runnable.go +++ b/pkg/clientcache/runnable.go @@ -54,6 +54,14 @@ func (c *CachingClient) Start(ctx context.Context) error { } } +// NeedLeaderElection reports that the CachingClient's Start lifecycle must run +// on every replica, not only the elected leader. The overlay is per-process +// state, so its eviction handlers and TTL cleanup have to run wherever the +// client is used, regardless of leader election. +func (c *CachingClient) NeedLeaderElection() bool { + return false +} + // evictionHandler returns an informer event handler that evicts overlay entries // for the GVK when the real object is observed at a >= ResourceVersion. func (c *CachingClient) evictionHandler(gvk schema.GroupVersionKind) toolscachek8s.ResourceEventHandler { From 72b1a015a7b335882d1f6358a9042cf11db54888 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Fri, 7 Aug 2026 10:22:28 +0200 Subject: [PATCH 09/16] feat: Add delete all of --- pkg/clientcache/client.go | 31 +++++++++++++++++++++++ pkg/clientcache/client_test.go | 46 +++++++++++++++++++++++++++++----- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/pkg/clientcache/client.go b/pkg/clientcache/client.go index 1eee1ea9f..16349998e 100644 --- a/pkg/clientcache/client.go +++ b/pkg/clientcache/client.go @@ -499,6 +499,37 @@ func (c *CachingClient) Delete(ctx context.Context, obj client.Object, opts ...c return nil } +// DeleteAllOf delegates to the inner client and, on success for a cached GVK, +// tombstones all overlay entries that match the delete options. Objects that +// live only in the informer cache (not in the overlay) will be evicted +// naturally once the deletion propagates through the informer. +func (c *CachingClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.DeleteAllOf(ctx, obj, opts...) + } + if err := c.Client.DeleteAllOf(ctx, obj, opts...); err != nil { + return err + } + dao := &client.DeleteAllOfOptions{} + dao.ApplyOptions(opts) + c.mu.Lock() + defer c.mu.Unlock() + for key, e := range c.byGVK[gvk] { + if !c.matchesLocked(gvk, e.obj, &dao.ListOptions) { + continue + } + c.byGVK[gvk][key] = &entry{ + obj: e.obj, + uid: e.uid, + resourceVersion: e.resourceVersion, + deleted: true, + expiresAt: time.Now().Add(c.ttl), + } + } + return nil +} + // Get delegates to the inner client, then applies the overlay: a tombstone // yields NotFound; a live overlay entry overrides the inner result; and an // overlay entry can satisfy a Get that the inner client reports as NotFound. diff --git a/pkg/clientcache/client_test.go b/pkg/clientcache/client_test.go index 6d7183e9a..f066b0bea 100644 --- a/pkg/clientcache/client_test.go +++ b/pkg/clientcache/client_test.go @@ -120,12 +120,13 @@ func newTestClient(t *testing.T, objs ...client.Object) *fakeClient { // touch the overlay on failure) can be exercised. type errClient struct { Client - createErr error - updateErr error - patchErr error - deleteErr error - getErr error - listErr error + createErr error + updateErr error + patchErr error + deleteErr error + deleteAllOfErr error + getErr error + listErr error } func (e *errClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { @@ -156,6 +157,13 @@ func (e *errClient) Delete(ctx context.Context, obj client.Object, opts ...clien return e.Client.Delete(ctx, obj, opts...) } +func (e *errClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + if e.deleteAllOfErr != nil { + return e.deleteAllOfErr + } + return e.Client.DeleteAllOf(ctx, obj, opts...) +} + func (e *errClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { if e.getErr != nil { return e.getErr @@ -427,6 +435,32 @@ func TestTombstone(t *testing.T) { } } +func TestDeleteAllOf(t *testing.T) { + r1 := newReservation("res-dao-1", "az-1", "1") + r1.Labels = map[string]string{"zone": "a"} + r2 := newReservation("res-dao-2", "az-2", "1") + r2.Labels = map[string]string{"zone": "b"} + inner := newTestClient(t, r1, r2) + c := newCaching(t, inner) + + // Populate overlay for both so we can verify tombstoning. + c.upsert(reservationGVK(), r1) + c.upsert(reservationGVK(), r2) + + // DeleteAllOf with a label selector — only r1 should be tombstoned. + if err := c.DeleteAllOf(context.Background(), &v1alpha1.Reservation{}, client.MatchingLabels{"zone": "a"}); err != nil { + t.Fatalf("DeleteAllOf: %v", err) + } + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-dao-1"}, &got); !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound for tombstoned res-dao-1, got %v", err) + } + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-dao-2"}, &got); err != nil { + t.Fatalf("res-dao-2 should still be visible, got %v", err) + } +} + func TestUpdateOverridesInner(t *testing.T) { r := newReservation("res-6", "az-old", "1") inner := newTestClient(t, r) From 8f9363635882c022687b406b553ec5277c33d42e Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Fri, 7 Aug 2026 10:25:59 +0200 Subject: [PATCH 10/16] feat: add label matching test for overlay changes in CachingClient Signed-off-by: Markus Wieland --- pkg/clientcache/client.go | 6 +++++ pkg/clientcache/client_test.go | 41 ++++++++++++++++++++++++++++------ pkg/clientcache/runnable.go | 7 ++++++ 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/pkg/clientcache/client.go b/pkg/clientcache/client.go index 16349998e..c80fde968 100644 --- a/pkg/clientcache/client.go +++ b/pkg/clientcache/client.go @@ -375,6 +375,12 @@ func (c *CachingClient) overlayList(gvk schema.GroupVersionKind, existing []runt // Tombstone: drop the object entirely. continue } + // The inner result matched the query against the informer's (old) field + // values. Re-check the overlay version: if a write changed a queried + // field, the overlay object no longer belongs in this result set. + if !c.matchesLocked(gvk, e.obj, lo) { + continue + } result = append(result, e.obj.DeepCopyObject()) } diff --git a/pkg/clientcache/client_test.go b/pkg/clientcache/client_test.go index f066b0bea..75aa3db9e 100644 --- a/pkg/clientcache/client_test.go +++ b/pkg/clientcache/client_test.go @@ -120,13 +120,13 @@ func newTestClient(t *testing.T, objs ...client.Object) *fakeClient { // touch the overlay on failure) can be exercised. type errClient struct { Client - createErr error - updateErr error - patchErr error - deleteErr error - deleteAllOfErr error - getErr error - listErr error + createErr error + updateErr error + patchErr error + deleteErr error + deleteAllOfErr error + getErr error + listErr error } func (e *errClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { @@ -484,6 +484,33 @@ func TestUpdateOverridesInner(t *testing.T) { } } +func TestLabelMatchingAfterOverlayChange(t *testing.T) { + // Regression: if a write changes a field that is part of the query, the + // overlay version must be re-matched against the list options. Before the + // fix, the inner List would include the object (matching the old value in + // the informer), and overlayList would silently swap in the new version, + // returning an object that does not satisfy the query. + r := newReservation("res-overlay-label", "az-1", "1") + r.Labels = map[string]string{"team": "a"} + inner := newTestClient(t, r) + c := newCaching(t, inner) + + // Update label in overlay only (bypass inner to simulate informer lag). + updated := r.DeepCopy() + updated.Labels = map[string]string{"team": "b"} + c.upsert(reservationGVK(), updated) + + // Query for the OLD label value — informer still returns the object, but + // the overlay version has team=b, so it must be excluded. + if got := listReservations(t, c, client.MatchingLabels{"team": "a"}); len(got) != 0 { + t.Fatalf("expected no results for team=a after overlay changed label to b, got %+v", got) + } + // Query for the NEW label value — overlay-only path must include it. + if got := listReservations(t, c, client.MatchingLabels{"team": "b"}); len(got) != 1 { + t.Fatalf("expected one result for team=b, got %+v", got) + } +} + func TestLabelMatching(t *testing.T) { c := newCaching(t, newTestClient(t)) r := newReservation("res-7", "az-1", "1") diff --git a/pkg/clientcache/runnable.go b/pkg/clientcache/runnable.go index b1749276e..7196dd787 100644 --- a/pkg/clientcache/runnable.go +++ b/pkg/clientcache/runnable.go @@ -75,6 +75,13 @@ func (c *CachingClient) evictionHandler(gvk schema.GroupVersionKind) toolscachek return toolscachek8s.ResourceEventHandlerFuncs{ AddFunc: func(o any) { evict(o) }, UpdateFunc: func(_, o any) { evict(o) }, + DeleteFunc: func(o any) { + if d, ok := o.(toolscachek8s.DeletedFinalStateUnknown); ok { + evict(d.Obj) + return + } + evict(o) + }, } } From 1dd9c5ca3b9aac1d0aa234774dea9e9b21448951 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 12 Aug 2026 10:04:20 +0200 Subject: [PATCH 11/16] feat: inflight controller using indexfields of overlay cache Signed-off-by: Markus Wieland --- cmd/manager/main.go | 4 ++-- .../reservations/inflight/controller.go | 16 ++++++++-------- .../reservations/inflight/controller_test.go | 9 --------- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 7e09976b2..ab888d37b 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -522,8 +522,8 @@ func main() { "controller", "inflight-reservation-controller") config := conf.GetConfigOrDie[inflight.NovaVMClientConfig]() vmClient := inflight.NewNovaVMClient(config) - controller := &inflight.Controller{Client: multiclusterClient, VMClient: vmClient} - if err := controller.SetupWithManager(ctx, mgr); err != nil { + controller := &inflight.Controller{Client: cachingClient, VMClient: vmClient} + if err := controller.SetupWithManager(ctx, mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "inflight-reservation-controller") os.Exit(1) diff --git a/internal/scheduling/reservations/inflight/controller.go b/internal/scheduling/reservations/inflight/controller.go index 4292fa334..e68713689 100644 --- a/internal/scheduling/reservations/inflight/controller.go +++ b/internal/scheduling/reservations/inflight/controller.go @@ -378,14 +378,7 @@ func (c *Controller) predicateHypervisors() predicate.Predicate { // SetupWithManager sets up the controller with the Manager and a multicluster // client. The multicluster client is used to watch for changes in the // Reservation CRD across all clusters and trigger reconciliations accordingly. -func (c *Controller) SetupWithManager(ctx context.Context, mgr ctrl.Manager) (err error) { - // Check that the provided client is a multicluster client, since we need - // that to watch for hypervisors across clusters. Do this before adding - // any runnables so a misconfigured setup fails fast. - mcl, ok := c.Client.(*multicluster.Client) - if !ok { - return errors.New("provided client must be a multicluster client") - } +func (c *Controller) SetupWithManager(ctx context.Context, mgr ctrl.Manager, mcl *multicluster.Client) (err error) { // Add the vm client as runnable to the manager. if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { return c.VMClient.StartWithKubernetesSecrets(ctx, c.Client) @@ -412,6 +405,13 @@ func (c *Controller) SetupWithManager(ctx context.Context, mgr ctrl.Manager) (er ); err != nil { return err } + // Register the same index with the overlay so that List calls with + // MatchingFields correctly filter in-flight overlay entries. + if fi, ok := c.Client.(client.FieldIndexer); ok { + if err := fi.IndexField(ctx, &v1alpha1.Reservation{}, idxReservationByTargetHost, idxReservationByTargetHostFn); err != nil { + return err + } + } // Watch hypervisor changes and requeue reservations targeting // the changed hypervisor. bldr, err = bldr.WatchesMulticluster(&hv1.Hypervisor{}, diff --git a/internal/scheduling/reservations/inflight/controller_test.go b/internal/scheduling/reservations/inflight/controller_test.go index 6f61d3a76..bfeb04754 100644 --- a/internal/scheduling/reservations/inflight/controller_test.go +++ b/internal/scheduling/reservations/inflight/controller_test.go @@ -716,12 +716,3 @@ func TestHandleHypervisors_NoMatchingReservations(t *testing.T) { t.Errorf("queue = %+v, want empty", q.items) } } - -func TestSetupWithManager_RejectsNonMulticlusterClient(t *testing.T) { - scheme := newTestScheme(t) - c := &Controller{Client: newTestClient(scheme), VMClient: &stubVMClient{}} - err := c.SetupWithManager(context.Background(), nil) - if err == nil { - t.Fatal("expected error for non-multicluster client, got nil") - } -} From efc33174f4b962a420e3bf8c80875cfea1cef44c Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 19 Aug 2026 10:39:36 +0200 Subject: [PATCH 12/16] Refactored pending cache to wrap around inner clusters of mcl instead if the mcl Signed-off-by: Markus Wieland --- cmd/manager/main.go | 95 +++---- helm/bundles/cortex-nova/values.yaml | 3 +- internal/shim/placement/field_index_test.go | 4 + pkg/clientcache/interfaces.go | 22 -- pkg/multicluster/client.go | 130 +++++----- pkg/multicluster/client_test.go | 68 +++++ pkg/multicluster/config.go | 52 ++++ pkg/{clientcache => pendingcache}/client.go | 106 ++++---- .../client_test.go | 238 ++++++++++++------ pkg/pendingcache/cluster.go | 31 +++ pkg/{clientcache => pendingcache}/config.go | 12 +- pkg/{clientcache => pendingcache}/runnable.go | 27 +- 12 files changed, 489 insertions(+), 299 deletions(-) delete mode 100644 pkg/clientcache/interfaces.go create mode 100644 pkg/multicluster/config.go rename pkg/{clientcache => pendingcache}/client.go (82%) rename pkg/{clientcache => pendingcache}/client_test.go (79%) create mode 100644 pkg/pendingcache/cluster.go rename pkg/{clientcache => pendingcache}/config.go (71%) rename pkg/{clientcache => pendingcache}/runnable.go (67%) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index ab888d37b..474a3cd08 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -65,7 +65,6 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/inflight" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/quota" - "github.com/cobaltcore-dev/cortex/pkg/clientcache" "github.com/cobaltcore-dev/cortex/pkg/conf" "github.com/cobaltcore-dev/cortex/pkg/monitoring" "github.com/cobaltcore-dev/cortex/pkg/multicluster" @@ -397,22 +396,6 @@ func main() { os.Exit(1) } - // Transparent in-process overlay cache for CRDs that are eventually - // consistent across in-pod clients (e.g. Reservations). Writes populate an - // overlay; reads merge it with the informer result until the real object is - // observed. *multicluster.Client satisfies clientcache.Client, providing - // both the inner client.Client and informer access for eviction. - clientCacheConfig := conf.GetConfigOrDie[clientcache.RootConfig]() - cachingClient, err := clientcache.New(multiclusterClient, scheme, clientCacheConfig.ClientCache) - if err != nil { - setupLog.Error(err, "unable to create client cache") - os.Exit(1) - } - if err := mgr.Add(cachingClient); err != nil { - setupLog.Error(err, "unable to add client cache to manager") - os.Exit(1) - } - // Our custom monitoring registry can add prometheus labels to all metrics. // This is useful to distinguish metrics from different deployments. metricsConfig := conf.GetConfigOrDie[monitoring.Config]() @@ -445,10 +428,10 @@ func main() { commitmentsConfig := conf.GetConfigOrDie[commitments.Config]() var commitmentsVMSource reservations.VMSource if commitmentsConfig.DatasourceName != "" { - commitmentsVMSource = reservations.NewPostgresVMSource(cachingClient, commitmentsConfig.DatasourceName) + commitmentsVMSource = reservations.NewPostgresVMSource(multiclusterClient, commitmentsConfig.DatasourceName) } if slices.Contains(mainConfig.EnabledControllers, "committed-resource-reservations-controller") { - commitmentsAPI := commitmentsapi.NewAPIWithConfig(cachingClient, commitmentsConfig.API, commitmentsVMSource) + commitmentsAPI := commitmentsapi.NewAPIWithConfig(multiclusterClient, commitmentsConfig.API, commitmentsVMSource) commitmentsAPI.Init(mux, metrics.Registry, ctrl.Log.WithName("commitments-api")) } @@ -468,8 +451,8 @@ func main() { metrics.Registry.MustRegister(noHostFoundCounter) metrics.Registry.MustRegister(placementCounter) // Inferred through the base controller. - filterWeigherController.Client = cachingClient - filterWeigherController.CRRecorder.Client = cachingClient + filterWeigherController.Client = multiclusterClient + filterWeigherController.CRRecorder.Client = multiclusterClient if err := filterWeigherController.SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "nova FilterWeigherPipelineController") os.Exit(1) @@ -484,7 +467,7 @@ func main() { novaClient := nova.NewNovaClient() novaClientConfig := conf.GetConfigOrDie[nova.NovaClientConfig]() if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - return novaClient.Init(ctx, cachingClient, novaClientConfig) + return novaClient.Init(ctx, multiclusterClient, novaClientConfig) })); err != nil { setupLog.Error(err, "unable to initialize nova client") os.Exit(1) @@ -495,7 +478,7 @@ func main() { Breaker: &nova.DetectorCycleBreaker{NovaClient: novaClient}, } // Inferred through the base controller. - deschedulingsController.Client = cachingClient + deschedulingsController.Client = multiclusterClient if err := (deschedulingsController).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "nova DetectorPipelineController") os.Exit(1) @@ -503,7 +486,7 @@ func main() { go deschedulingsController.CreateDeschedulingsPeriodically(ctx) // Deschedulings cleanup on startup if err := (&nova.DeschedulingsCleanup{ - Client: cachingClient, + Client: multiclusterClient, Scheme: mgr.GetScheme(), }).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Cleanup") @@ -522,7 +505,7 @@ func main() { "controller", "inflight-reservation-controller") config := conf.GetConfigOrDie[inflight.NovaVMClientConfig]() vmClient := inflight.NewNovaVMClient(config) - controller := &inflight.Controller{Client: cachingClient, VMClient: vmClient} + controller := &inflight.Controller{Client: multiclusterClient, VMClient: vmClient} if err := controller.SetupWithManager(ctx, mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "inflight-reservation-controller") @@ -535,13 +518,13 @@ func main() { novaClient := nova.NewNovaClient() novaClientConfig := conf.GetConfigOrDie[nova.NovaClientConfig]() if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - return novaClient.Init(ctx, cachingClient, novaClientConfig) + return novaClient.Init(ctx, multiclusterClient, novaClientConfig) })); err != nil { setupLog.Error(err, "unable to initialize nova client") os.Exit(1) } if err := (&nova.DeschedulingsExecutor{ - Client: cachingClient, + Client: multiclusterClient, Scheme: mgr.GetScheme(), Conf: executorConfig, NovaClient: novaClient, @@ -552,7 +535,7 @@ func main() { } if slices.Contains(mainConfig.EnabledControllers, "hypervisor-overcommit-controller") { hypervisorOvercommitController := &nova.HypervisorOvercommitController{} - hypervisorOvercommitController.Client = cachingClient + hypervisorOvercommitController.Client = multiclusterClient if err := hypervisorOvercommitController.SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "HypervisorOvercommitController") @@ -565,7 +548,7 @@ func main() { Monitor: filterWeigherPipelineMonitor, } // Inferred through the base controller. - controller.Client = cachingClient + controller.Client = multiclusterClient if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler") os.Exit(1) @@ -585,7 +568,7 @@ func main() { Monitor: filterWeigherPipelineMonitor, } // Inferred through the base controller. - controller.Client = cachingClient + controller.Client = multiclusterClient if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler") os.Exit(1) @@ -605,7 +588,7 @@ func main() { Monitor: filterWeigherPipelineMonitor, } // Inferred through the base controller. - controller.Client = cachingClient + controller.Client = multiclusterClient if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler") os.Exit(1) @@ -624,7 +607,7 @@ func main() { Monitor: filterWeigherPipelineMonitor, } // Inferred through the base controller. - controller.Client = cachingClient + controller.Client = multiclusterClient if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler") os.Exit(1) @@ -640,11 +623,11 @@ func main() { if slices.Contains(mainConfig.EnabledControllers, "committed-resource-reservations-controller") { setupLog.Info("enabling controller", "controller", "committed-resource-reservations-controller") - monitor := reservations.NewMonitor(cachingClient) + monitor := reservations.NewMonitor(multiclusterClient) metrics.Registry.MustRegister(&monitor) if err := (&commitments.CommitmentReservationController{ - Client: cachingClient, + Client: multiclusterClient, Scheme: mgr.GetScheme(), Conf: commitmentsConfig.ReservationController, }).SetupWithManager(mgr, multiclusterClient); err != nil { @@ -654,11 +637,11 @@ func main() { crControllerConf := commitmentsConfig.CommittedResourceController - crControllerMonitor := commitments.NewCRControllerMonitor(cachingClient) + crControllerMonitor := commitments.NewCRControllerMonitor(multiclusterClient) metrics.Registry.MustRegister(&crControllerMonitor) if err := (&commitments.CommittedResourceController{ - Client: cachingClient, + Client: multiclusterClient, Scheme: mgr.GetScheme(), Conf: crControllerConf, Monitor: &crControllerMonitor, @@ -676,7 +659,7 @@ func main() { usageReconcilerConf := commitmentsConfig.UsageReconciler usageReconcilerConf.ApplyDefaults() if err := (&commitments.UsageReconciler{ - Client: cachingClient, + Client: multiclusterClient, Conf: usageReconcilerConf, VMSource: commitmentsVMSource, Monitor: usageReconcilerMonitor, @@ -691,7 +674,7 @@ func main() { monitor := datasources.NewMonitor() metrics.Registry.MustRegister(&monitor) if err := (&openstack.OpenStackDatasourceReconciler{ - Client: cachingClient, + Client: multiclusterClient, Scheme: mgr.GetScheme(), Monitor: monitor, }).SetupWithManager(mgr, multiclusterClient); err != nil { @@ -699,7 +682,7 @@ func main() { os.Exit(1) } if err := (&prometheus.PrometheusDatasourceReconciler{ - Client: cachingClient, + Client: multiclusterClient, Scheme: mgr.GetScheme(), Monitor: monitor, }).SetupWithManager(mgr, multiclusterClient); err != nil { @@ -712,7 +695,7 @@ func main() { monitor := extractor.NewMonitor() metrics.Registry.MustRegister(&monitor) if err := (&extractor.KnowledgeReconciler{ - Client: cachingClient, + Client: multiclusterClient, Scheme: mgr.GetScheme(), Monitor: monitor, Conf: conf.GetConfigOrDie[extractor.KnowledgeReconcilerConfig](), @@ -721,7 +704,7 @@ func main() { os.Exit(1) } if err := (&extractor.TriggerReconciler{ - Client: cachingClient, + Client: multiclusterClient, Scheme: mgr.GetScheme(), Conf: conf.GetConfigOrDie[extractor.TriggerReconcilerConfig](), }).SetupWithManager(mgr, multiclusterClient); err != nil { @@ -733,7 +716,7 @@ func main() { setupLog.Info("enabling controller", "controller", "kpis-controller") kpisControllerConfig := conf.GetConfigOrDie[kpis.ControllerConfig]() if err := (&kpis.Controller{ - Client: cachingClient, + Client: multiclusterClient, Config: kpisControllerConfig, }).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "KPIController") @@ -765,7 +748,7 @@ func main() { if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { // Create PostgresReader from the configured Datasource CRD // This runs after the cache is started - postgresReader, err := external.NewPostgresReader(ctx, cachingClient, failoverConfig.DatasourceName) + postgresReader, err := external.NewPostgresReader(ctx, multiclusterClient, failoverConfig.DatasourceName) if err != nil { setupLog.Error(err, "unable to create postgres reader for failover controller", "datasourceName", failoverConfig.DatasourceName) @@ -781,7 +764,7 @@ func main() { // 1. Watch-based per-reservation reconciliation (acknowledgment, validation) // 2. Periodic bulk VM processing (creating/assigning reservations) failoverController := failover.NewFailoverReservationController( - cachingClient, + multiclusterClient, vmSource, failoverConfig, schedulerClient, @@ -824,12 +807,12 @@ func main() { os.Exit(1) } - capacityMonitor := capacity.NewMonitor(cachingClient) + capacityMonitor := capacity.NewMonitor(multiclusterClient) if err := metrics.Registry.Register(&capacityMonitor); err != nil { setupLog.Error(err, "failed to register capacity monitor metrics, continuing without metrics") } - if err := capacity.NewController(cachingClient, capacityConfig, commitmentsVMSource). + if err := capacity.NewController(multiclusterClient, capacityConfig, commitmentsVMSource). SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "capacity") os.Exit(1) @@ -861,7 +844,7 @@ func main() { // Defer initialization until the manager starts (cache must be ready for postgres reader) if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { // Create PostgresReader from the configured Datasource CRD - postgresReader, err := external.NewPostgresReader(ctx, cachingClient, datasourceName) + postgresReader, err := external.NewPostgresReader(ctx, multiclusterClient, datasourceName) if err != nil { setupLog.Error(err, "unable to create postgres reader for quota controller", "datasourceName", datasourceName) @@ -874,7 +857,7 @@ func main() { // Create the quota controller quotaController := quota.NewQuotaController( - cachingClient, + multiclusterClient, vmSource, quotaConfig, quotaMetrics, @@ -936,11 +919,11 @@ func main() { setupLog.Info("starting commitments syncer") syncerMonitor := commitments.NewSyncerMonitor() must.Succeed(metrics.Registry.Register(syncerMonitor)) - syncer := commitments.NewSyncer(cachingClient, syncerMonitor) + syncer := commitments.NewSyncer(multiclusterClient, syncerMonitor) syncerConfig := conf.GetConfigOrDie[commitments.SyncerConfig]() syncerConfig.FlavorGroupResourceConfig = commitmentsConfig.API.FlavorGroupResourceConfig if err := (&task.Runner{ - Client: cachingClient, + Client: multiclusterClient, Interval: syncerConfig.SyncInterval.Duration, Name: "commitments-sync-task", Run: func(ctx context.Context) error { return syncer.SyncReservations(ctx) }, @@ -954,11 +937,11 @@ func main() { setupLog.Info("starting nova history cleanup task") historyCleanupConfig := conf.GetConfigOrDie[nova.HistoryCleanupConfig]() if err := (&task.Runner{ - Client: cachingClient, + Client: multiclusterClient, Interval: time.Hour, Name: "nova-history-cleanup-task", Run: func(ctx context.Context) error { - return nova.HistoryCleanup(ctx, cachingClient, historyCleanupConfig) + return nova.HistoryCleanup(ctx, multiclusterClient, historyCleanupConfig) }, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to add nova history cleanup task to manager") @@ -969,11 +952,11 @@ func main() { setupLog.Info("starting manila history cleanup task") historyCleanupConfig := conf.GetConfigOrDie[manila.HistoryCleanupConfig]() if err := (&task.Runner{ - Client: cachingClient, + Client: multiclusterClient, Interval: time.Hour, Name: "manila-history-cleanup-task", Run: func(ctx context.Context) error { - return manila.HistoryCleanup(ctx, cachingClient, historyCleanupConfig) + return manila.HistoryCleanup(ctx, multiclusterClient, historyCleanupConfig) }, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to add manila history cleanup task to manager") @@ -984,11 +967,11 @@ func main() { setupLog.Info("starting cinder history cleanup task") historyCleanupConfig := conf.GetConfigOrDie[cinder.HistoryCleanupConfig]() if err := (&task.Runner{ - Client: cachingClient, + Client: multiclusterClient, Interval: time.Hour, Name: "cinder-history-cleanup-task", Run: func(ctx context.Context) error { - return cinder.HistoryCleanup(ctx, cachingClient, historyCleanupConfig) + return cinder.HistoryCleanup(ctx, multiclusterClient, historyCleanupConfig) }, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to add cinder history cleanup task to manager") diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index fb5387864..fd644a352 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -110,7 +110,8 @@ cortex: &cortex - kvm.cloud.sap/v1/Hypervisor - kvm.cloud.sap/v1/HypervisorList - v1/Secret - clientcache: + pendingcache: + enabled: true gvks: - cortex.cloud/v1alpha1/Reservation keystoneSecretRef: diff --git a/internal/shim/placement/field_index_test.go b/internal/shim/placement/field_index_test.go index 1f74ad9a3..a81ec2879 100644 --- a/internal/shim/placement/field_index_test.go +++ b/internal/shim/placement/field_index_test.go @@ -50,6 +50,10 @@ type stubCluster struct { func (s *stubCluster) GetClient() client.Client { return s.cl } func (s *stubCluster) GetCache() cache.Cache { return s.cache } +// GetFieldIndexer returns the cache, which implements client.FieldIndexer via +// its IndexField method — mirroring an unwrapped controller-runtime cluster. +func (s *stubCluster) GetFieldIndexer() client.FieldIndexer { return s.cache } + type stubManager struct { manager.Manager } diff --git a/pkg/clientcache/interfaces.go b/pkg/clientcache/interfaces.go deleted file mode 100644 index c2c3e6dd5..000000000 --- a/pkg/clientcache/interfaces.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package clientcache - -import ( - "context" - - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Client is the interface the CachingClient requires of its inner client. -// It extends client.Client with the informer access needed for overlay eviction. -// *multicluster.Client satisfies this interface. -type Client interface { - client.Client - // GetInformersForKind returns all informers serving the GVK of the given - // object. The cache attaches Add/Update event handlers to each informer to - // evict overlay entries once the real object appears in the informer cache. - GetInformersForKind(ctx context.Context, obj client.Object) ([]cache.Informer, error) -} diff --git a/pkg/multicluster/client.go b/pkg/multicluster/client.go index d5c74eeb0..abb83fe4e 100644 --- a/pkg/multicluster/client.go +++ b/pkg/multicluster/client.go @@ -18,9 +18,11 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/cobaltcore-dev/cortex/pkg/pendingcache" ) // A remote cluster with routing labels used to match resources to clusters. @@ -54,48 +56,11 @@ type Client struct { // GVKs explicitly configured for the home cluster. homeGVKs map[schema.GroupVersionKind]bool -} -type ClientConfig struct { - // Apiserver configuration mapping GVKs to home or remote clusters. - // Every GVK used through the multicluster client must be listed - // in either Home or Remotes. Unknown GVKs will cause an error. - APIServers APIServersConfig `json:"apiservers"` -} - -// APIServersConfig separates resources into home and remote clusters. -type APIServersConfig struct { - // Resources managed in the cluster where cortex is deployed. - Home HomeConfig `json:"home"` - // Resources managed in remote clusters. - Remotes []RemoteConfig `json:"remotes,omitempty"` -} - -// HomeConfig lists GVKs that are managed in the home cluster. -type HomeConfig struct { - // The resource GVKs formatted as "//". - GVKs []string `json:"gvks"` -} - -// RemoteConfig maps multiple GVKs to a remote kubernetes apiserver with -// routing labels. It is assumed that the remote apiserver accepts the -// serviceaccount tokens issued by the local cluster. -type RemoteConfig struct { - // The remote kubernetes apiserver url, e.g. "https://my-apiserver:6443". - Host string `json:"host"` - // The root CA certificate to verify the remote apiserver. - // Ignored if InsecureSkipTLSVerify is true. - CACert string `json:"caCert,omitempty"` - // InsecureSkipTLSVerify disables verification of the remote apiserver's - // TLS certificate. Use this for apiservers whose CA certificate rotates - // frequently and does not chain to a stable root. Mutually exclusive - // with CACert: when true, CACert is ignored. - InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty"` - // The resource GVKs this apiserver serves, formatted as "//". - GVKs []string `json:"gvks"` - // Labels used by ResourceRouters to match resources to this cluster - // for write operations (Create/Update/Delete/Patch). - Labels map[string]string `json:"labels,omitempty"` + // cacheConf configures the optional in-process overlay cache. When + // Enabled, each home/remote cluster is wrapped so its GetClient/ + // GetFieldIndexer route through a per-cluster overlay. Set in InitFromConf. + cacheConf pendingcache.Config } // Helper function to initialize a new multicluster client during service startup, @@ -103,6 +68,7 @@ type RemoteConfig struct { func (c *Client) InitFromConf(ctx context.Context, mgr ctrl.Manager, conf ClientConfig) error { log := ctrl.LoggerFrom(ctx) log.Info("initializing multicluster client with config", "config", conf) + c.cacheConf = conf.PendingCache // Map the formatted gvk from the config to the actual gvk object so that we // can look up the right cluster for a given API server override. gvksByConfStr := make(map[string]schema.GroupVersionKind) @@ -133,13 +99,34 @@ func (c *Client) InitFromConf(ctx context.Context, mgr ctrl.Manager, conf Client } resolvedGVKs = append(resolvedGVKs, gvk) } - cl, err := c.AddRemote(ctx, remote.Host, remote.CACert, remote.InsecureSkipTLSVerify, remote.Labels, resolvedGVKs...) + cl, overlay, err := c.AddRemote(ctx, remote.Host, remote.CACert, remote.InsecureSkipTLSVerify, remote.Labels, resolvedGVKs...) if err != nil { return err } + // Add the raw inner cluster so its informers/caches start (as today). if err := mgr.Add(cl); err != nil { return err } + // When caching is enabled, also add the overlay's lifecycle Runnable + // (eviction handlers + TTL cleanup). It does not re-Start the cluster. + if overlay != nil { + if err := mgr.Add(overlay); err != nil { + return err + } + } + } + // When caching is enabled, wrap the home cluster the same way. The manager + // already owns the home cluster's lifecycle, so we only add the overlay + // Runnable and must NOT re-Start the inner home cluster. + if c.cacheConf.Enabled && c.HomeCluster != nil { + wrapped, overlay, err := pendingcache.WrapCluster(c.HomeCluster, c.cacheConf) + if err != nil { + return err + } + c.HomeCluster = wrapped + if err := mgr.Add(overlay); err != nil { + return err + } } return nil } @@ -154,7 +141,13 @@ func (c *Client) InitFromConf(ctx context.Context, mgr ctrl.Manager, conf Client // This can be used when the remote cluster accepts the home cluster's service // account tokens. See the kubernetes documentation on structured auth to // learn more about jwt-based authentication across clusters. -func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSkipTLSVerify bool, labels map[string]string, gvks ...schema.GroupVersionKind) (cluster.Cluster, error) { +// AddRemote returns the raw inner cluster.Cluster (which the caller must add to +// the manager so its informers/caches start) and, when caching is enabled, the +// overlay's lifecycle Runnable (which the caller must also add to the manager). +// The overlay Runnable is nil when caching is disabled. The wrapped cluster (or +// the raw cluster when caching is disabled) is stored in remoteClusters so all +// routing goes through the per-cluster overlay. +func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSkipTLSVerify bool, labels map[string]string, gvks ...schema.GroupVersionKind) (cluster.Cluster, manager.Runnable, error) { log := ctrl.LoggerFrom(ctx) homeRestConfig := *c.HomeRestConfig restConfigCopy := homeRestConfig @@ -172,7 +165,20 @@ func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSki o.Logger = ctrl.LoggerFrom(ctx).WithValues("host", host) }) if err != nil { - return nil, err + return nil, nil, err + } + // stored is the cluster placed in remoteClusters and used for all routing. + // When caching is enabled it is the overlay-wrapped cluster; otherwise the + // raw cluster. overlay is the overlay's lifecycle Runnable (nil when off). + stored := cl + var overlay manager.Runnable + if c.cacheConf.Enabled { + wrapped, ov, werr := pendingcache.WrapCluster(cl, c.cacheConf) + if werr != nil { + return nil, nil, werr + } + stored = wrapped + overlay = ov } c.remoteClustersMu.Lock() defer c.remoteClustersMu.Unlock() @@ -182,11 +188,13 @@ func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSki for _, gvk := range gvks { log.Info("adding remote cluster for resource", "gvk", gvk, "host", host, "labels", labels, "insecureSkipTLSVerify", insecureSkipTLSVerify) c.remoteClusters[gvk] = append(c.remoteClusters[gvk], remoteCluster{ - cluster: cl, + cluster: stored, labels: labels, }) } - return cl, nil + // Return the raw inner cluster so the caller starts its informers/caches; + // the overlay Runnable (if any) is returned separately. + return cl, overlay, nil } // Get the gvk registered for the given resource in the home cluster's scheme. @@ -225,30 +233,6 @@ func (c *Client) ClustersForGVK(gvk schema.GroupVersionKind) ([]cluster.Cluster, return clusters, nil } -// GetInformersForKind returns the informers of all clusters serving the GVK of -// the given object. It is used by the in-process client cache to attach -// eviction event handlers. The GVK is resolved against the home scheme and must -// be explicitly configured in home or a remote cluster. -func (c *Client) GetInformersForKind(ctx context.Context, obj client.Object) ([]cache.Informer, error) { - gvk, err := c.GVKFromHomeScheme(obj) - if err != nil { - return nil, err - } - clusters, err := c.ClustersForGVK(gvk) - if err != nil { - return nil, err - } - informers := make([]cache.Informer, 0, len(clusters)) - for _, cl := range clusters { - inf, err := cl.GetCache().GetInformer(ctx, obj) - if err != nil { - return nil, err - } - informers = append(informers, inf) - } - return informers, nil -} - // clusterForWrite uses a ResourceRouter to determine which remote cluster // a resource should be written to based on the resource content and cluster labels. // @@ -879,7 +863,7 @@ func (c *Client) IndexField(ctx context.Context, obj client.Object, list client. continue } indexed[ch] = true - if err := ch.IndexField(ctx, obj, field, extractValue); err != nil { + if err := cl.GetFieldIndexer().IndexField(ctx, obj, field, extractValue); err != nil { log.Error(err, "failed to register field index for cluster — objects from this cluster will be absent from index queries; restart required to recover", "field", field) continue } @@ -894,7 +878,7 @@ func (c *Client) IndexField(ctx context.Context, obj client.Object, list client. continue } indexed[ch] = true - if err := ch.IndexField(ctx, obj, field, extractValue); err != nil { + if err := cl.GetFieldIndexer().IndexField(ctx, obj, field, extractValue); err != nil { log.Error(err, "failed to register field index for cluster — objects from this cluster will be absent from index queries; restart required to recover", "field", field) continue } diff --git a/pkg/multicluster/client_test.go b/pkg/multicluster/client_test.go index 41fd3a6b9..efbd538d4 100644 --- a/pkg/multicluster/client_test.go +++ b/pkg/multicluster/client_test.go @@ -26,6 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/pkg/pendingcache" ) // unversionedType is a type that is registered as unversioned in the scheme. @@ -82,16 +83,28 @@ type fakeCluster struct { fakeClient client.Client fakeCache *fakeCache fakeRecorder events.EventRecorder + scheme *runtime.Scheme } func (f *fakeCluster) GetClient() client.Client { return f.fakeClient } +func (f *fakeCluster) GetScheme() *runtime.Scheme { + return f.scheme +} + func (f *fakeCluster) GetCache() cache.Cache { return f.fakeCache } +// GetFieldIndexer returns the fake cache, which implements client.FieldIndexer +// via its IndexField method. This mirrors production, where an unwrapped +// cluster's field indexer is its cache. +func (f *fakeCluster) GetFieldIndexer() client.FieldIndexer { + return f.fakeCache +} + func (f *fakeCluster) GetEventRecorder(_ string) events.EventRecorder { if f.fakeRecorder != nil { return f.fakeRecorder @@ -107,6 +120,7 @@ func newFakeCluster(scheme *runtime.Scheme, objs ...client.Object) *fakeCluster return &fakeCluster{ fakeClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build(), fakeCache: &fakeCache{}, + scheme: scheme, } } @@ -115,6 +129,7 @@ func newFakeClusterWithCache(scheme *runtime.Scheme, fakeCache *fakeCache, objs return &fakeCluster{ fakeClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build(), fakeCache: fakeCache, + scheme: scheme, } } @@ -1960,3 +1975,56 @@ func TestClient_ListMetadataPerCluster(t *testing.T) { } }) } + +// TestClient_CacheEnabled_WrappedHomeOverlay verifies that when a home cluster +// is wrapped by the pendingcache overlay, a write immediately followed by a read +// of a cached GVK is served from the overlay (before any informer catches up), +// and that ListMetadataPerCluster passes through the overlay unchanged (its +// PartialObjectMetadata queries do not resolve a cached GVK). +func TestClient_CacheEnabled_WrappedHomeOverlay(t *testing.T) { + scheme := newTestScheme(t) + inner := newFakeCluster(scheme) + + cacheConf := pendingcache.Config{ + Enabled: true, + GVKs: []string{"v1/ConfigMap"}, + } + wrapped, runnable, err := pendingcache.WrapCluster(inner, cacheConf) + if err != nil { + t.Fatalf("pendingcache.WrapCluster: %v", err) + } + if runnable == nil { + t.Fatalf("expected a non-nil overlay Runnable") + } + + c := &Client{ + HomeCluster: wrapped, + HomeScheme: scheme, + homeGVKs: map[schema.GroupVersionKind]bool{configMapGVK: true}, + } + + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-1", Namespace: "default"}} + if err := c.Create(context.Background(), cm); err != nil { + t.Fatalf("Create: %v", err) + } + + // Immediately read back through the multicluster client — the overlay must + // serve it even though the fake cache/informer never observed it. + var got corev1.ConfigMap + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: "cm-1"}, &got); err != nil { + t.Fatalf("Get after create: %v", err) + } + if got.Name != "cm-1" { + t.Fatalf("expected cm-1 from overlay, got %q", got.Name) + } + + // ListMetadataPerCluster uses PartialObjectMetadataList → the overlay passes + // through. It must still return the underlying cluster's metadata result. + results, err := c.ListMetadataPerCluster(context.Background(), configMapGVK) + if err != nil { + t.Fatalf("ListMetadataPerCluster: %v", err) + } + if len(results) != 1 || !results[0].IsHome { + t.Fatalf("expected one home result, got %+v", results) + } +} diff --git a/pkg/multicluster/config.go b/pkg/multicluster/config.go new file mode 100644 index 000000000..ed73af4e4 --- /dev/null +++ b/pkg/multicluster/config.go @@ -0,0 +1,52 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package multicluster + +import "github.com/cobaltcore-dev/cortex/pkg/pendingcache" + +type ClientConfig struct { + // Apiserver configuration mapping GVKs to home or remote clusters. + // Every GVK used through the multicluster client must be listed + // in either Home or Remotes. Unknown GVKs will cause an error. + APIServers APIServersConfig `json:"apiservers"` + + // PendingCache configures the optional transparent in-process overlay cache. When + // PendingCache.Enabled is false (the default), clusters are used unwrapped. + PendingCache pendingcache.Config `json:"pendingcache"` +} + +// APIServersConfig separates resources into home and remote clusters. +type APIServersConfig struct { + // Resources managed in the cluster where cortex is deployed. + Home HomeConfig `json:"home"` + // Resources managed in remote clusters. + Remotes []RemoteConfig `json:"remotes,omitempty"` +} + +// HomeConfig lists GVKs that are managed in the home cluster. +type HomeConfig struct { + // The resource GVKs formatted as "//". + GVKs []string `json:"gvks"` +} + +// RemoteConfig maps multiple GVKs to a remote kubernetes apiserver with +// routing labels. It is assumed that the remote apiserver accepts the +// serviceaccount tokens issued by the local cluster. +type RemoteConfig struct { + // The remote kubernetes apiserver url, e.g. "https://my-apiserver:6443". + Host string `json:"host"` + // The root CA certificate to verify the remote apiserver. + // Ignored if InsecureSkipTLSVerify is true. + CACert string `json:"caCert,omitempty"` + // InsecureSkipTLSVerify disables verification of the remote apiserver's + // TLS certificate. Use this for apiservers whose CA certificate rotates + // frequently and does not chain to a stable root. Mutually exclusive + // with CACert: when true, CACert is ignored. + InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty"` + // The resource GVKs this apiserver serves, formatted as "//". + GVKs []string `json:"gvks"` + // Labels used by ResourceRouters to match resources to this cluster + // for write operations (Create/Update/Delete/Patch). + Labels map[string]string `json:"labels,omitempty"` +} diff --git a/pkg/clientcache/client.go b/pkg/pendingcache/client.go similarity index 82% rename from pkg/clientcache/client.go rename to pkg/pendingcache/client.go index c80fde968..1d68a600d 100644 --- a/pkg/clientcache/client.go +++ b/pkg/pendingcache/client.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package clientcache +package pendingcache import ( "context" @@ -17,7 +17,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/manager" ) // entry is a single overlaid object with the bookkeeping needed for eviction. @@ -136,7 +139,7 @@ func resourceVersionAtLeast(observed, cached string) bool { // defaultTTL is used when Config.TTL is zero. const defaultTTL = 2 * time.Minute -// CachingClient wraps an inner client.Client with a transparent in-process +// Overlay wraps an inner client.Client with a transparent in-process // overlay. Writes populate the overlay; reads merge the overlay with the inner // (informer-backed) result; entries are evicted once the real object appears in // an informer (see runnable.go) or after TTL expiry. @@ -149,13 +152,13 @@ const defaultTTL = 2 * time.Minute // the informer) or a tombstone (a Delete that has not yet propagated). Reads // merge the inner result with these entries: tombstones suppress objects; // live entries override or supplement the inner result. -type CachingClient struct { +type Overlay struct { client.Client // inner client, used for delegation - inner Client - scheme *runtime.Scheme - ttl time.Duration - gvks map[schema.GroupVersionKind]bool + informerCache cache.Cache // used by Start to attach eviction event handlers + scheme *runtime.Scheme + ttl time.Duration + gvks map[schema.GroupVersionKind]bool mu sync.RWMutex byGVK map[schema.GroupVersionKind]map[objectKey]*entry @@ -167,29 +170,40 @@ type CachingClient struct { writeLocks *keyedMutex } -// New builds a CachingClient wrapping inner. informers supplies the informers -// used for eviction, scheme resolves object GVKs, and conf lists the GVKs to -// overlay and the TTL. GVK strings are formatted as "//" -// and are resolved against scheme. -func New(inner Client, scheme *runtime.Scheme, conf Config) (*CachingClient, error) { +// WrapCluster wraps inner with a transparent in-process overlay cache and +// returns the wrapped cluster.Cluster together with the overlay's lifecycle +// Runnable. +// +// The returned cluster.Cluster delegates everything to inner except GetClient +// and GetFieldIndexer, which return the overlay client so per-cluster reads and +// writes flow through the cache. GVK strings in conf are formatted as +// "//" and resolved against inner.GetScheme(). +// +// The returned manager.Runnable is the overlay itself: it attaches informer +// eviction handlers (reading informers from inner.GetCache()) and runs the TTL +// cleanup loop. The caller MUST add it to the manager. It does NOT re-Start the +// inner cluster — the manager owns the inner cluster's lifecycle separately. +func WrapCluster(inner cluster.Cluster, conf Config) (cluster.Cluster, manager.Runnable, error) { + scheme := inner.GetScheme() gvks, err := resolveGVKs(scheme, conf.GVKs) if err != nil { - return nil, err + return nil, nil, err } ttl := conf.TTL.Duration if ttl <= 0 { ttl = defaultTTL } - return &CachingClient{ - Client: inner, - inner: inner, - scheme: scheme, - ttl: ttl, - gvks: gvks, - byGVK: make(map[schema.GroupVersionKind]map[objectKey]*entry), - indexers: make(map[schema.GroupVersionKind]map[string]client.IndexerFunc), - writeLocks: newKeyedMutex(), - }, nil + cc := &Overlay{ + Client: inner.GetClient(), + informerCache: inner.GetCache(), + scheme: scheme, + ttl: ttl, + gvks: gvks, + byGVK: make(map[schema.GroupVersionKind]map[objectKey]*entry), + indexers: make(map[schema.GroupVersionKind]map[string]client.IndexerFunc), + writeLocks: newKeyedMutex(), + } + return &overlayCluster{Cluster: inner, cache: cc}, cc, nil } // resolveGVKs maps "//" strings to GVKs via the scheme's @@ -204,7 +218,7 @@ func resolveGVKs(scheme *runtime.Scheme, gvkStrs []string) (map[schema.GroupVers for _, s := range gvkStrs { gvk, ok := byStr[s] if !ok { - return nil, errors.New("clientcache: no gvk registered in scheme for " + s) + return nil, errors.New("pendingcache: no gvk registered in scheme for " + s) } out[gvk] = true } @@ -212,7 +226,7 @@ func resolveGVKs(scheme *runtime.Scheme, gvkStrs []string) (map[schema.GroupVers } // gvkFor resolves the GVK of obj and reports whether it is cached. -func (c *CachingClient) gvkFor(obj runtime.Object) (schema.GroupVersionKind, bool) { +func (c *Overlay) gvkFor(obj runtime.Object) (schema.GroupVersionKind, bool) { gvks, _, err := c.scheme.ObjectKinds(obj) if err != nil || len(gvks) != 1 { return schema.GroupVersionKind{}, false @@ -223,7 +237,7 @@ func (c *CachingClient) gvkFor(obj runtime.Object) (schema.GroupVersionKind, boo // itemGVKForList resolves the singular item GVK for a list object and reports // whether that item GVK is cached. The list GVK's Kind ends with "List". -func (c *CachingClient) itemGVKForList(list client.ObjectList) (schema.GroupVersionKind, bool) { +func (c *Overlay) itemGVKForList(list client.ObjectList) (schema.GroupVersionKind, bool) { gvks, _, err := c.scheme.ObjectKinds(list) if err != nil || len(gvks) != 1 { return schema.GroupVersionKind{}, false @@ -245,7 +259,7 @@ func trimListSuffix(kind string) (string, bool) { } // upsert stores a live (non-tombstone) entry for the object. -func (c *CachingClient) upsert(gvk schema.GroupVersionKind, obj client.Object) { +func (c *Overlay) upsert(gvk schema.GroupVersionKind, obj client.Object) { c.mu.Lock() defer c.mu.Unlock() c.ensureGVK(gvk) @@ -260,7 +274,7 @@ func (c *CachingClient) upsert(gvk schema.GroupVersionKind, obj client.Object) { // tombstone marks the object as deleted in the overlay so it is filtered out // of reads until the deletion is observed in an informer. -func (c *CachingClient) tombstone(gvk schema.GroupVersionKind, obj client.Object) { +func (c *Overlay) tombstone(gvk schema.GroupVersionKind, obj client.Object) { c.mu.Lock() defer c.mu.Unlock() c.ensureGVK(gvk) @@ -275,7 +289,7 @@ func (c *CachingClient) tombstone(gvk schema.GroupVersionKind, obj client.Object // evictIfSeen removes the overlay entry for obj if the informer-observed object // matches by UID and its ResourceVersion is at least as new as the cached one. -func (c *CachingClient) evictIfSeen(gvk schema.GroupVersionKind, obj client.Object) { +func (c *Overlay) evictIfSeen(gvk schema.GroupVersionKind, obj client.Object) { c.mu.Lock() defer c.mu.Unlock() entries, ok := c.byGVK[gvk] @@ -300,7 +314,7 @@ func (c *CachingClient) evictIfSeen(gvk schema.GroupVersionKind, obj client.Obje } // getEntry returns the overlay entry for the key, if present. -func (c *CachingClient) getEntry(gvk schema.GroupVersionKind, key objectKey) (*entry, bool) { +func (c *Overlay) getEntry(gvk schema.GroupVersionKind, key objectKey) (*entry, bool) { c.mu.RLock() defer c.mu.RUnlock() entries, ok := c.byGVK[gvk] @@ -312,7 +326,7 @@ func (c *CachingClient) getEntry(gvk schema.GroupVersionKind, key objectKey) (*e } // cleanupExpired removes entries whose TTL has passed. -func (c *CachingClient) cleanupExpired(now time.Time) { +func (c *Overlay) cleanupExpired(now time.Time) { c.mu.Lock() defer c.mu.Unlock() for _, entries := range c.byGVK { @@ -326,7 +340,7 @@ func (c *CachingClient) cleanupExpired(now time.Time) { // registerIndex captures an IndexerFunc for a field so overlay entries can be // matched against MatchingFields queries. -func (c *CachingClient) registerIndex(gvk schema.GroupVersionKind, field string, fn client.IndexerFunc) { +func (c *Overlay) registerIndex(gvk schema.GroupVersionKind, field string, fn client.IndexerFunc) { c.mu.Lock() defer c.mu.Unlock() if c.indexers[gvk] == nil { @@ -336,7 +350,7 @@ func (c *CachingClient) registerIndex(gvk schema.GroupVersionKind, field string, } // ensureGVK initialises the per-GVK entry map if absent. Callers must hold the write lock. -func (c *CachingClient) ensureGVK(gvk schema.GroupVersionKind) { +func (c *Overlay) ensureGVK(gvk schema.GroupVersionKind) { if c.byGVK[gvk] == nil { c.byGVK[gvk] = make(map[objectKey]*entry) } @@ -345,7 +359,7 @@ func (c *CachingClient) ensureGVK(gvk schema.GroupVersionKind) { // overlayList merges the overlay entries for the GVK into the informer result, // deduplicating by objectKey (overlay wins), dropping tombstones, and filtering // overlay-only entries against the list options' label and field selectors. -func (c *CachingClient) overlayList(gvk schema.GroupVersionKind, existing []runtime.Object, lo *client.ListOptions) []runtime.Object { +func (c *Overlay) overlayList(gvk schema.GroupVersionKind, existing []runtime.Object, lo *client.ListOptions) []runtime.Object { c.mu.RLock() defer c.mu.RUnlock() entries := c.byGVK[gvk] @@ -403,7 +417,7 @@ func (c *CachingClient) overlayList(gvk schema.GroupVersionKind, existing []runt // matchesLocked reports whether obj satisfies the list options' namespace, // label and field selectors. Callers must hold at least the read lock. -func (c *CachingClient) matchesLocked(gvk schema.GroupVersionKind, obj client.Object, lo *client.ListOptions) bool { +func (c *Overlay) matchesLocked(gvk schema.GroupVersionKind, obj client.Object, lo *client.ListOptions) bool { if lo == nil { return true } @@ -424,7 +438,7 @@ func (c *CachingClient) matchesLocked(gvk schema.GroupVersionKind, obj client.Ob // fieldSetLocked builds a fields.Set for obj using the registered IndexerFuncs // for the GVK. Callers must hold at least the read lock. -func (c *CachingClient) fieldSetLocked(gvk schema.GroupVersionKind, obj client.Object) fields.Set { +func (c *Overlay) fieldSetLocked(gvk schema.GroupVersionKind, obj client.Object) fields.Set { set := fields.Set{} for field, fn := range c.indexers[gvk] { for _, v := range fn(obj) { @@ -439,7 +453,7 @@ func (c *CachingClient) fieldSetLocked(gvk schema.GroupVersionKind, obj client.O // Create delegates to the inner client and, on success for a cached GVK, adds // the object to the overlay. -func (c *CachingClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { +func (c *Overlay) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { gvk, cached := c.gvkFor(obj) if !cached { return c.Client.Create(ctx, obj, opts...) @@ -455,7 +469,7 @@ func (c *CachingClient) Create(ctx context.Context, obj client.Object, opts ...c // Update delegates to the inner client and, on success for a cached GVK, // refreshes the overlay entry. -func (c *CachingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { +func (c *Overlay) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { gvk, cached := c.gvkFor(obj) if !cached { return c.Client.Update(ctx, obj, opts...) @@ -471,7 +485,7 @@ func (c *CachingClient) Update(ctx context.Context, obj client.Object, opts ...c // Patch delegates to the inner client and, on success for a cached GVK, // refreshes the overlay entry with the patched object. -func (c *CachingClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { +func (c *Overlay) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { gvk, cached := c.gvkFor(obj) if !cached { return c.Client.Patch(ctx, obj, patch, opts...) @@ -491,7 +505,7 @@ func (c *CachingClient) Patch(ctx context.Context, obj client.Object, patch clie // through the informer (which triggers eviction) or the TTL expires. This // ensures that reads between the Delete call and the informer event correctly // return NotFound rather than serving a stale object from the informer cache. -func (c *CachingClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { +func (c *Overlay) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { gvk, cached := c.gvkFor(obj) if !cached { return c.Client.Delete(ctx, obj, opts...) @@ -509,7 +523,7 @@ func (c *CachingClient) Delete(ctx context.Context, obj client.Object, opts ...c // tombstones all overlay entries that match the delete options. Objects that // live only in the informer cache (not in the overlay) will be evicted // naturally once the deletion propagates through the informer. -func (c *CachingClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { +func (c *Overlay) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { gvk, cached := c.gvkFor(obj) if !cached { return c.Client.DeleteAllOf(ctx, obj, opts...) @@ -539,7 +553,7 @@ func (c *CachingClient) DeleteAllOf(ctx context.Context, obj client.Object, opts // Get delegates to the inner client, then applies the overlay: a tombstone // yields NotFound; a live overlay entry overrides the inner result; and an // overlay entry can satisfy a Get that the inner client reports as NotFound. -func (c *CachingClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { +func (c *Overlay) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { gvk, cached := c.gvkFor(obj) if !cached { return c.Client.Get(ctx, key, obj, opts...) @@ -567,7 +581,7 @@ func (c *CachingClient) Get(ctx context.Context, key client.ObjectKey, obj clien } // List delegates to the inner client, then merges the overlay into the result. -func (c *CachingClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { +func (c *Overlay) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { itemGVK, cached := c.itemGVKForList(list) if !cached { return c.Client.List(ctx, list, opts...) @@ -588,7 +602,7 @@ func (c *CachingClient) List(ctx context.Context, list client.ObjectList, opts . // IndexField delegates to the inner client (if it is a FieldIndexer) and also // registers the IndexerFunc with the overlay so overlay entries can be matched // against MatchingFields. -func (c *CachingClient) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { +func (c *Overlay) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { if indexer, ok := c.Client.(client.FieldIndexer); ok { if err := indexer.IndexField(ctx, obj, field, extractValue); err != nil { return err @@ -602,14 +616,14 @@ func (c *CachingClient) IndexField(ctx context.Context, obj client.Object, field // Status returns a status writer that mirrors status Update/Patch writes for // cached GVKs into the overlay. -func (c *CachingClient) Status() client.StatusWriter { +func (c *Overlay) Status() client.StatusWriter { return &statusWriter{c: c, inner: c.Client.Status()} } // statusWriter wraps the inner status writer and reflects status writes into // the overlay for cached GVKs. type statusWriter struct { - c *CachingClient + c *Overlay inner client.StatusWriter } diff --git a/pkg/clientcache/client_test.go b/pkg/pendingcache/client_test.go similarity index 79% rename from pkg/clientcache/client_test.go rename to pkg/pendingcache/client_test.go index 75aa3db9e..fc6716449 100644 --- a/pkg/clientcache/client_test.go +++ b/pkg/pendingcache/client_test.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package clientcache +package pendingcache import ( "context" @@ -21,6 +21,8 @@ import ( ccache "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) @@ -85,41 +87,68 @@ func (f *fakeInformer) fireUpdate(oldObj, newObj any) { } } -// fakeClient composes a fake client.Client with a fakeInformer to satisfy the -// clientcache.Client interface. -type fakeClient struct { - client.Client +// fakeCache is a minimal cache.Cache that returns a single fakeInformer for any +// GetInformer call, so the Overlay runnable can wire eviction handlers. +type fakeCache struct { + ccache.Cache inf *fakeInformer } -func (f *fakeClient) GetInformersForKind(_ context.Context, _ client.Object) ([]ccache.Informer, error) { - return []ccache.Informer{f.inf}, nil +func (f *fakeCache) GetInformer(_ context.Context, _ client.Object, _ ...ccache.InformerGetOption) (ccache.Informer, error) { + return f.inf, nil +} + +// fakeCluster composes a fake client.Client with a fakeCache to satisfy the +// cluster.Cluster interface consumed by pendingcache.WrapCluster. +type fakeCluster struct { + cluster.Cluster + client client.Client + cache *fakeCache + scheme *runtime.Scheme +} + +func (f *fakeCluster) GetClient() client.Client { return f.client } +func (f *fakeCluster) GetCache() ccache.Cache { return f.cache } +func (f *fakeCluster) GetScheme() *runtime.Scheme { return f.scheme } + +// fakeClient exposes the underlying informer so eviction tests can fire events, +// while itself acting as a cluster.Cluster wrapping the fake client.Client. +type fakeClient struct { + *fakeCluster + inf *fakeInformer } func newTestClient(t *testing.T, objs ...client.Object) *fakeClient { t.Helper() + scheme := testScheme(t) + inf := &fakeInformer{} + inner := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objs...). + WithStatusSubresource(&v1alpha1.Reservation{}). + WithIndex(&v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok || res.Spec.AvailabilityZone == "" { + return nil + } + return []string{res.Spec.AvailabilityZone} + }). + Build() return &fakeClient{ - Client: fake.NewClientBuilder(). - WithScheme(testScheme(t)). - WithObjects(objs...). - WithStatusSubresource(&v1alpha1.Reservation{}). - WithIndex(&v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { - res, ok := obj.(*v1alpha1.Reservation) - if !ok || res.Spec.AvailabilityZone == "" { - return nil - } - return []string{res.Spec.AvailabilityZone} - }). - Build(), - inf: &fakeInformer{}, + fakeCluster: &fakeCluster{ + client: inner, + cache: &fakeCache{inf: inf}, + scheme: scheme, + }, + inf: inf, } } -// errClient wraps a Client and injects configurable errors into mutating/read -// operations, so the error-propagation paths of CachingClient (which must not -// touch the overlay on failure) can be exercised. +// errClient wraps a client.Client and injects configurable errors into +// mutating/read operations, so the error-propagation paths of Overlay +// (which must not touch the overlay on failure) can be exercised. type errClient struct { - Client + client.Client createErr error updateErr error patchErr error @@ -222,13 +251,43 @@ func reservationConfig() Config { } } -func newCaching(t *testing.T, inner Client) *CachingClient { +// clusterFor wraps a client.Client (and the given informer) as a cluster.Cluster +// so it can be passed to New. Used by tests that inject a custom client.Client +// (e.g. errClient, orderingClient) below the overlay. +func clusterFor(t *testing.T, inner client.Client, inf *fakeInformer) cluster.Cluster { t.Helper() - c, err := New(inner, testScheme(t), reservationConfig()) + if inf == nil { + inf = &fakeInformer{} + } + return &fakeCluster{ + client: inner, + cache: &fakeCache{inf: inf}, + scheme: testScheme(t), + } +} + +// cachingFrom builds a *Overlay over the given cluster.Cluster. +func cachingFrom(t *testing.T, cl cluster.Cluster, conf Config) *Overlay { + t.Helper() + wrapped, runnable, err := WrapCluster(cl, conf) if err != nil { t.Fatalf("New: %v", err) } - return c + cc, ok := wrapped.GetClient().(*Overlay) + if !ok { + t.Fatalf("WrapCluster did not return an Overlay") + } + if _, ok := runnable.(*Overlay); !ok { + t.Fatalf("WrapCluster did not return a *Overlay runnable") + } + return cc +} + +// newCaching builds a *Overlay over the given fake cluster using the +// default reservation config. +func newCaching(t *testing.T, inner *fakeClient) *Overlay { + t.Helper() + return cachingFrom(t, inner.fakeCluster, reservationConfig()) } func listReservations(t *testing.T, c client.Client, opts ...client.ListOption) []v1alpha1.Reservation { @@ -259,7 +318,7 @@ func waitFor(t *testing.T, cond func() bool) { // --- constructor tests --- func TestNewUnknownGVKError(t *testing.T) { - _, err := New(newTestClient(t), testScheme(t), Config{ + _, _, err := WrapCluster(newTestClient(t).fakeCluster, Config{ GVKs: []string{"cortex.cloud/v1alpha1/DoesNotExist"}, }) if err == nil { @@ -268,27 +327,49 @@ func TestNewUnknownGVKError(t *testing.T) { } func TestNewDefaultTTL(t *testing.T) { - c, err := New(newTestClient(t), testScheme(t), Config{ + c := cachingFrom(t, newTestClient(t).fakeCluster, Config{ GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, }) - if err != nil { - t.Fatalf("New: %v", err) - } if c.ttl != defaultTTL { t.Fatalf("expected ttl %v, got %v", defaultTTL, c.ttl) } } func TestNewExplicitTTL(t *testing.T) { - c, err := New(newTestClient(t), testScheme(t), Config{ + c := cachingFrom(t, newTestClient(t).fakeCluster, Config{ GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, TTL: metav1.Duration{Duration: 90 * time.Second}, }) + if c.ttl != 90*time.Second { + t.Fatalf("expected ttl 90s, got %v", c.ttl) + } +} + +func TestNewWrapsClusterClientAndIndexer(t *testing.T) { + inner := newTestClient(t) + wrapped, runnable, err := WrapCluster(inner.fakeCluster, reservationConfig()) if err != nil { t.Fatalf("New: %v", err) } - if c.ttl != 90*time.Second { - t.Fatalf("expected ttl 90s, got %v", c.ttl) + // GetClient and GetFieldIndexer must both return the overlay so per-cluster + // reads/writes and IndexField flow through the cache. + cc, ok := wrapped.GetClient().(*Overlay) + if !ok { + t.Fatalf("GetClient did not return the overlay") + } + if wrapped.GetFieldIndexer() != client.FieldIndexer(cc) { + t.Fatalf("GetFieldIndexer did not return the overlay") + } + // The returned Runnable is the same overlay. + if runnable != manager.Runnable(cc) { + t.Fatalf("returned Runnable is not the overlay") + } + // Everything else delegates to the inner cluster. + if wrapped.GetScheme() != inner.GetScheme() { + t.Fatalf("GetScheme did not delegate to inner") + } + if wrapped.GetCache() != inner.GetCache() { + t.Fatalf("GetCache did not delegate to inner") } } @@ -422,7 +503,7 @@ func TestTombstone(t *testing.T) { t.Fatalf("Delete: %v", err) } // Re-create directly in inner to simulate informer lag. - if err := inner.Create(context.Background(), newReservation("res-5", "az-1", "")); err != nil { + if err := inner.GetClient().Create(context.Background(), newReservation("res-5", "az-1", "")); err != nil { t.Fatalf("re-create inner: %v", err) } @@ -467,7 +548,7 @@ func TestUpdateOverridesInner(t *testing.T) { c := newCaching(t, inner) var cur v1alpha1.Reservation - if err := inner.Get(context.Background(), types.NamespacedName{Name: "res-6"}, &cur); err != nil { + if err := inner.GetClient().Get(context.Background(), types.NamespacedName{Name: "res-6"}, &cur); err != nil { t.Fatalf("inner get: %v", err) } cur.Spec.AvailabilityZone = "az-new" @@ -548,10 +629,7 @@ func TestFieldMatching(t *testing.T) { func TestNonCachedGVKPassthrough(t *testing.T) { inner := newTestClient(t) - c, err := New(inner, testScheme(t), Config{}) - if err != nil { - t.Fatalf("New: %v", err) - } + c := cachingFrom(t, inner.fakeCluster, Config{}) r := newReservation("res-9", "az-1", "") if err := c.Create(context.Background(), r); err != nil { t.Fatalf("Create: %v", err) @@ -593,35 +671,43 @@ func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { name string rv string seed bool - mkClient func(inner *fakeClient) Client - op func(c *CachingClient, r *v1alpha1.Reservation) error + mkClient func(inner *fakeClient) client.Client + op func(c *Overlay, r *v1alpha1.Reservation) error }{ { - name: "create", - mkClient: func(inner *fakeClient) Client { return &errClient{Client: inner, createErr: sentinel} }, - op: func(c *CachingClient, r *v1alpha1.Reservation) error { return c.Create(context.Background(), r) }, + name: "create", + mkClient: func(inner *fakeClient) client.Client { + return &errClient{Client: inner.GetClient(), createErr: sentinel} + }, + op: func(c *Overlay, r *v1alpha1.Reservation) error { return c.Create(context.Background(), r) }, }, { - name: "update", - rv: "1", - mkClient: func(inner *fakeClient) Client { return &errClient{Client: inner, updateErr: sentinel} }, - op: func(c *CachingClient, r *v1alpha1.Reservation) error { return c.Update(context.Background(), r) }, + name: "update", + rv: "1", + mkClient: func(inner *fakeClient) client.Client { + return &errClient{Client: inner.GetClient(), updateErr: sentinel} + }, + op: func(c *Overlay, r *v1alpha1.Reservation) error { return c.Update(context.Background(), r) }, }, { - name: "patch", - rv: "1", - mkClient: func(inner *fakeClient) Client { return &errClient{Client: inner, patchErr: sentinel} }, - op: func(c *CachingClient, r *v1alpha1.Reservation) error { + name: "patch", + rv: "1", + mkClient: func(inner *fakeClient) client.Client { + return &errClient{Client: inner.GetClient(), patchErr: sentinel} + }, + op: func(c *Overlay, r *v1alpha1.Reservation) error { p := r.DeepCopy() p.Spec.AvailabilityZone = "az-new" return c.Patch(context.Background(), p, client.MergeFrom(r)) }, }, { - name: "delete", - seed: true, - mkClient: func(inner *fakeClient) Client { return &errClient{Client: inner, deleteErr: sentinel} }, - op: func(c *CachingClient, r *v1alpha1.Reservation) error { return c.Delete(context.Background(), r) }, + name: "delete", + seed: true, + mkClient: func(inner *fakeClient) client.Client { + return &errClient{Client: inner.GetClient(), deleteErr: sentinel} + }, + op: func(c *Overlay, r *v1alpha1.Reservation) error { return c.Delete(context.Background(), r) }, }, } for _, tc := range cases { @@ -633,7 +719,7 @@ func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { } else { base = newTestClient(t) } - c := newCaching(t, tc.mkClient(base)) + c := cachingFrom(t, clusterFor(t, tc.mkClient(base), nil), reservationConfig()) if err := tc.op(c, r); !errors.Is(err, sentinel) { t.Fatalf("expected sentinel error, got %v", err) @@ -648,13 +734,13 @@ func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { func TestWriteServedFromOverlay(t *testing.T) { cases := []struct { name string - write func(t *testing.T, c *CachingClient, cur *v1alpha1.Reservation) string + write func(t *testing.T, c *Overlay, cur *v1alpha1.Reservation) string diverge func(t *testing.T, inner client.Client, name string) read func(*v1alpha1.Reservation) string }{ { name: "patch spec", - write: func(t *testing.T, c *CachingClient, cur *v1alpha1.Reservation) string { + write: func(t *testing.T, c *Overlay, cur *v1alpha1.Reservation) string { base := cur.DeepCopy() cur.Spec.AvailabilityZone = "az-new" if err := c.Patch(context.Background(), cur, client.MergeFrom(base)); err != nil { @@ -667,7 +753,7 @@ func TestWriteServedFromOverlay(t *testing.T) { }, { name: "status update", - write: func(t *testing.T, c *CachingClient, cur *v1alpha1.Reservation) string { + write: func(t *testing.T, c *Overlay, cur *v1alpha1.Reservation) string { cur.Status.Host = "host-active" if err := c.Status().Update(context.Background(), cur); err != nil { t.Fatalf("Status().Update: %v", err) @@ -681,7 +767,7 @@ func TestWriteServedFromOverlay(t *testing.T) { }, { name: "status patch", - write: func(t *testing.T, c *CachingClient, cur *v1alpha1.Reservation) string { + write: func(t *testing.T, c *Overlay, cur *v1alpha1.Reservation) string { base := cur.DeepCopy() cur.Status.Host = "host-patched" if err := c.Status().Patch(context.Background(), cur, client.MergeFrom(base)); err != nil { @@ -702,11 +788,11 @@ func TestWriteServedFromOverlay(t *testing.T) { c := newCaching(t, inner) var cur v1alpha1.Reservation - if err := inner.Get(context.Background(), types.NamespacedName{Name: r.Name}, &cur); err != nil { + if err := inner.GetClient().Get(context.Background(), types.NamespacedName{Name: r.Name}, &cur); err != nil { t.Fatalf("inner get: %v", err) } want := tc.write(t, c, &cur) - tc.diverge(t, inner, r.Name) + tc.diverge(t, inner.GetClient(), r.Name) var got v1alpha1.Reservation if err := c.Get(context.Background(), types.NamespacedName{Name: r.Name}, &got); err != nil { @@ -721,7 +807,7 @@ func TestWriteServedFromOverlay(t *testing.T) { func TestGetPropagatesNonNotFoundError(t *testing.T) { sentinel := errors.New("get boom") - c := newCaching(t, &errClient{Client: newTestClient(t), getErr: sentinel}) + c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), getErr: sentinel}, nil), reservationConfig()) c.upsert(reservationGVK(), newReservation("res-ge", "az-1", "1")) var got v1alpha1.Reservation @@ -754,10 +840,7 @@ func TestGetNotFoundWithNoOverlay(t *testing.T) { func TestGetNonCachedPropagatesError(t *testing.T) { sentinel := errors.New("get boom") - c, err := New(&errClient{Client: newTestClient(t), getErr: sentinel}, testScheme(t), Config{}) - if err != nil { - t.Fatalf("New: %v", err) - } + c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), getErr: sentinel}, nil), Config{}) var got v1alpha1.Reservation if gerr := c.Get(context.Background(), types.NamespacedName{Name: "x"}, &got); !errors.Is(gerr, sentinel) { t.Fatalf("expected sentinel error, got %v", gerr) @@ -766,7 +849,7 @@ func TestGetNonCachedPropagatesError(t *testing.T) { func TestListPropagatesError(t *testing.T) { sentinel := errors.New("list boom") - c := newCaching(t, &errClient{Client: newTestClient(t), listErr: sentinel}) + c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), listErr: sentinel}, nil), reservationConfig()) c.upsert(reservationGVK(), newReservation("res-le", "az-1", "1")) var list v1alpha1.ReservationList @@ -802,12 +885,9 @@ func TestStatusCreateDelegates(t *testing.T) { func TestStatusUpdateNonCachedNoOverlay(t *testing.T) { r := newReservation("res-sn", "az-1", "") inner := newTestClient(t, r) - c, err := New(inner, testScheme(t), Config{}) - if err != nil { - t.Fatalf("New: %v", err) - } + c := cachingFrom(t, inner.fakeCluster, Config{}) var cur v1alpha1.Reservation - if err := inner.Get(context.Background(), types.NamespacedName{Name: "res-sn"}, &cur); err != nil { + if err := inner.GetClient().Get(context.Background(), types.NamespacedName{Name: "res-sn"}, &cur); err != nil { t.Fatalf("inner get: %v", err) } cur.Status.Host = "host-active" @@ -832,7 +912,7 @@ func TestStatusUpdateNonCachedNoOverlay(t *testing.T) { // commit order and the overlay upsert order are identical: the overlay always // reflects the last write that reached the inner client (lastRV). type orderingClient struct { - Client + client.Client mu sync.Mutex lastRV string // ResourceVersion of the most recent inner commit } @@ -859,8 +939,8 @@ func TestConcurrentUpdatesOverlayNotBehind(t *testing.T) { n = 8 ) for round := range rounds { - oc := &orderingClient{Client: newTestClient(t)} - c := newCaching(t, oc) + oc := &orderingClient{Client: newTestClient(t).GetClient()} + c := cachingFrom(t, clusterFor(t, oc, nil), reservationConfig()) var wg sync.WaitGroup for i := 1; i <= n; i++ { diff --git a/pkg/pendingcache/cluster.go b/pkg/pendingcache/cluster.go new file mode 100644 index 000000000..a784d1879 --- /dev/null +++ b/pkg/pendingcache/cluster.go @@ -0,0 +1,31 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package pendingcache + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" +) + +// overlayCluster wraps a cluster.Cluster so its client and field indexer are +// served through the in-process overlay cache. Everything else (config, scheme, +// REST mapper, HTTP client, API reader, event recorder, and Start) is delegated +// to the embedded inner cluster unchanged. +// +// It deliberately does NOT override Start: the manager already owns the inner +// cluster's lifecycle (home cluster) or drives it via mgr.Add (remotes). The +// overlay's own lifecycle (eviction handlers + TTL cleanup) is a separate +// manager.Runnable — the *Overlay returned alongside this wrapper by WrapCluster. +type overlayCluster struct { + cluster.Cluster + cache *Overlay +} + +// GetClient returns the overlay client so per-cluster reads and writes flow +// through the cache. +func (w *overlayCluster) GetClient() client.Client { return w.cache } + +// GetFieldIndexer returns the overlay so IndexField dual-registers the indexer +// with both the real cache (via Overlay.IndexField) and the overlay. +func (w *overlayCluster) GetFieldIndexer() client.FieldIndexer { return w.cache } diff --git a/pkg/clientcache/config.go b/pkg/pendingcache/config.go similarity index 71% rename from pkg/clientcache/config.go rename to pkg/pendingcache/config.go index 57d5d7a70..8dd18536b 100644 --- a/pkg/clientcache/config.go +++ b/pkg/pendingcache/config.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package clientcache +package pendingcache import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -9,16 +9,14 @@ import ( // Config configures the transparent in-process overlay cache. type Config struct { + // Enabled turns the overlay cache on. When false (the default), clusters are + // used unwrapped: no overlay, no eviction Runnable, zero overhead. + Enabled bool `json:"enabled"` // GVKs the cache should overlay, formatted as "//". // Calls for GVKs not listed here are passed through unchanged. GVKs []string `json:"gvks"` // TTL is the maximum lifetime of an overlay entry before it is evicted by // the background cleanup goroutine, guarding against entries that never // appear in the informer (e.g. after a crash). Defaults to 2m when zero. - TTL metav1.Duration `json:"ttl,omitempty"` -} - -// RootConfig is the top-level config key for the client cache. -type RootConfig struct { - ClientCache Config `json:"clientcache"` + TTL metav1.Duration `json:"ttl,omitzero"` } diff --git a/pkg/clientcache/runnable.go b/pkg/pendingcache/runnable.go similarity index 67% rename from pkg/clientcache/runnable.go rename to pkg/pendingcache/runnable.go index 7196dd787..30c6e20a6 100644 --- a/pkg/clientcache/runnable.go +++ b/pkg/pendingcache/runnable.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package clientcache +package pendingcache import ( "context" @@ -19,8 +19,8 @@ const minCleanupInterval = 30 * time.Second // Start implements manager.Runnable. It attaches informer event handlers for // eviction and runs the TTL cleanup loop until ctx is done. -func (c *CachingClient) Start(ctx context.Context) error { - log := ctrl.LoggerFrom(ctx).WithName("clientcache") +func (c *Overlay) Start(ctx context.Context) error { + log := ctrl.LoggerFrom(ctx).WithName("pendingcache") for gvk := range c.gvks { obj, err := c.newObjectForGVK(gvk) @@ -28,16 +28,13 @@ func (c *CachingClient) Start(ctx context.Context) error { log.Error(err, "failed to build object for gvk; eviction disabled for it", "gvk", gvk) continue } - informers, err := c.inner.GetInformersForKind(ctx, obj) + inf, err := c.informerCache.GetInformer(ctx, obj) if err != nil { - log.Error(err, "failed to get informers for gvk; eviction disabled for it", "gvk", gvk) + log.Error(err, "failed to get informer for gvk; eviction disabled for it", "gvk", gvk) continue } - handler := c.evictionHandler(gvk) - for _, inf := range informers { - if _, err := inf.AddEventHandler(handler); err != nil { - log.Error(err, "failed to add eviction event handler", "gvk", gvk) - } + if _, err := inf.AddEventHandler(c.evictionHandler(gvk)); err != nil { + log.Error(err, "failed to add eviction event handler", "gvk", gvk) } } @@ -54,17 +51,17 @@ func (c *CachingClient) Start(ctx context.Context) error { } } -// NeedLeaderElection reports that the CachingClient's Start lifecycle must run +// NeedLeaderElection reports that the Overlay's Start lifecycle must run // on every replica, not only the elected leader. The overlay is per-process // state, so its eviction handlers and TTL cleanup have to run wherever the // client is used, regardless of leader election. -func (c *CachingClient) NeedLeaderElection() bool { +func (c *Overlay) NeedLeaderElection() bool { return false } // evictionHandler returns an informer event handler that evicts overlay entries // for the GVK when the real object is observed at a >= ResourceVersion. -func (c *CachingClient) evictionHandler(gvk schema.GroupVersionKind) toolscachek8s.ResourceEventHandler { +func (c *Overlay) evictionHandler(gvk schema.GroupVersionKind) toolscachek8s.ResourceEventHandler { evict := func(o any) { obj, ok := o.(client.Object) if !ok { @@ -86,14 +83,14 @@ func (c *CachingClient) evictionHandler(gvk schema.GroupVersionKind) toolscachek } // newObjectForGVK builds an empty typed object for the GVK using the scheme. -func (c *CachingClient) newObjectForGVK(gvk schema.GroupVersionKind) (client.Object, error) { +func (c *Overlay) newObjectForGVK(gvk schema.GroupVersionKind) (client.Object, error) { ro, err := c.scheme.New(gvk) if err != nil { return nil, err } obj, ok := ro.(client.Object) if !ok { - return nil, fmt.Errorf("clientcache: object for gvk %s does not implement client.Object", gvk) + return nil, fmt.Errorf("pendingcache: object for gvk %s does not implement client.Object", gvk) } return obj, nil } From 5fbe22a4641bf2c0921907320dd59367ae3ed66e Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 19 Aug 2026 10:58:53 +0200 Subject: [PATCH 13/16] test: enhance client tests and fix cache indexing logic Signed-off-by: Markus Wieland --- pkg/multicluster/client_test.go | 7 +++++++ pkg/pendingcache/client.go | 6 ++---- pkg/pendingcache/client_test.go | 27 ++++++++++++++------------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/pkg/multicluster/client_test.go b/pkg/multicluster/client_test.go index efbd538d4..890b3f1fd 100644 --- a/pkg/multicluster/client_test.go +++ b/pkg/multicluster/client_test.go @@ -2027,4 +2027,11 @@ func TestClient_CacheEnabled_WrappedHomeOverlay(t *testing.T) { if len(results) != 1 || !results[0].IsHome { t.Fatalf("expected one home result, got %+v", results) } + if len(results[0].Items) != 1 { + t.Fatalf("expected one item in home result, got %d", len(results[0].Items)) + } + if got := results[0].Items[0]; got.Name != cm.Name || got.Namespace != cm.Namespace { + t.Fatalf("expected item {Name:%q Namespace:%q}, got {Name:%q Namespace:%q}", + cm.Name, cm.Namespace, got.Name, got.Namespace) + } } diff --git a/pkg/pendingcache/client.go b/pkg/pendingcache/client.go index 1d68a600d..6f8f7325c 100644 --- a/pkg/pendingcache/client.go +++ b/pkg/pendingcache/client.go @@ -603,10 +603,8 @@ func (c *Overlay) List(ctx context.Context, list client.ObjectList, opts ...clie // registers the IndexerFunc with the overlay so overlay entries can be matched // against MatchingFields. func (c *Overlay) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { - if indexer, ok := c.Client.(client.FieldIndexer); ok { - if err := indexer.IndexField(ctx, obj, field, extractValue); err != nil { - return err - } + if err := c.informerCache.IndexField(ctx, obj, field, extractValue); err != nil { + return err } if gvk, cached := c.gvkFor(obj); cached { c.registerIndex(gvk, field, extractValue) diff --git a/pkg/pendingcache/client_test.go b/pkg/pendingcache/client_test.go index fc6716449..6cd964088 100644 --- a/pkg/pendingcache/client_test.go +++ b/pkg/pendingcache/client_test.go @@ -98,6 +98,10 @@ func (f *fakeCache) GetInformer(_ context.Context, _ client.Object, _ ...ccache. return f.inf, nil } +func (f *fakeCache) IndexField(_ context.Context, _ client.Object, _ string, _ client.IndexerFunc) error { + return nil +} + // fakeCluster composes a fake client.Client with a fakeCache to satisfy the // cluster.Cluster interface consumed by pendingcache.WrapCluster. type fakeCluster struct { @@ -251,17 +255,14 @@ func reservationConfig() Config { } } -// clusterFor wraps a client.Client (and the given informer) as a cluster.Cluster -// so it can be passed to New. Used by tests that inject a custom client.Client -// (e.g. errClient, orderingClient) below the overlay. -func clusterFor(t *testing.T, inner client.Client, inf *fakeInformer) cluster.Cluster { +// clusterFor wraps a client.Client as a cluster.Cluster so it can be passed to +// WrapCluster. Used by tests that inject a custom client.Client (e.g. errClient, +// orderingClient) below the overlay. +func clusterFor(t *testing.T, inner client.Client) cluster.Cluster { t.Helper() - if inf == nil { - inf = &fakeInformer{} - } return &fakeCluster{ client: inner, - cache: &fakeCache{inf: inf}, + cache: &fakeCache{inf: &fakeInformer{}}, scheme: testScheme(t), } } @@ -719,7 +720,7 @@ func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { } else { base = newTestClient(t) } - c := cachingFrom(t, clusterFor(t, tc.mkClient(base), nil), reservationConfig()) + c := cachingFrom(t, clusterFor(t, tc.mkClient(base)), reservationConfig()) if err := tc.op(c, r); !errors.Is(err, sentinel) { t.Fatalf("expected sentinel error, got %v", err) @@ -807,7 +808,7 @@ func TestWriteServedFromOverlay(t *testing.T) { func TestGetPropagatesNonNotFoundError(t *testing.T) { sentinel := errors.New("get boom") - c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), getErr: sentinel}, nil), reservationConfig()) + c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), getErr: sentinel}), reservationConfig()) c.upsert(reservationGVK(), newReservation("res-ge", "az-1", "1")) var got v1alpha1.Reservation @@ -840,7 +841,7 @@ func TestGetNotFoundWithNoOverlay(t *testing.T) { func TestGetNonCachedPropagatesError(t *testing.T) { sentinel := errors.New("get boom") - c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), getErr: sentinel}, nil), Config{}) + c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), getErr: sentinel}), Config{}) var got v1alpha1.Reservation if gerr := c.Get(context.Background(), types.NamespacedName{Name: "x"}, &got); !errors.Is(gerr, sentinel) { t.Fatalf("expected sentinel error, got %v", gerr) @@ -849,7 +850,7 @@ func TestGetNonCachedPropagatesError(t *testing.T) { func TestListPropagatesError(t *testing.T) { sentinel := errors.New("list boom") - c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), listErr: sentinel}, nil), reservationConfig()) + c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), listErr: sentinel}), reservationConfig()) c.upsert(reservationGVK(), newReservation("res-le", "az-1", "1")) var list v1alpha1.ReservationList @@ -940,7 +941,7 @@ func TestConcurrentUpdatesOverlayNotBehind(t *testing.T) { ) for round := range rounds { oc := &orderingClient{Client: newTestClient(t).GetClient()} - c := cachingFrom(t, clusterFor(t, oc, nil), reservationConfig()) + c := cachingFrom(t, clusterFor(t, oc), reservationConfig()) var wg sync.WaitGroup for i := 1; i <= n; i++ { From 6cc28686de5d2125b0526bcd7e869e38696f9520 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 19 Aug 2026 11:09:22 +0200 Subject: [PATCH 14/16] chore: replace changes that were needed because cache wraps mcl but are now not required anymore Signed-off-by: Markus Wieland --- cmd/manager/main.go | 4 ++-- .../nova/hypervisor_overcommit_controller.go | 8 +++++++- .../hypervisor_overcommit_controller_test.go | 2 +- .../reservations/inflight/controller.go | 16 ++++++++-------- .../reservations/inflight/controller_test.go | 9 +++++++++ 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index cf6ae5b8d..f69cc9005 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -506,7 +506,7 @@ func main() { config := conf.GetConfigOrDie[inflight.NovaVMClientConfig]() vmClient := inflight.NewNovaVMClient(config) controller := &inflight.Controller{Client: multiclusterClient, VMClient: vmClient} - if err := controller.SetupWithManager(ctx, mgr, multiclusterClient); err != nil { + if err := controller.SetupWithManager(ctx, mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "inflight-reservation-controller") os.Exit(1) @@ -536,7 +536,7 @@ func main() { if slices.Contains(mainConfig.EnabledControllers, "hypervisor-overcommit-controller") { hypervisorOvercommitController := &nova.HypervisorOvercommitController{} hypervisorOvercommitController.Client = multiclusterClient - if err := hypervisorOvercommitController.SetupWithManager(mgr, multiclusterClient); err != nil { + if err := hypervisorOvercommitController.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "HypervisorOvercommitController") os.Exit(1) diff --git a/internal/scheduling/nova/hypervisor_overcommit_controller.go b/internal/scheduling/nova/hypervisor_overcommit_controller.go index 7df253849..72e4507fc 100644 --- a/internal/scheduling/nova/hypervisor_overcommit_controller.go +++ b/internal/scheduling/nova/hypervisor_overcommit_controller.go @@ -217,7 +217,7 @@ func (c *HypervisorOvercommitController) predicateRemoteHypervisor() predicate.P // SetupWithManager sets up the controller with the Manager and a multicluster // client. The multicluster client is used to watch for changes in the // Hypervisor CRD across all clusters and trigger reconciliations accordingly. -func (c *HypervisorOvercommitController) SetupWithManager(mgr ctrl.Manager, mcl *multicluster.Client) (err error) { +func (c *HypervisorOvercommitController) SetupWithManager(mgr ctrl.Manager) (err error) { // This will load the config in a safe way and gracefully handle errors. c.config, err = conf.GetConfig[HypervisorOvercommitConfig]() if err != nil { @@ -227,6 +227,12 @@ func (c *HypervisorOvercommitController) SetupWithManager(mgr ctrl.Manager, mcl if err := c.config.Validate(); err != nil { return err } + // Check that the provided client is a multicluster client, since we need + // that to watch for hypervisors across clusters. + mcl, ok := c.Client.(*multicluster.Client) + if !ok { + return errors.New("provided client must be a multicluster client") + } bldr := multicluster.BuildController(mcl, mgr) // The hypervisor crd may be distributed across multiple remote clusters. bldr, err = bldr.WatchesMulticluster(&hv1.Hypervisor{}, diff --git a/internal/scheduling/nova/hypervisor_overcommit_controller_test.go b/internal/scheduling/nova/hypervisor_overcommit_controller_test.go index f122831eb..e52669c3a 100644 --- a/internal/scheduling/nova/hypervisor_overcommit_controller_test.go +++ b/internal/scheduling/nova/hypervisor_overcommit_controller_test.go @@ -725,7 +725,7 @@ func TestHypervisorOvercommitController_SetupWithManager_InvalidClient(t *testin // SetupWithManager should fail - either because config loading fails // (in test environment without config files) or because the client // is not a multicluster client. - err := controller.SetupWithManager(mgr, nil) + err := controller.SetupWithManager(mgr) if err == nil { t.Error("expected error when calling SetupWithManager, got nil") } diff --git a/internal/scheduling/reservations/inflight/controller.go b/internal/scheduling/reservations/inflight/controller.go index e68713689..4292fa334 100644 --- a/internal/scheduling/reservations/inflight/controller.go +++ b/internal/scheduling/reservations/inflight/controller.go @@ -378,7 +378,14 @@ func (c *Controller) predicateHypervisors() predicate.Predicate { // SetupWithManager sets up the controller with the Manager and a multicluster // client. The multicluster client is used to watch for changes in the // Reservation CRD across all clusters and trigger reconciliations accordingly. -func (c *Controller) SetupWithManager(ctx context.Context, mgr ctrl.Manager, mcl *multicluster.Client) (err error) { +func (c *Controller) SetupWithManager(ctx context.Context, mgr ctrl.Manager) (err error) { + // Check that the provided client is a multicluster client, since we need + // that to watch for hypervisors across clusters. Do this before adding + // any runnables so a misconfigured setup fails fast. + mcl, ok := c.Client.(*multicluster.Client) + if !ok { + return errors.New("provided client must be a multicluster client") + } // Add the vm client as runnable to the manager. if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { return c.VMClient.StartWithKubernetesSecrets(ctx, c.Client) @@ -405,13 +412,6 @@ func (c *Controller) SetupWithManager(ctx context.Context, mgr ctrl.Manager, mcl ); err != nil { return err } - // Register the same index with the overlay so that List calls with - // MatchingFields correctly filter in-flight overlay entries. - if fi, ok := c.Client.(client.FieldIndexer); ok { - if err := fi.IndexField(ctx, &v1alpha1.Reservation{}, idxReservationByTargetHost, idxReservationByTargetHostFn); err != nil { - return err - } - } // Watch hypervisor changes and requeue reservations targeting // the changed hypervisor. bldr, err = bldr.WatchesMulticluster(&hv1.Hypervisor{}, diff --git a/internal/scheduling/reservations/inflight/controller_test.go b/internal/scheduling/reservations/inflight/controller_test.go index bfeb04754..6f61d3a76 100644 --- a/internal/scheduling/reservations/inflight/controller_test.go +++ b/internal/scheduling/reservations/inflight/controller_test.go @@ -716,3 +716,12 @@ func TestHandleHypervisors_NoMatchingReservations(t *testing.T) { t.Errorf("queue = %+v, want empty", q.items) } } + +func TestSetupWithManager_RejectsNonMulticlusterClient(t *testing.T) { + scheme := newTestScheme(t) + c := &Controller{Client: newTestClient(scheme), VMClient: &stubVMClient{}} + err := c.SetupWithManager(context.Background(), nil) + if err == nil { + t.Fatal("expected error for non-multicluster client, got nil") + } +} From 10ef5110056abd0c8cf1ebd5d3119254dc59c604 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 19 Aug 2026 16:03:06 +0200 Subject: [PATCH 15/16] feedback Signed-off-by: Markus Wieland --- cmd/manager/main.go | 5 + helm/bundles/cortex-nova/values.yaml | 12 +- pkg/{pendingcache => cache}/client.go | 122 ++++----------------- pkg/{pendingcache => cache}/client_test.go | 24 ++-- pkg/{pendingcache => cache}/cluster.go | 30 ++++- pkg/{pendingcache => cache}/config.go | 7 +- pkg/cache/lock.go | 85 ++++++++++++++ pkg/{pendingcache => cache}/runnable.go | 6 +- pkg/multicluster/client.go | 89 ++++++++------- pkg/multicluster/client_test.go | 61 ----------- pkg/multicluster/config.go | 6 - 11 files changed, 207 insertions(+), 240 deletions(-) rename pkg/{pendingcache => cache}/client.go (81%) rename pkg/{pendingcache => cache}/client_test.go (97%) rename pkg/{pendingcache => cache}/cluster.go (52%) rename pkg/{pendingcache => cache}/config.go (85%) create mode 100644 pkg/cache/lock.go rename pkg/{pendingcache => cache}/runnable.go (93%) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index f69cc9005..e7dd070f9 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -65,6 +65,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/inflight" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/quota" + "github.com/cobaltcore-dev/cortex/pkg/cache" "github.com/cobaltcore-dev/cortex/pkg/conf" "github.com/cobaltcore-dev/cortex/pkg/monitoring" "github.com/cobaltcore-dev/cortex/pkg/multicluster" @@ -391,6 +392,10 @@ func main() { }, } multiclusterClientConfig := conf.GetConfigOrDie[multicluster.ClientConfig]() + + if c := conf.GetConfigOrDie[cache.RootConfig](); c.Cache.Enabled { + multiclusterClient.Wrappers = append(multiclusterClient.Wrappers, cache.NewWrapper(c.Cache)) + } if err := multiclusterClient.InitFromConf(ctx, mgr, multiclusterClientConfig); err != nil { setupLog.Error(err, "unable to initialize multicluster client") os.Exit(1) diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 3babbc19d..82cc898b1 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -74,6 +74,12 @@ kvm: # Useful when rolling out a new KVM region before it is production-ready. criticalAlerts: true +# Cache configuration for the transparent in-process overlay cache. +cache: + enabled: true + gvks: + - cortex.cloud/v1alpha1/Reservation + cortex: &cortex crd: {enable: false} # Disable the default ServiceMonitor and metrics service from the kubebuilder stack. @@ -110,10 +116,6 @@ cortex: &cortex - kvm.cloud.sap/v1/Hypervisor - kvm.cloud.sap/v1/HypervisorList - v1/Secret - pendingcache: - enabled: true - gvks: - - cortex.cloud/v1alpha1/Reservation keystoneSecretRef: name: cortex-nova-openstack-keystone namespace: default @@ -189,7 +191,7 @@ cortex-scheduling-controllers: # How long after a VM is allocated to a reservation before it is expected to appear # on the target host; allocations not confirmed within this window are removed allocationGracePeriod: "15m" - # How long to wait after detecting host over-subscription before evicting reservation slots. + # How long to wait after detecting host over-subscription before evicting reservation slots. # Gives other controllers (e.g. failover) time to self-heal. oversubscriptionGracePeriod: "3m" # Minimum time between consecutive over-subscription checks for the same host. diff --git a/pkg/pendingcache/client.go b/pkg/cache/client.go similarity index 81% rename from pkg/pendingcache/client.go rename to pkg/cache/client.go index 6f8f7325c..6fb5745de 100644 --- a/pkg/pendingcache/client.go +++ b/pkg/cache/client.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package pendingcache +package cache import ( "context" @@ -32,90 +32,6 @@ type entry struct { expiresAt time.Time } -// objectKey identifies an object within a GVK by namespace and name. -type objectKey struct { - namespace string - name string -} - -func keyForObject(obj client.Object) objectKey { - return objectKey{namespace: obj.GetNamespace(), name: obj.GetName()} -} - -// lockKey identifies the object whose write path a keyedMutex serializes. -type lockKey struct { - gvk schema.GroupVersionKind - key objectKey -} - -// refMutex is the actual per-object lock plus a reference count of how many -// callers currently hold or are waiting for it. The count lets keyedMutex know -// when the entry is unused so it can be deleted (see keyedMutex.lock). -type refMutex struct { - mu sync.Mutex - ref int -} - -// keyedMutex hands out one mutex per key, serializing operations that share a -// key while letting distinct keys proceed concurrently. -// -// A sync.Map (or a map we only ever insert into) would be simpler, but it would leak: -// this cache wraps a single client for the whole lifetime of the controller-manager process, -// and a controller reconciles a continuous stream of (often short-lived) objects. -// A map would therefore accumulate one mutex per distinct object ever written and -// grow without bound. To avoid that we reference count each entry and delete it -// once the last holder unlocks, so the map only ever holds locks for objects -// with writes currently in flight. -// -// The two mutexes have distinct, non-overlapping roles: -// - k.mu guards the locks map itself. It is only ever held for the tiny -// bookkeeping critical sections below (map lookup/insert/delete and the -// ref counter), never across the caller's I/O. -// - rm.mu is the real per-object lock, held by the caller across the inner -// client call and the overlay mutation. -// -// Because k.mu is never held while rm.mu is locked (we release k.mu before -// taking rm.mu), the two can never deadlock against each other. -type keyedMutex struct { - mu sync.Mutex - locks map[lockKey]*refMutex -} - -func newKeyedMutex() *keyedMutex { - return &keyedMutex{locks: make(map[lockKey]*refMutex)} -} - -// lock acquires the per-key mutex and returns a function that releases it. -func (k *keyedMutex) lock(lk lockKey) func() { - // Look up (or create) the entry for this key and register our interest by - // bumping ref, all under k.mu so the map stays consistent. We increment ref - // here, before taking rm.mu, so that a concurrent unlock cannot see ref==0 - // and delete the entry out from under us while we are blocked waiting on it. - k.mu.Lock() - rm := k.locks[lk] - if rm == nil { - rm = &refMutex{} - k.locks[lk] = rm - } - rm.ref++ - k.mu.Unlock() - - // Take the real per-object lock outside k.mu; this is where a second caller - // for the same key blocks (and where we may block across the caller's I/O). - rm.mu.Lock() - return func() { - rm.mu.Unlock() - // Drop our reference and, if we were the last holder, remove the entry - // so the map does not grow unbounded. - k.mu.Lock() - rm.ref-- - if rm.ref == 0 { - delete(k.locks, lk) - } - k.mu.Unlock() - } -} - // resourceVersionAtLeast reports whether observed >= cached, treating // ResourceVersions as opaque monotonically increasing integers (as the // kubernetes apiserver guarantees per resource). Unparsable or empty values @@ -161,7 +77,7 @@ type Overlay struct { gvks map[schema.GroupVersionKind]bool mu sync.RWMutex - byGVK map[schema.GroupVersionKind]map[objectKey]*entry + byGVK map[schema.GroupVersionKind]map[client.ObjectKey]*entry indexers map[schema.GroupVersionKind]map[string]client.IndexerFunc // writeLocks serializes writes to the same object so the inner call and the @@ -199,7 +115,7 @@ func WrapCluster(inner cluster.Cluster, conf Config) (cluster.Cluster, manager.R scheme: scheme, ttl: ttl, gvks: gvks, - byGVK: make(map[schema.GroupVersionKind]map[objectKey]*entry), + byGVK: make(map[schema.GroupVersionKind]map[client.ObjectKey]*entry), indexers: make(map[schema.GroupVersionKind]map[string]client.IndexerFunc), writeLocks: newKeyedMutex(), } @@ -218,7 +134,7 @@ func resolveGVKs(scheme *runtime.Scheme, gvkStrs []string) (map[schema.GroupVers for _, s := range gvkStrs { gvk, ok := byStr[s] if !ok { - return nil, errors.New("pendingcache: no gvk registered in scheme for " + s) + return nil, errors.New("cache: no gvk registered in scheme for " + s) } out[gvk] = true } @@ -263,7 +179,7 @@ func (c *Overlay) upsert(gvk schema.GroupVersionKind, obj client.Object) { c.mu.Lock() defer c.mu.Unlock() c.ensureGVK(gvk) - c.byGVK[gvk][keyForObject(obj)] = &entry{ + c.byGVK[gvk][client.ObjectKeyFromObject(obj)] = &entry{ obj: obj.DeepCopyObject().(client.Object), uid: obj.GetUID(), resourceVersion: obj.GetResourceVersion(), @@ -278,7 +194,7 @@ func (c *Overlay) tombstone(gvk schema.GroupVersionKind, obj client.Object) { c.mu.Lock() defer c.mu.Unlock() c.ensureGVK(gvk) - c.byGVK[gvk][keyForObject(obj)] = &entry{ + c.byGVK[gvk][client.ObjectKeyFromObject(obj)] = &entry{ obj: obj.DeepCopyObject().(client.Object), uid: obj.GetUID(), resourceVersion: obj.GetResourceVersion(), @@ -296,7 +212,7 @@ func (c *Overlay) evictIfSeen(gvk schema.GroupVersionKind, obj client.Object) { if !ok { return } - key := keyForObject(obj) + key := client.ObjectKeyFromObject(obj) e, ok := entries[key] if !ok { return @@ -314,7 +230,7 @@ func (c *Overlay) evictIfSeen(gvk schema.GroupVersionKind, obj client.Object) { } // getEntry returns the overlay entry for the key, if present. -func (c *Overlay) getEntry(gvk schema.GroupVersionKind, key objectKey) (*entry, bool) { +func (c *Overlay) getEntry(gvk schema.GroupVersionKind, key client.ObjectKey) (*entry, bool) { c.mu.RLock() defer c.mu.RUnlock() entries, ok := c.byGVK[gvk] @@ -352,12 +268,12 @@ func (c *Overlay) registerIndex(gvk schema.GroupVersionKind, field string, fn cl // ensureGVK initialises the per-GVK entry map if absent. Callers must hold the write lock. func (c *Overlay) ensureGVK(gvk schema.GroupVersionKind) { if c.byGVK[gvk] == nil { - c.byGVK[gvk] = make(map[objectKey]*entry) + c.byGVK[gvk] = make(map[client.ObjectKey]*entry) } } // overlayList merges the overlay entries for the GVK into the informer result, -// deduplicating by objectKey (overlay wins), dropping tombstones, and filtering +// deduplicating by client.ObjectKey (overlay wins), dropping tombstones, and filtering // overlay-only entries against the list options' label and field selectors. func (c *Overlay) overlayList(gvk schema.GroupVersionKind, existing []runtime.Object, lo *client.ListOptions) []runtime.Object { c.mu.RLock() @@ -369,7 +285,7 @@ func (c *Overlay) overlayList(gvk schema.GroupVersionKind, existing []runtime.Ob result := make([]runtime.Object, 0, len(existing)+len(entries)) // Track which overlay keys are handled so overlay-only entries can be added. - handled := make(map[objectKey]bool, len(entries)) + handled := make(map[client.ObjectKey]bool, len(entries)) for _, item := range existing { obj, ok := item.(client.Object) @@ -377,7 +293,7 @@ func (c *Overlay) overlayList(gvk schema.GroupVersionKind, existing []runtime.Ob result = append(result, item) continue } - key := keyForObject(obj) + key := client.ObjectKeyFromObject(obj) e, present := entries[key] if !present { result = append(result, item) @@ -458,7 +374,7 @@ func (c *Overlay) Create(ctx context.Context, obj client.Object, opts ...client. if !cached { return c.Client.Create(ctx, obj, opts...) } - unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) defer unlock() if err := c.Client.Create(ctx, obj, opts...); err != nil { return err @@ -474,7 +390,7 @@ func (c *Overlay) Update(ctx context.Context, obj client.Object, opts ...client. if !cached { return c.Client.Update(ctx, obj, opts...) } - unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) defer unlock() if err := c.Client.Update(ctx, obj, opts...); err != nil { return err @@ -490,7 +406,7 @@ func (c *Overlay) Patch(ctx context.Context, obj client.Object, patch client.Pat if !cached { return c.Client.Patch(ctx, obj, patch, opts...) } - unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) defer unlock() if err := c.Client.Patch(ctx, obj, patch, opts...); err != nil { return err @@ -510,7 +426,7 @@ func (c *Overlay) Delete(ctx context.Context, obj client.Object, opts ...client. if !cached { return c.Client.Delete(ctx, obj, opts...) } - unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) defer unlock() if err := c.Client.Delete(ctx, obj, opts...); err != nil { return err @@ -562,7 +478,7 @@ func (c *Overlay) Get(ctx context.Context, key client.ObjectKey, obj client.Obje if err != nil && !apierrors.IsNotFound(err) { return err } - e, ok := c.getEntry(gvk, objectKey{namespace: key.Namespace, name: key.Name}) + e, ok := c.getEntry(gvk, key) if !ok { // No overlay entry: return the inner result (value or NotFound) as-is. return err @@ -634,7 +550,7 @@ func (s *statusWriter) Update(ctx context.Context, obj client.Object, opts ...cl if !cached { return s.inner.Update(ctx, obj, opts...) } - unlock := s.c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + unlock := s.c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) defer unlock() if err := s.inner.Update(ctx, obj, opts...); err != nil { return err @@ -648,7 +564,7 @@ func (s *statusWriter) Patch(ctx context.Context, obj client.Object, patch clien if !cached { return s.inner.Patch(ctx, obj, patch, opts...) } - unlock := s.c.writeLocks.lock(lockKey{gvk: gvk, key: keyForObject(obj)}) + unlock := s.c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) defer unlock() if err := s.inner.Patch(ctx, obj, patch, opts...); err != nil { return err diff --git a/pkg/pendingcache/client_test.go b/pkg/cache/client_test.go similarity index 97% rename from pkg/pendingcache/client_test.go rename to pkg/cache/client_test.go index 6cd964088..f1d43dac6 100644 --- a/pkg/pendingcache/client_test.go +++ b/pkg/cache/client_test.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package pendingcache +package cache import ( "context" @@ -103,7 +103,7 @@ func (f *fakeCache) IndexField(_ context.Context, _ client.Object, _ string, _ c } // fakeCluster composes a fake client.Client with a fakeCache to satisfy the -// cluster.Cluster interface consumed by pendingcache.WrapCluster. +// cluster.Cluster interface consumed by cache.WrapCluster. type fakeCluster struct { cluster.Cluster client client.Client @@ -447,7 +447,7 @@ func TestEviction(t *testing.T) { inner.inf.fireAdd(observed) } - _, present := c.getEntry(reservationGVK(), objectKey{name: "res-3"}) + _, present := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-3"}) if present == tc.wantEvicted { t.Fatalf("evicted=%v, want evicted=%v", !present, tc.wantEvicted) } @@ -459,7 +459,7 @@ func TestEvictionIgnoresNonObject(t *testing.T) { inner := newTestClient(t) c := newCaching(t, inner) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(t.Context()) defer cancel() go func() { if err := c.Start(ctx); err != nil && ctx.Err() == nil { @@ -474,7 +474,7 @@ func TestEvictionIgnoresNonObject(t *testing.T) { c.upsert(reservationGVK(), newReservation("res-x", "az-1", "1")) inner.inf.fireAdd("not-an-object") - if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-x"}); !ok { + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-x"}); !ok { t.Fatalf("non-object event must not evict the entry") } } @@ -490,7 +490,7 @@ func TestTTLCleanup(t *testing.T) { } c.mu.Unlock() c.cleanupExpired(time.Now()) - if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-4"}); ok { + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-4"}); ok { t.Fatalf("expired entry should be removed") } } @@ -635,7 +635,7 @@ func TestNonCachedGVKPassthrough(t *testing.T) { if err := c.Create(context.Background(), r); err != nil { t.Fatalf("Create: %v", err) } - if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-9"}); ok { + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-9"}); ok { t.Fatalf("non-cached GVK should not populate overlay") } if err := c.Delete(context.Background(), r); err != nil { @@ -725,7 +725,7 @@ func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { if err := tc.op(c, r); !errors.Is(err, sentinel) { t.Fatalf("expected sentinel error, got %v", err) } - if _, ok := c.getEntry(reservationGVK(), objectKey{name: r.Name}); ok { + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: r.Name}); ok { t.Fatalf("overlay must not be touched on %s failure", tc.name) } }) @@ -866,7 +866,7 @@ func TestStatusUpdateErrorLeavesOverlayUntouched(t *testing.T) { if err := c.Status().Update(context.Background(), r); err == nil { t.Fatalf("expected status update to fail for missing object") } - if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-se"}); ok { + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-se"}); ok { t.Fatalf("overlay must not be populated on status update failure") } } @@ -878,7 +878,7 @@ func TestStatusCreateDelegates(t *testing.T) { if err := c.Status().Create(context.Background(), r, r); err == nil { t.Fatalf("expected Status().Create to fail on fake client") } - if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-sc"}); ok { + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-sc"}); ok { t.Fatalf("Status().Create must not populate the overlay") } } @@ -895,7 +895,7 @@ func TestStatusUpdateNonCachedNoOverlay(t *testing.T) { if err := c.Status().Update(context.Background(), &cur); err != nil { t.Fatalf("Status().Update: %v", err) } - if _, ok := c.getEntry(reservationGVK(), objectKey{name: "res-sn"}); ok { + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-sn"}); ok { t.Fatalf("non-cached GVK status update should not populate overlay") } } @@ -960,7 +960,7 @@ func TestConcurrentUpdatesOverlayNotBehind(t *testing.T) { lastRV := oc.lastRV oc.mu.Unlock() - e, ok := c.getEntry(reservationGVK(), objectKey{name: "res-conc"}) + e, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-conc"}) if !ok { t.Fatalf("round %d: expected overlay entry for res-conc", round) } diff --git a/pkg/pendingcache/cluster.go b/pkg/cache/cluster.go similarity index 52% rename from pkg/pendingcache/cluster.go rename to pkg/cache/cluster.go index a784d1879..0fc51cdcd 100644 --- a/pkg/pendingcache/cluster.go +++ b/pkg/cache/cluster.go @@ -1,11 +1,12 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package pendingcache +package cache import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/manager" ) // overlayCluster wraps a cluster.Cluster so its client and field indexer are @@ -15,8 +16,8 @@ import ( // // It deliberately does NOT override Start: the manager already owns the inner // cluster's lifecycle (home cluster) or drives it via mgr.Add (remotes). The -// overlay's own lifecycle (eviction handlers + TTL cleanup) is a separate -// manager.Runnable — the *Overlay returned alongside this wrapper by WrapCluster. +// overlay's own lifecycle (eviction handlers + TTL cleanup) is registered with +// the manager directly by Wrapper.WrapCluster. type overlayCluster struct { cluster.Cluster cache *Overlay @@ -29,3 +30,26 @@ func (w *overlayCluster) GetClient() client.Client { return w.cache } // GetFieldIndexer returns the overlay so IndexField dual-registers the indexer // with both the real cache (via Overlay.IndexField) and the overlay. func (w *overlayCluster) GetFieldIndexer() client.FieldIndexer { return w.cache } + +// Wrapper implements the multicluster.ClusterWrapper interface for the overlay +// cache. It does not import the multicluster package; interface satisfaction is +// structural and checked at the call site (e.g. in main.go). +type Wrapper struct{ conf Config } + +// NewWrapper returns a Wrapper that applies the overlay cache to any cluster +// passed to WrapCluster. +func NewWrapper(conf Config) *Wrapper { return &Wrapper{conf} } + +// WrapCluster applies the overlay cache to cl, registers the overlay's lifecycle +// Runnable with mgr, and returns the wrapped cluster. +// Satisfies multicluster.ClusterWrapper structurally. +func (w *Wrapper) WrapCluster(mgr manager.Manager, cl cluster.Cluster) (cluster.Cluster, error) { + wrapped, cleanUpRunnable, err := WrapCluster(cl, w.conf) + if err != nil { + return nil, err + } + if err := mgr.Add(cleanUpRunnable); err != nil { + return nil, err + } + return wrapped, nil +} diff --git a/pkg/pendingcache/config.go b/pkg/cache/config.go similarity index 85% rename from pkg/pendingcache/config.go rename to pkg/cache/config.go index 8dd18536b..08c58d573 100644 --- a/pkg/pendingcache/config.go +++ b/pkg/cache/config.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package pendingcache +package cache import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -20,3 +20,8 @@ type Config struct { // appear in the informer (e.g. after a crash). Defaults to 2m when zero. TTL metav1.Duration `json:"ttl,omitzero"` } + +type RootConfig struct { + // Cache configures the transparent in-process overlay cache. + Cache Config `json:"cache"` +} diff --git a/pkg/cache/lock.go b/pkg/cache/lock.go new file mode 100644 index 000000000..22d2a351a --- /dev/null +++ b/pkg/cache/lock.go @@ -0,0 +1,85 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "sync" + + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// lockKey identifies the object whose write path a keyedMutex serializes. +type lockKey struct { + gvk schema.GroupVersionKind + key client.ObjectKey +} + +// refMutex is the actual per-object lock plus a reference count of how many +// callers currently hold or are waiting for it. The count lets keyedMutex know +// when the entry is unused so it can be deleted (see keyedMutex.lock). +type refMutex struct { + mu sync.Mutex + ref int +} + +// keyedMutex hands out one mutex per key, serializing operations that share a +// key while letting distinct keys proceed concurrently. +// +// A sync.Map (or a map we only ever insert into) would be simpler, but it would leak: +// this cache wraps a single client for the whole lifetime of the controller-manager process, +// and a controller reconciles a continuous stream of (often short-lived) objects. +// A map would therefore accumulate one mutex per distinct object ever written and +// grow without bound. To avoid that we reference count each entry and delete it +// once the last holder unlocks, so the map only ever holds locks for objects +// with writes currently in flight. +// +// The two mutexes have distinct, non-overlapping roles: +// - k.mu guards the locks map itself. It is only ever held for the tiny +// bookkeeping critical sections below (map lookup/insert/delete and the +// ref counter), never across the caller's I/O. +// - rm.mu is the real per-object lock, held by the caller across the inner +// client call and the overlay mutation. +// +// Because k.mu is never held while rm.mu is locked (we release k.mu before +// taking rm.mu), the two can never deadlock against each other. +type keyedMutex struct { + mu sync.Mutex + locks map[lockKey]*refMutex +} + +func newKeyedMutex() *keyedMutex { + return &keyedMutex{locks: make(map[lockKey]*refMutex)} +} + +// lock acquires the per-key mutex and returns a function that releases it. +func (k *keyedMutex) lock(lk lockKey) func() { + // Look up (or create) the entry for this key and register our interest by + // bumping ref, all under k.mu so the map stays consistent. We increment ref + // here, before taking rm.mu, so that a concurrent unlock cannot see ref==0 + // and delete the entry out from under us while we are blocked waiting on it. + k.mu.Lock() + rm := k.locks[lk] + if rm == nil { + rm = &refMutex{} + k.locks[lk] = rm + } + rm.ref++ + k.mu.Unlock() + + // Take the real per-object lock outside k.mu; this is where a second caller + // for the same key blocks (and where we may block across the caller's I/O). + rm.mu.Lock() + return func() { + rm.mu.Unlock() + // Drop our reference and, if we were the last holder, remove the entry + // so the map does not grow unbounded. + k.mu.Lock() + rm.ref-- + if rm.ref == 0 { + delete(k.locks, lk) + } + k.mu.Unlock() + } +} diff --git a/pkg/pendingcache/runnable.go b/pkg/cache/runnable.go similarity index 93% rename from pkg/pendingcache/runnable.go rename to pkg/cache/runnable.go index 30c6e20a6..a4d8a9a21 100644 --- a/pkg/pendingcache/runnable.go +++ b/pkg/cache/runnable.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package pendingcache +package cache import ( "context" @@ -20,7 +20,7 @@ const minCleanupInterval = 30 * time.Second // Start implements manager.Runnable. It attaches informer event handlers for // eviction and runs the TTL cleanup loop until ctx is done. func (c *Overlay) Start(ctx context.Context) error { - log := ctrl.LoggerFrom(ctx).WithName("pendingcache") + log := ctrl.LoggerFrom(ctx).WithName("cache") for gvk := range c.gvks { obj, err := c.newObjectForGVK(gvk) @@ -90,7 +90,7 @@ func (c *Overlay) newObjectForGVK(gvk schema.GroupVersionKind) (client.Object, e } obj, ok := ro.(client.Object) if !ok { - return nil, fmt.Errorf("pendingcache: object for gvk %s does not implement client.Object", gvk) + return nil, fmt.Errorf("cache: object for gvk %s does not implement client.Object", gvk) } return obj, nil } diff --git a/pkg/multicluster/client.go b/pkg/multicluster/client.go index abb83fe4e..d6e60f65f 100644 --- a/pkg/multicluster/client.go +++ b/pkg/multicluster/client.go @@ -21,10 +21,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/manager" - - "github.com/cobaltcore-dev/cortex/pkg/pendingcache" ) +// ClusterWrapper can be registered on the Client to transparently transform each +// cluster before it is stored for routing. WrapCluster receives the manager and +// the raw inner cluster; it must return the (possibly wrapped) cluster to use for +// all routing and is responsible for registering any lifecycle Runnables with mgr +// itself. Wrappers are applied in order for every home and remote cluster during +// InitFromConf. The raw inner cluster is always added to the manager separately +// so its informers start regardless of wrapping. +type ClusterWrapper interface { + WrapCluster(mgr manager.Manager, cl cluster.Cluster) (cluster.Cluster, error) +} + // A remote cluster with routing labels used to match resources to clusters. type remoteCluster struct { cluster cluster.Cluster @@ -36,6 +45,12 @@ type Client struct { // when multiple clusters serve the same GVK. ResourceRouters map[schema.GroupVersionKind]ResourceRouter + // Wrappers are applied to every cluster (home and remotes) during InitFromConf. + // Each wrapper transforms the cluster before it is stored for routing. + // Applied in slice order; the raw inner cluster is always added to the manager + // so its informers start independently of wrapping. + Wrappers []ClusterWrapper + // The cluster in which cortex is deployed. HomeCluster cluster.Cluster // The REST config for the home cluster in which cortex is deployed. @@ -56,11 +71,6 @@ type Client struct { // GVKs explicitly configured for the home cluster. homeGVKs map[schema.GroupVersionKind]bool - - // cacheConf configures the optional in-process overlay cache. When - // Enabled, each home/remote cluster is wrapped so its GetClient/ - // GetFieldIndexer route through a per-cluster overlay. Set in InitFromConf. - cacheConf pendingcache.Config } // Helper function to initialize a new multicluster client during service startup, @@ -68,7 +78,6 @@ type Client struct { func (c *Client) InitFromConf(ctx context.Context, mgr ctrl.Manager, conf ClientConfig) error { log := ctrl.LoggerFrom(ctx) log.Info("initializing multicluster client with config", "config", conf) - c.cacheConf = conf.PendingCache // Map the formatted gvk from the config to the actual gvk object so that we // can look up the right cluster for a given API server override. gvksByConfStr := make(map[string]schema.GroupVersionKind) @@ -99,33 +108,25 @@ func (c *Client) InitFromConf(ctx context.Context, mgr ctrl.Manager, conf Client } resolvedGVKs = append(resolvedGVKs, gvk) } - cl, overlay, err := c.AddRemote(ctx, remote.Host, remote.CACert, remote.InsecureSkipTLSVerify, remote.Labels, resolvedGVKs...) + cl, err := c.AddRemote(ctx, mgr, remote.Host, remote.CACert, remote.InsecureSkipTLSVerify, remote.Labels, resolvedGVKs...) if err != nil { return err } - // Add the raw inner cluster so its informers/caches start (as today). + // Add the raw inner cluster so its informers/caches start. if err := mgr.Add(cl); err != nil { return err } - // When caching is enabled, also add the overlay's lifecycle Runnable - // (eviction handlers + TTL cleanup). It does not re-Start the cluster. - if overlay != nil { - if err := mgr.Add(overlay); err != nil { - return err - } - } } - // When caching is enabled, wrap the home cluster the same way. The manager - // already owns the home cluster's lifecycle, so we only add the overlay - // Runnable and must NOT re-Start the inner home cluster. - if c.cacheConf.Enabled && c.HomeCluster != nil { - wrapped, overlay, err := pendingcache.WrapCluster(c.HomeCluster, c.cacheConf) - if err != nil { - return err - } - c.HomeCluster = wrapped - if err := mgr.Add(overlay); err != nil { - return err + // Apply wrappers to the home cluster. The manager already owns the home + // cluster's lifecycle, so we only apply wrappers (each wrapper registers its + // own Runnables) and must NOT re-Start the inner home cluster. + if c.HomeCluster != nil { + for _, w := range c.Wrappers { + wrapped, werr := w.WrapCluster(mgr, c.HomeCluster) + if werr != nil { + return werr + } + c.HomeCluster = wrapped } } return nil @@ -142,12 +143,11 @@ func (c *Client) InitFromConf(ctx context.Context, mgr ctrl.Manager, conf Client // account tokens. See the kubernetes documentation on structured auth to // learn more about jwt-based authentication across clusters. // AddRemote returns the raw inner cluster.Cluster (which the caller must add to -// the manager so its informers/caches start) and, when caching is enabled, the -// overlay's lifecycle Runnable (which the caller must also add to the manager). -// The overlay Runnable is nil when caching is disabled. The wrapped cluster (or -// the raw cluster when caching is disabled) is stored in remoteClusters so all -// routing goes through the per-cluster overlay. -func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSkipTLSVerify bool, labels map[string]string, gvks ...schema.GroupVersionKind) (cluster.Cluster, manager.Runnable, error) { +// the manager so its informers/caches start). Each registered Wrapper is +// responsible for adding its own lifecycle Runnables to mgr directly. +// The wrapped cluster is stored in remoteClusters so all routing goes through +// any per-cluster wrapper. +func (c *Client) AddRemote(ctx context.Context, mgr manager.Manager, host, caCert string, insecureSkipTLSVerify bool, labels map[string]string, gvks ...schema.GroupVersionKind) (cluster.Cluster, error) { log := ctrl.LoggerFrom(ctx) homeRestConfig := *c.HomeRestConfig restConfigCopy := homeRestConfig @@ -165,20 +165,18 @@ func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSki o.Logger = ctrl.LoggerFrom(ctx).WithValues("host", host) }) if err != nil { - return nil, nil, err + return nil, err } - // stored is the cluster placed in remoteClusters and used for all routing. - // When caching is enabled it is the overlay-wrapped cluster; otherwise the - // raw cluster. overlay is the overlay's lifecycle Runnable (nil when off). + // Apply each registered wrapper in order. stored is the cluster placed in + // remoteClusters. Each wrapper is responsible for registering its own + // lifecycle Runnables with mgr directly. stored := cl - var overlay manager.Runnable - if c.cacheConf.Enabled { - wrapped, ov, werr := pendingcache.WrapCluster(cl, c.cacheConf) + for _, w := range c.Wrappers { + wrapped, werr := w.WrapCluster(mgr, stored) if werr != nil { - return nil, nil, werr + return nil, werr } stored = wrapped - overlay = ov } c.remoteClustersMu.Lock() defer c.remoteClustersMu.Unlock() @@ -192,9 +190,8 @@ func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSki labels: labels, }) } - // Return the raw inner cluster so the caller starts its informers/caches; - // the overlay Runnable (if any) is returned separately. - return cl, overlay, nil + // Return the raw inner cluster so the caller starts its informers/caches. + return cl, nil } // Get the gvk registered for the given resource in the home cluster's scheme. diff --git a/pkg/multicluster/client_test.go b/pkg/multicluster/client_test.go index 890b3f1fd..29ea8e0ca 100644 --- a/pkg/multicluster/client_test.go +++ b/pkg/multicluster/client_test.go @@ -26,7 +26,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/pkg/pendingcache" ) // unversionedType is a type that is registered as unversioned in the scheme. @@ -1975,63 +1974,3 @@ func TestClient_ListMetadataPerCluster(t *testing.T) { } }) } - -// TestClient_CacheEnabled_WrappedHomeOverlay verifies that when a home cluster -// is wrapped by the pendingcache overlay, a write immediately followed by a read -// of a cached GVK is served from the overlay (before any informer catches up), -// and that ListMetadataPerCluster passes through the overlay unchanged (its -// PartialObjectMetadata queries do not resolve a cached GVK). -func TestClient_CacheEnabled_WrappedHomeOverlay(t *testing.T) { - scheme := newTestScheme(t) - inner := newFakeCluster(scheme) - - cacheConf := pendingcache.Config{ - Enabled: true, - GVKs: []string{"v1/ConfigMap"}, - } - wrapped, runnable, err := pendingcache.WrapCluster(inner, cacheConf) - if err != nil { - t.Fatalf("pendingcache.WrapCluster: %v", err) - } - if runnable == nil { - t.Fatalf("expected a non-nil overlay Runnable") - } - - c := &Client{ - HomeCluster: wrapped, - HomeScheme: scheme, - homeGVKs: map[schema.GroupVersionKind]bool{configMapGVK: true}, - } - - cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-1", Namespace: "default"}} - if err := c.Create(context.Background(), cm); err != nil { - t.Fatalf("Create: %v", err) - } - - // Immediately read back through the multicluster client — the overlay must - // serve it even though the fake cache/informer never observed it. - var got corev1.ConfigMap - if err := c.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: "cm-1"}, &got); err != nil { - t.Fatalf("Get after create: %v", err) - } - if got.Name != "cm-1" { - t.Fatalf("expected cm-1 from overlay, got %q", got.Name) - } - - // ListMetadataPerCluster uses PartialObjectMetadataList → the overlay passes - // through. It must still return the underlying cluster's metadata result. - results, err := c.ListMetadataPerCluster(context.Background(), configMapGVK) - if err != nil { - t.Fatalf("ListMetadataPerCluster: %v", err) - } - if len(results) != 1 || !results[0].IsHome { - t.Fatalf("expected one home result, got %+v", results) - } - if len(results[0].Items) != 1 { - t.Fatalf("expected one item in home result, got %d", len(results[0].Items)) - } - if got := results[0].Items[0]; got.Name != cm.Name || got.Namespace != cm.Namespace { - t.Fatalf("expected item {Name:%q Namespace:%q}, got {Name:%q Namespace:%q}", - cm.Name, cm.Namespace, got.Name, got.Namespace) - } -} diff --git a/pkg/multicluster/config.go b/pkg/multicluster/config.go index ed73af4e4..359d225cb 100644 --- a/pkg/multicluster/config.go +++ b/pkg/multicluster/config.go @@ -3,17 +3,11 @@ package multicluster -import "github.com/cobaltcore-dev/cortex/pkg/pendingcache" - type ClientConfig struct { // Apiserver configuration mapping GVKs to home or remote clusters. // Every GVK used through the multicluster client must be listed // in either Home or Remotes. Unknown GVKs will cause an error. APIServers APIServersConfig `json:"apiservers"` - - // PendingCache configures the optional transparent in-process overlay cache. When - // PendingCache.Enabled is false (the default), clusters are used unwrapped. - PendingCache pendingcache.Config `json:"pendingcache"` } // APIServersConfig separates resources into home and remote clusters. From f36218a9a10378a9d2907536fb9ee880ffb8806f Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 19 Aug 2026 16:11:43 +0200 Subject: [PATCH 16/16] docs: clarify ClusterWrapper interface documentation for better understanding Signed-off-by: Markus Wieland --- pkg/multicluster/client.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/multicluster/client.go b/pkg/multicluster/client.go index d6e60f65f..cb8d1c0b5 100644 --- a/pkg/multicluster/client.go +++ b/pkg/multicluster/client.go @@ -25,11 +25,13 @@ import ( // ClusterWrapper can be registered on the Client to transparently transform each // cluster before it is stored for routing. WrapCluster receives the manager and -// the raw inner cluster; it must return the (possibly wrapped) cluster to use for -// all routing and is responsible for registering any lifecycle Runnables with mgr -// itself. Wrappers are applied in order for every home and remote cluster during -// InitFromConf. The raw inner cluster is always added to the manager separately -// so its informers start regardless of wrapping. +// the current cluster: raw for the first wrapper in the chain, or the previous +// wrapper's result for subsequent ones. It must return the (possibly wrapped) +// cluster to use for routing and is responsible for registering any lifecycle +// Runnables with mgr itself. Wrappers are applied in order for every home and +// remote cluster during InitFromConf. For remote clusters the original unwrapped +// cluster is always added to the manager separately so its informers start +// independently of wrapping. type ClusterWrapper interface { WrapCluster(mgr manager.Manager, cl cluster.Cluster) (cluster.Cluster, error) }