feat: make the tier1 and tier2 cache volumes optional

core/cmd/server/initialize.go

@@ -22,10 +22,12 @@ package server
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 
 	"github.com/cloudnative-pg/machinery/pkg/log"
 
 	"github.com/cloudnative-pg/klio/core/cmd/initialize"
+	"github.com/cloudnative-pg/klio/core/internal/kopia"
 	"github.com/cloudnative-pg/klio/core/internal/tier2"
 	"github.com/cloudnative-pg/klio/core/pkg/config"
 )
@@ -46,7 +48,26 @@ func initializeRepository(ctx context.Context, opts serverOpts) error {
 	return nil
 }
 
+// reclaimStaleCache removes a Kopia cache directory left behind on another
+// volume by a cache that moved. Kopia rebuilds the cache on demand, so the
+// leftover is only wasted space.
+func reclaimStaleCache(ctx context.Context, stale, inUse string) error {
+	if stale == "" || filepath.Clean(stale) == filepath.Clean(inUse) {
+		return nil
+	}
+
+	log.FromContext(ctx).Info("Reclaiming stale Kopia cache directory", "directory", stale)
+
+	return kopia.CleanupCacheDirectory(stale)
+}
+
 func initializeTier1(ctx context.Context, cfg *config.ServerConfig) error {
+	if err := reclaimStaleCache(
+		ctx, cfg.Tier1.Base.StaleCacheDirectory, cfg.Tier1.Base.CacheDirectory,
+	); err != nil {
+		return err
+	}
+
 	walDirectory := cfg.Tier1.Wal.WALPath
 	kopiaDirectory := cfg.Tier1.Base.RepositoryDirectory
 
@@ -60,6 +81,12 @@ func initializeTier1(ctx context.Context, cfg *config.ServerConfig) error {
 }
 
 func initializeTier2(ctx context.Context, cfg *config.ServerConfig) error {
+	if err := reclaimStaleCache(
+		ctx, cfg.Tier2.StaleCacheDirectory, cfg.Tier2.CacheDirectory,
+	); err != nil {
+		return err
+	}
+
 	tier2BaseFS, err := tier2.ConnectBase(ctx, &cfg.Tier2)
 	if err != nil {
 		return fmt.Errorf("error while connecting to tier2 (base): %w", err)

core/cmd/server/initialize_test.go

@@ -0,0 +1,69 @@
+/*
+Copyright © contributors to CloudNativePG, established as
+CloudNativePG a Series of LF Projects, LLC.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package server
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newCacheDirectory(t *testing.T, parent, name string) string {
+	t.Helper()
+
+	dir := filepath.Join(parent, name)
+	require.NoError(t, os.MkdirAll(dir, 0o750))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "blob"), []byte("cached"), 0o600))
+
+	return dir
+}
+
+func TestReclaimStaleCacheRemovesTheUnusedDirectory(t *testing.T) {
+	root := t.TempDir()
+	stale := newCacheDirectory(t, root, "stale")
+	inUse := newCacheDirectory(t, root, "in-use")
+
+	require.NoError(t, reclaimStaleCache(context.Background(), stale, inUse))
+
+	assert.NoDirExists(t, stale)
+	assert.DirExists(t, inUse)
+}
+
+func TestReclaimStaleCacheKeepsTheDirectoryInUse(t *testing.T) {
+	root := t.TempDir()
+	inUse := newCacheDirectory(t, root, "in-use")
+
+	// The two paths point at the same directory, spelled differently.
+	require.NoError(t, reclaimStaleCache(context.Background(), inUse+"/", inUse))
+
+	assert.DirExists(t, inUse)
+}
+
+func TestReclaimStaleCacheIsANoOpWhenUnset(t *testing.T) {
+	inUse := newCacheDirectory(t, t.TempDir(), "in-use")
+
+	require.NoError(t, reclaimStaleCache(context.Background(), "", inUse))
+
+	assert.DirExists(t, inUse)
+}

core/pkg/config/server.go

@@ -98,6 +98,11 @@ type Tier2Config struct {
 	// CacheDirectory is the directory of the Kopia cache
 	CacheDirectory string `mapstructure:"cache"`
 
+	// StaleCacheDirectory is a cache directory that is no longer in use and
+	// whose space must be reclaimed at startup. It is set when the cache moved
+	// to a different volume. Empty when there is nothing to reclaim.
+	StaleCacheDirectory string `mapstructure:"stale_cache"`
+
 	// S3 contains the configuration parameters for an S3-based tier 2
 	S3 S3Configuration `json:"s3" mapstructure:"s3"`
 }
@@ -108,6 +113,11 @@ type BaseServerConfig struct {
 	// CacheDirectory is the directory of the Kopia cache
 	CacheDirectory string `mapstructure:"cache"`
 
+	// StaleCacheDirectory is a cache directory that is no longer in use and
+	// whose space must be reclaimed at startup. It is set when the cache moved
+	// to a different volume. Empty when there is nothing to reclaim.
+	StaleCacheDirectory string `mapstructure:"stale_cache"`
+
 	// RepositoryDirectory is the directory where the Kopia repository is stored.
 	RepositoryDirectory string `mapstructure:"repository"`
 

documentation/web/docs/developer/running-e2e-tests.md

@@ -128,7 +128,8 @@ The E2E tests are located in `operator/test/e2e/` and include:
   driven by backup completion rather than a client command
   (`WALRetentionQueueAwareness`)
 - **`server_reconfig_test.go`** - Adding tier2 storage to an existing
-  tier1+queue server (`ServerTierReconfiguration`)
+  tier1+queue server, then removing and restoring the dedicated cache
+  volumes (`ServerTierReconfiguration`)
 - **`pluginconfiguration_update_test.go`** - PluginConfiguration updates
   and sidecar restart behavior (`PluginConfigurationUpdate`)
 - **`pvc_resize_test.go`** - PVC resize for data, cache, and queue

documentation/web/docs/user/api/_klio_api.md

@@ -362,7 +362,7 @@ _Appears in:_
 
 | Field | Description | Required | Default | Validation |
 | --- | --- | --- | --- | --- |
-| `cache` _[Cache](#cache)_ | Cache is the configuration of the PVC that should be<br />used for the cache. | True |  |  |
+| `cache` _[Cache](#cache)_ | Cache is the configuration of the PVC that should be<br />used for the cache. When omitted, the Kopia cache is stored in a<br />directory inside the tier1 data volume. |  |  | Optional: \{\} <br /> |
 | `data` _[Data](#data)_ | Data is the configuration of the PVC that should be used<br />for the base backups. | True |  |  |
 | `encryptionKeyFile` _[FileSource](#filesource)_ | EncryptionKeyFile specifies the Age-encrypted encryption key file. | True |  | ExactlyOneOf: [fileReference] <br /> |
 | `identityFile` _[FileSource](#filesource)_ | IdentityFile specifies the Age identity (private key) file used to<br />decrypt the encryption key. | True |  | ExactlyOneOf: [fileReference] <br /> |
@@ -397,7 +397,7 @@ _Appears in:_
 
 | Field | Description | Required | Default | Validation |
 | --- | --- | --- | --- | --- |
-| `cache` _[Cache](#cache)_ | Cache is the configuration of the PVC that should be<br />used for the cache. | True |  |  |
+| `cache` _[Cache](#cache)_ | Cache is the configuration of the PVC that should be<br />used for the cache. When omitted, the Kopia cache is stored in a<br />directory inside the tier1 data volume, and is therefore required<br />when tier1 is not configured. |  |  | Optional: \{\} <br /> |
 | `s3` _[S3Configuration](#s3configuration)_ | S3 contains the configuration parameters for an S3-based tier 2. | True |  |  |
 | `encryptionKeyFile` _[FileSource](#filesource)_ | EncryptionKeyFile specifies the Age-encrypted encryption key file. | True |  | ExactlyOneOf: [fileReference] <br /> |
 | `identityFile` _[FileSource](#filesource)_ | IdentityFile specifies the Age identity (private key) file used to<br />decrypt the encryption key. | True |  | ExactlyOneOf: [fileReference] <br /> |

documentation/web/docs/user/klio_server.md

@@ -74,9 +74,20 @@ The following factors should be considered when defining the PVC size:
 
 ### Cache PVCs
 
-The cache PVCs (one for Tier 1 and Tier 2 each) are used by Kopia for its
-[caching operations](https://kopia.io/docs/advanced/caching/).
-They are used to speed up snapshot operations.
+Kopia keeps a [cache](https://kopia.io/docs/advanced/caching/) to speed up
+snapshot operations. Each tier has its own cache, and the optional
+`tier1.cache` and `tier2.cache` stanzas dedicate a PVC to it.
+
+When a tier has no `cache` stanza, its cache is stored inside the Tier 1 data
+volume, under `/data/cache_tier1` and `/data/cache_tier2` respectively. This is
+the simplest configuration and the one to start from: size the data PVC so that
+it accommodates the cache as well. Dedicate a cache PVC when you want the cache
+on a different storage class, or when you want its growth to be unable to eat
+into the space of your backups.
+
+Since the fallback location lives in the Tier 1 data volume, `tier2.cache` is
+required on a [read-only server](#read-only-mode), where Tier 1 is not
+configured.
 
 :::warning
 Klio is currently limited to use the default cache size when creating a Kopia
@@ -86,6 +97,17 @@ so users should have a space buffer to account for this additional space.
 This limitation will be removed in a future version.
 :::
 
+#### Moving a cache between volumes
+
+The `cache` stanza can be added to, or removed from, a running server. The
+operator recreates the StatefulSet, which restarts the server pod, and Kopia
+rebuilds the cache in its new location on demand: no backup or WAL file is
+affected, only the first operations after the restart are slower.
+
+The old location is reclaimed without manual intervention. The PVC of a cache
+that is no longer dedicated is deleted once the pod no longer mounts it, and a
+cache left behind in the data volume is removed when the server starts.
+
 ### Queue PVC
 
 The queue PVC is required when Tier 1 is configured. It stores the NATS

documentation/web/docs/user/managing_storage.md

@@ -126,6 +126,12 @@ Apply the updated Server resource:
 kubectl apply -f klio-server.yaml
 ```
 
+:::note
+`cache` is optional. When a tier has no dedicated cache volume its cache lives
+inside the data PVC, so expanding `tier1.data` covers it too. See
+[Cache PVCs](klio_server.md#cache-pvcs).
+:::
+
 #### What Happens During Resize
 
 When you update the Server spec with larger PVC sizes, the following

documentation/web/docs/user/quickstart.md

@@ -256,23 +256,16 @@ spec:
   caSecretName: klio-server-ca
 
   tier1:
-    # Kopia cache. The default Kopia cache is 5 GB of content plus
-    # 5 GB of metadata, so leave some headroom.
-    cache:
-      pvcTemplate:
-        accessModes:
-          - ReadWriteOnce
-        resources:
-          requests:
-            storage: 10Gi
-    # Base backups and the WAL archive
+    # Base backups, the WAL archive and, with no dedicated cache volume, the
+    # Kopia cache. The default Kopia cache is 5 GB of content plus 5 GB of
+    # metadata, so leave some headroom on top of your backup size.
     data:
       pvcTemplate:
         accessModes:
           - ReadWriteOnce
         resources:
           requests:
-            storage: 20Gi
+            storage: 30Gi
     encryptionKeyFile:
       fileReference:
         volume:

operator/api/v1alpha1/server_types.go

@@ -41,6 +41,7 @@ const (
 // +kubebuilder:validation:XValidation:rule="!(self.mode == 'read-only' && has(self.tier1))",message="tier1 cannot be set when mode is read-only"
 // +kubebuilder:validation:XValidation:rule="!(self.mode == 'read-only' && has(self.queue))",message="queue cannot be set when mode is read-only"
 // +kubebuilder:validation:XValidation:rule="self.mode == 'read-only' || has(self.queue)",message="queue is required when tier1 is configured"
+// +kubebuilder:validation:XValidation:rule="!has(self.tier2) || has(self.tier1) || has(self.tier2.cache)",message="tier2.cache is required when tier1 is not configured"
 type ServerSpec struct {
 	// ImageConfiguration tells how to download the Klio
 	// image.
@@ -178,8 +179,10 @@ type FileSource struct {
 // Tier1Configuration is the tier 1 configuration.
 type Tier1Configuration struct {
 	// Cache is the configuration of the PVC that should be
-	// used for the cache.
-	Cache Cache `json:"cache"`
+	// used for the cache. When omitted, the Kopia cache is stored in a
+	// directory inside the tier1 data volume.
+	// +optional
+	Cache *Cache `json:"cache,omitempty"`
 
 	// Data is the configuration of the PVC that should be used
 	// for the base backups.
@@ -196,8 +199,11 @@ type Tier1Configuration struct {
 // Tier2Configuration is the tier 2 configuration.
 type Tier2Configuration struct {
 	// Cache is the configuration of the PVC that should be
-	// used for the cache.
-	Cache Cache `json:"cache"`
+	// used for the cache. When omitted, the Kopia cache is stored in a
+	// directory inside the tier1 data volume, and is therefore required
+	// when tier1 is not configured.
+	// +optional
+	Cache *Cache `json:"cache,omitempty"`
 
 	// S3 contains the configuration parameters for an S3-based tier 2.
 	S3 *S3Configuration `json:"s3"`
@@ -254,8 +260,8 @@ type ServerStatus struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec.tier1) || !has(self.spec.tier1) || !('storage' in oldSelf.spec.tier1.data.pvcTemplate.resources.requests) || !('storage' in self.spec.tier1.data.pvcTemplate.resources.requests) || !quantity(self.spec.tier1.data.pvcTemplate.resources.requests['storage']).isLessThan(quantity(oldSelf.spec.tier1.data.pvcTemplate.resources.requests['storage']))",message="tier1.data PVC size cannot be decreased"
-// +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec.tier1) || !has(self.spec.tier1) || !('storage' in oldSelf.spec.tier1.cache.pvcTemplate.resources.requests) || !('storage' in self.spec.tier1.cache.pvcTemplate.resources.requests) || !quantity(self.spec.tier1.cache.pvcTemplate.resources.requests['storage']).isLessThan(quantity(oldSelf.spec.tier1.cache.pvcTemplate.resources.requests['storage']))",message="tier1.cache PVC size cannot be decreased"
-// +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec.tier2) || !has(self.spec.tier2) || !('storage' in oldSelf.spec.tier2.cache.pvcTemplate.resources.requests) || !('storage' in self.spec.tier2.cache.pvcTemplate.resources.requests) || !quantity(self.spec.tier2.cache.pvcTemplate.resources.requests['storage']).isLessThan(quantity(oldSelf.spec.tier2.cache.pvcTemplate.resources.requests['storage']))",message="tier2.cache PVC size cannot be decreased"
+// +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec.tier1) || !has(self.spec.tier1) || !has(oldSelf.spec.tier1.cache) || !has(self.spec.tier1.cache) || !('storage' in oldSelf.spec.tier1.cache.pvcTemplate.resources.requests) || !('storage' in self.spec.tier1.cache.pvcTemplate.resources.requests) || !quantity(self.spec.tier1.cache.pvcTemplate.resources.requests['storage']).isLessThan(quantity(oldSelf.spec.tier1.cache.pvcTemplate.resources.requests['storage']))",message="tier1.cache PVC size cannot be decreased"
+// +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec.tier2) || !has(self.spec.tier2) || !has(oldSelf.spec.tier2.cache) || !has(self.spec.tier2.cache) || !('storage' in oldSelf.spec.tier2.cache.pvcTemplate.resources.requests) || !('storage' in self.spec.tier2.cache.pvcTemplate.resources.requests) || !quantity(self.spec.tier2.cache.pvcTemplate.resources.requests['storage']).isLessThan(quantity(oldSelf.spec.tier2.cache.pvcTemplate.resources.requests['storage']))",message="tier2.cache PVC size cannot be decreased"
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec.queue) || !has(self.spec.queue) || !('storage' in oldSelf.spec.queue.pvcTemplate.resources.requests) || !('storage' in self.spec.queue.pvcTemplate.resources.requests) || !quantity(self.spec.queue.pvcTemplate.resources.requests['storage']).isLessThan(quantity(oldSelf.spec.queue.pvcTemplate.resources.requests['storage']))",message="queue PVC size cannot be decreased"
 
 // Server is the Schema for the servers API.