Wire backend key.existingSecret into deployments (GH #34)

.github/workflows/functional-test.yaml

@@ -12,6 +12,8 @@ name: Functional Test (kind)
 #                              backends, so s3proxy starts and serves.
 #   azureblob               -> Azurite mock, full round-trip through the fixed
 #                              jclouds.endpoint (provider-agnostic).
+#   azureblob-existing-secret -> Azurite mock, account key delivered via a
+#                              pre-created Secret (key.existingSecret path, GH #34).
 #   multi-backend           -> s3 (MinIO) + azureblob (Azurite) + filesystem (PVC)
 #                              in ONE release, with per-backend bucket-locator
 #                              routing. assert-routing.sh writes one bucket per
@@ -59,6 +61,10 @@ jobs:
           - backend: azureblob
             values: ci/functional/values/azureblob.yaml
             mock: ci/functional/mocks/azurite.yaml
+          - backend: azureblob-existing-secret
+            values: ci/functional/values/azureblob-existing-secret.yaml
+            mock: ci/functional/mocks/azurite.yaml
+            azureKeySecret: true
           - backend: multi-backend
             values: ci/functional/values/multi-backend.yaml
             mock: ci/functional/mocks/minio.yaml ci/functional/mocks/azurite.yaml
@@ -105,6 +111,14 @@ jobs:
             --from-file=keystore.p12=/tmp/keystore.p12 \
             --from-literal=keystore-password=changeit
 
+      - name: Create Azure Blob key Secret (existingSecret leg)
+        if: matrix.azureKeySecret
+        # Azurite's devstoreaccount1 key in the Secret the chart references via
+        # azureblob.key.existingSecret (GH #34).
+        run: |
+          kubectl -n "${NAMESPACE}" create secret generic s3proxy-azureblob-key \
+            --from-literal=accountKey='Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='
+
       - name: Deploy mock backends (${{ matrix.backend }})
         if: matrix.mock
         # matrix.mock is a space-separated list of manifests (multi-backend

.github/workflows/lint-render.yaml

@@ -29,6 +29,7 @@ jobs:
           - test-values/transient.yaml
           - test-values/s3.yaml
           - test-values/azureblob.yaml
+          - test-values/azureblob-existing-secret.yaml
           - test-values/gcs.yaml
           - test-values/b2.yaml
           - test-values/openstack-swift.yaml

charts/s3proxy/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.1
+version: 0.4.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/s3proxy/templates/_helpers.tpl

@@ -27,3 +27,32 @@ secure-endpoint), otherwise "http". Shared by service.yaml and deployment.yaml
 {{- define "s3proxy.portName" -}}
   {{- ternary "https" "http" .Values.config.tls.enabled -}}
 {{- end }}
+
+{{- /*
+Backend credential existingSecrets as data ({name, backend, prop, secret, key}), so
+deployment.yaml renders the mounts/volumes with one range. <backend> matches the
+backend-<name>.properties filename; <prop> is the jclouds key the initContainer writes.
+*/}}
+{{- define "s3proxy.externalBackendSecrets" -}}
+{{- $b := .Values.config.backends -}}
+{{- $out := list -}}
+  {{- if and $b.s3.enabled $b.s3.secretAccessKey.existingSecret -}}
+    {{- $out = append $out (dict "name" "s3-credential" "backend" "s3" "prop" "jclouds.credential" "secret" $b.s3.secretAccessKey.existingSecret "key" $b.s3.secretAccessKey.secretKey) -}}
+  {{- end -}}
+  {{- if and $b.azureblob.enabled $b.azureblob.key.existingSecret -}}
+    {{- $out = append $out (dict "name" "azureblob-credential" "backend" "azureblob" "prop" "jclouds.credential" "secret" $b.azureblob.key.existingSecret "key" $b.azureblob.key.secretKey) -}}
+  {{- end -}}
+  {{- if and $b.azureblob.enabled $b.azureblob.sasToken.existingSecret -}}
+    {{- $out = append $out (dict "name" "azureblob-sas" "backend" "azureblob" "prop" "jclouds.azureblob.sas" "secret" $b.azureblob.sasToken.existingSecret "key" $b.azureblob.sasToken.secretKey) -}}
+  {{- end -}}
+  {{- if and $b.b2.enabled $b.b2.applicationKey.existingSecret -}}
+    {{- $out = append $out (dict "name" "b2-credential" "backend" "b2" "prop" "jclouds.credential" "secret" $b.b2.applicationKey.existingSecret "key" $b.b2.applicationKey.secretKey) -}}
+  {{- end -}}
+  {{- if and $b.openstackSwift.enabled $b.openstackSwift.password.existingSecret -}}
+    {{- $out = append $out (dict "name" "openstack-swift-credential" "backend" "openstack-swift" "prop" "jclouds.credential" "secret" $b.openstackSwift.password.existingSecret "key" $b.openstackSwift.password.secretKey) -}}
+  {{- end -}}
+  {{- if and $b.rackspaceCloudfiles.enabled $b.rackspaceCloudfiles.apiKey.existingSecret -}}
+    {{- $out = append $out (dict "name" "rackspace-cloudfiles-credential" "backend" "rackspace-cloudfiles" "prop" "jclouds.credential" "secret" $b.rackspaceCloudfiles.apiKey.existingSecret "key" $b.rackspaceCloudfiles.apiKey.secretKey) -}}
+  {{- end -}}
+{{- $out | toJson -}}
+{{- end }}

charts/s3proxy/templates/configmap.yaml

@@ -177,13 +177,11 @@ data:
   {{- if .Values.config.backends.azureblob.account }}
     jclouds.identity={{ .Values.config.backends.azureblob.account }}
   {{- end }}
-  {{- /* When a storage account key is set, jclouds.credential is merged in from the
-         Secret. Otherwise the azureblob-sdk provider authenticates with the SAS token
-         (jclouds.azureblob.sas, merged from the Secret) or the ambient Azure Workload
-         Identity credentials injected into the pod. S3Proxy still requires
-         jclouds.credential in every backend properties file (only filesystem-nio2,
-         transient-nio2 and google-cloud-storage-sdk are exempt), so emit it empty. */}}
-  {{- if not .Values.config.backends.azureblob.key.value }}
+  {{- /* With a key (key.value or key.existingSecret) the initContainer merges
+         jclouds.credential in from a Secret. S3Proxy requires the property in every
+         backend file, so emit it empty when neither is set (SAS-token or Azure
+         Workload Identity auth). */}}
+  {{- if and (not .Values.config.backends.azureblob.key.value) (not .Values.config.backends.azureblob.key.existingSecret) }}
     jclouds.credential=
   {{- end }}
   {{- range $index, $bucket := .Values.config.backends.azureblob.bucketLocators }}

charts/s3proxy/templates/deployment.yaml

@@ -83,6 +83,19 @@ spec:
                     cat "$backend_secret" >> "$output_file"
                   fi
 
+                  # Backend credentials from an existingSecret: each file under
+                  # /backend-secret/<backend>/ is named for the jclouds property it holds.
+                  backend_secret_dir="/backend-secret/${backend}"
+                  if [ -d "$backend_secret_dir" ]
+                  then
+                    for prop_file in "$backend_secret_dir"/*
+                    do
+                      [ -f "$prop_file" ] || continue
+                      echo "" >> "$output_file"  # Add newline separator
+                      printf '%s=%s\n' "$(basename "$prop_file")" "$(cat "$prop_file")" >> "$output_file"
+                    done
+                  fi
+
                   # TLS keystore password from an existing Secret (kept out of the
                   # ConfigMap). Mounted only when config.tls.keystorePassword.existingSecret
                   # is set; an inline password arrives via secret-common.properties instead.
@@ -108,6 +121,12 @@ spec:
             - name: tls-keystore-password
               mountPath: /tls-password
               readOnly: true
+{{- end }}
+{{- range $s := (include "s3proxy.externalBackendSecrets" . | fromJsonArray) }}
+            - name: {{ $s.name }}
+              mountPath: /backend-secret/{{ $s.backend }}/{{ $s.prop }}
+              subPath: {{ $s.prop }}
+              readOnly: true
 {{- end }}
       containers:
         - name: {{ .Chart.Name }}
@@ -272,6 +291,14 @@ spec:
               - key: {{ .Values.config.backends.googleCloudStorage.privateKey.secretKey }}
                 path: gcs-private.key
 {{- end }}
+{{- range $s := (include "s3proxy.externalBackendSecrets" . | fromJsonArray) }}
+        - name: {{ $s.name }}
+          secret:
+            secretName: {{ $s.secret }}
+            items:
+              - key: {{ $s.key }}
+                path: {{ $s.prop }}
+{{- end }}
 {{- with .Values.extraVolumes }}
         {{- toYaml . | nindent 8 }}
 {{- end }}

ci/functional/values/azureblob-existing-secret.yaml

@@ -0,0 +1,25 @@
+# Functional test: Azure Blob against the Azurite mock, account key from a
+# workflow-created Secret (key.existingSecret). End-to-end proof for GH #34 —
+# broken wiring leaves jclouds.credential empty and the round-trip 403s.
+config:
+  auth:
+    type: aws-v4
+    identity: test-access-key
+    secret: test-secret-key
+  backends:
+    filesystem:
+      enabled: false
+    azureblob:
+      enabled: true
+      provider: azureblob-sdk
+      account: devstoreaccount1
+      endpoint: http://azurite:10000/devstoreaccount1
+      key:
+        existingSecret: s3proxy-azureblob-key
+        secretKey: accountKey
+persistence:
+  enabled: false
+resources:
+  requests:
+    cpu: 100m
+    memory: 256Mi

test-values/azureblob-existing-secret.yaml

@@ -0,0 +1,19 @@
+# Render/lint scenario: Azure Blob account key from a pre-existing Secret instead
+# of an inline value (GH #34) — the externalBackendSecrets path.
+config:
+  auth:
+    type: aws-v4
+    identity: test-access-key
+    secret: test-secret-key
+  backends:
+    filesystem:
+      enabled: false
+    azureblob:
+      enabled: true
+      provider: azureblob-sdk
+      account: teststorageaccount
+      key:
+        existingSecret: my-azure-key
+        secretKey: accountKey
+persistence:
+  enabled: false