feat(k8s): external S3 for LakeFS & Lakekeeper, MinIO optional

bin/k8s/Chart.yaml

@@ -49,6 +49,7 @@ dependencies:
   - name: minio
     version: 15.0.7
     repository: https://charts.bitnami.com/bitnami
+    condition: minio.enabled
 
   - name: lakefs
     version: 1.8.1

bin/k8s/templates/base/external-names/external-names.yaml

@@ -73,13 +73,16 @@ to access services in the main namespace using the same service names.
   "externalName" (printf "%s-svc.%s.svc.cluster.local" .Values.webserver.name $namespace)
 ) | nindent 0 }}
 
+{{- if .Values.minio.enabled }}
 ---
-{{/* MinIO ExternalName */}}
-{{- include "external-name-service" (dict 
+{{/* MinIO ExternalName -- only when the in-cluster MinIO is enabled; with an
+     external S3 store the CU pods reach it directly via STORAGE_S3_ENDPOINT. */}}
+{{- include "external-name-service" (dict
   "name" (printf "%s-minio" .Release.Name)
   "namespace" $workflowComputingUnitPoolNamespace
   "externalName" (printf "%s-minio.%s.svc.cluster.local" .Release.Name $namespace)
 ) | nindent 0 }}
+{{- end }}
 
 ---
 {{/* Lakekeeper ExternalName */}}

bin/k8s/templates/base/lakekeeper/lakekeeper-init-job.yaml

@@ -33,17 +33,17 @@ spec:
           image: alpine:3.19
           env:
             - name: STORAGE_S3_ENDPOINT
-              value: http://{{ .Release.Name }}-minio:9000
+              value: {{ include "texera.s3.endpoint" . }}
             - name: STORAGE_S3_AUTH_USERNAME
               valueFrom:
                 secretKeyRef:
-                  name: {{ .Release.Name }}-minio
-                  key: root-user
+                  name: {{ include "texera.s3.secretName" . }}
+                  key: {{ include "texera.s3.accessKeyIdKey" . }}
             - name: STORAGE_S3_AUTH_PASSWORD
               valueFrom:
                 secretKeyRef:
-                  name: {{ .Release.Name }}-minio
-                  key: root-password
+                  name: {{ include "texera.s3.secretName" . }}
+                  key: {{ include "texera.s3.secretAccessKeyKey" . }}
             - name: STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET
               value: {{ .Values.lakekeeperInit.warehouse.s3Bucket | quote }}
             - name: STORAGE_ICEBERG_CATALOG_REST_REGION
@@ -85,14 +85,18 @@ spec:
                 sleep 3
               done
 
-              echo "Step 1: Initializing MinIO bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}'..."
+{{- if .Values.lakekeeperInit.createBucket }}
+              echo "Step 1: Initializing object-store bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}'..."
               mc alias set minio "${STORAGE_S3_ENDPOINT}" "${STORAGE_S3_AUTH_USERNAME}" "${STORAGE_S3_AUTH_PASSWORD}" || true
               if mc ls minio/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET} > /dev/null 2>&1; then
-                echo "MinIO bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' already exists."
+                echo "Bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' already exists."
               else
-                mc mb minio/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}
-                echo "MinIO bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' created successfully."
+                mc mb minio/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET} || true
+                echo "Bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' created successfully."
               fi
+{{- else }}
+              echo "Step 1: skipping bucket creation (createBucket=false; using pre-existing external S3 bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}')."
+{{- end }}
 
               echo "Step 2: Initializing default project..."
               PROJECT_PAYLOAD="{\"project-id\":\"${LAKEKEEPER_PROJECT_ID}\",\"project-name\":\"${LAKEKEEPER_PROJECT_NAME}\"}"
@@ -112,9 +116,14 @@ spec:
                   "type": "s3",
                   "bucket": "${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}",
                   "region": "${STORAGE_ICEBERG_CATALOG_REST_REGION}",
+{{- if .Values.lakekeeperInit.warehouse.keyPrefix }}
+                  "key-prefix": {{ .Values.lakekeeperInit.warehouse.keyPrefix | quote }},
+{{- end }}
+{{- if eq .Values.lakekeeperInit.warehouse.flavor "s3-compat" }}
                   "endpoint": "${STORAGE_S3_ENDPOINT}",
-                  "flavor": "s3-compat",
                   "path-style-access": true,
+{{- end }}
+                  "flavor": {{ .Values.lakekeeperInit.warehouse.flavor | quote }},
                   "sts-enabled": false
                 },
                 "storage-credential": {

bin/k8s/templates/on-prem/minio-persistence.yaml

@@ -19,7 +19,7 @@
 {{/* This path only works for local-path storage class */}}
 {{- $hostBasePath := .Values.persistence.minioHostLocalPath }}
 
-{{- if .Values.minio.persistence.enabled }}
+{{- if and .Values.minio.enabled .Values.minio.persistence.enabled }}
 {{- $name := "minio" }}
 {{- $persistence := .Values.minio.persistence }}
 {{- $volumeName := printf "%s-data-pv" $name }}

bin/k8s/values-aws.yaml

@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ---------------------------------------------------------------------------
+# Example overlay for running Texera against an external S3 store (e.g. AWS S3)
+# instead of the bundled in-cluster MinIO.
+#
+#   helm install texera bin/k8s -f bin/k8s/values-aws.yaml
+#
+# Replace the placeholder bucket names, region, and credentials below with your
+# own. The buckets must already exist (the init job does not create them).
+# ---------------------------------------------------------------------------
+
+# Turn off the bundled MinIO object store; the services talk to external S3.
+minio:
+  enabled: false
+  persistence:
+    enabled: false
+
+# External S3 object store shared by the app tier, LakeFS, and Lakekeeper.
+storage:
+  s3:
+    # Regional S3 endpoint. For AWS S3 use https://s3.<region>.amazonaws.com.
+    endpoint: "https://s3.us-west-2.amazonaws.com"
+    region: "us-west-2"
+    # Recommended: reference a pre-created Secret with keys access-key-id /
+    # secret-access-key (e.g. one synced from AWS Secrets Manager), and leave
+    # accessKeyId / secretAccessKey empty. Otherwise the chart creates a Secret
+    # named "<release>-s3-credentials" from the inline values below.
+    existingSecret: ""
+    accessKeyId: "REPLACE_WITH_ACCESS_KEY_ID"
+    secretAccessKey: "REPLACE_WITH_SECRET_ACCESS_KEY"
+
+# Point LakeFS at external S3: a region-only blockstore (the AWS SDK resolves
+# the endpoint from the region) with credentials injected via the same Secret
+# the chart/app uses. Override the secretKeyRef name if you set
+# storage.s3.existingSecret above.
+lakefs:
+  lakefsConfig: |
+    database:
+      type: postgres
+    blockstore:
+      type: s3
+      s3:
+        region: us-west-2
+        pre_signed_expiry: 15m
+  extraEnvVars:
+    - name: AWS_ACCESS_KEY_ID
+      valueFrom:
+        secretKeyRef:
+          name: texera-s3-credentials
+          key: access-key-id
+    - name: AWS_SECRET_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: texera-s3-credentials
+          key: secret-access-key
+
+# Lakekeeper Iceberg warehouse on external S3: skip bucket creation (the bucket
+# pre-exists and the IAM principal may lack CreateBucket) and use the native
+# AWS S3 storage-profile flavor (no custom endpoint / path-style access).
+lakekeeperInit:
+  createBucket: false
+  warehouse:
+    region: us-west-2
+    s3Bucket: texera-iceberg
+    flavor: aws

bin/k8s/values.yaml

@@ -78,6 +78,10 @@ storage:
     secretAccessKey: ""
 
 minio:
+  # Set to false to disable the in-cluster MinIO and point the services at an
+  # external S3 store instead (configure storage.s3 above and the lakefs/
+  # lakekeeperInit blocks below). See values-aws.yaml for a complete example.
+  enabled: true
   mode: standalone
   image:
     repository: bitnamilegacy/minio
@@ -102,6 +106,9 @@ minio:
     existingClaim: "minio-data-pvc"
 
 lakefs:
+  # The blockstore below points LakeFS at the in-cluster MinIO. To use an
+  # external S3 store, override lakefsConfig (region-only blockstore) and inject
+  # the S3 credentials via extraEnvVars -- see values-aws.yaml for an example.
   secrets:
     authEncryptSecretKey: random_string_for_lakefs
     databaseConnectionString: postgres://postgres:root_password@texera-postgresql:5432/texera_lakefs?sslmode=disable
@@ -149,13 +156,23 @@ lakekeeper:
 
 lakekeeperInit:
   enabled: true
+  # Create the object-store bucket from the init job (true for in-cluster MinIO).
+  # Set false when pointing at a pre-existing external bucket (e.g. AWS S3) whose
+  # IAM principal can't CreateBucket.
+  createBucket: true
   defaultProject:
     id: "00000000-0000-0000-0000-000000000000"
     name: default
   warehouse:
     name: texera
     region: us-west-2
     s3Bucket: texera-iceberg
+    # Lakekeeper S3 storage-profile flavor: "s3-compat" (MinIO/other; adds
+    # endpoint + path-style-access) or "aws" (real AWS S3; omits both).
+    flavor: s3-compat
+    # Optional object-key prefix to isolate the Iceberg warehouse within a
+    # shared bucket. Empty = bucket root.
+    keyPrefix: ""
 
 # Part2: configurations of Texera-related micro services
 texeraImages: