From a406a05e9a1ed4174e321e1ac459621a82c74365 Mon Sep 17 00:00:00 2001 From: ali Date: Thu, 25 Jun 2026 10:11:08 -0700 Subject: [PATCH] feat(k8s): make in-cluster MinIO optional with external S3 for LakeFS & Lakekeeper PR2 made the app tier (file-service, computing-unit-manager) S3-pluggable via the storage.s3 helpers. This completes the object-storage story so the whole chart can run against an external S3 store (e.g. AWS S3) instead of the bundled in-cluster MinIO, while keeping MinIO the default so on-prem is unchanged. - Chart.yaml: gate the minio subchart on `minio.enabled`. - values.yaml: add `minio.enabled: true`; add `lakekeeperInit.createBucket`, `warehouse.flavor`, and `warehouse.keyPrefix`; document the lakefs override. - lakekeeper-init-job: resolve S3 endpoint/credentials through the storage.s3 helpers instead of hardcoding the in-cluster MinIO; gate bucket creation on `createBucket`; drive the warehouse storage-profile from `flavor`/`keyPrefix` (`s3-compat` => endpoint + path-style; `aws` => neither). - external-names: only emit the MinIO ExternalName when `minio.enabled`. - minio-persistence: only emit the PV/PVC when `minio.enabled`. - values-aws.yaml: new example overlay (MinIO off, external S3, LakeFS region blockstore + injected creds, Lakekeeper warehouse with createBucket=false / flavor=aws). Default render is functionally unchanged (S3 env values still resolve to the in-cluster MinIO); `helm lint` passes for both the default and AWS value sets. Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/k8s/Chart.yaml | 1 + .../base/external-names/external-names.yaml | 7 +- .../base/lakekeeper/lakekeeper-init-job.yaml | 29 ++++--- .../templates/on-prem/minio-persistence.yaml | 2 +- bin/k8s/values-aws.yaml | 81 +++++++++++++++++++ bin/k8s/values.yaml | 17 ++++ 6 files changed, 124 insertions(+), 13 deletions(-) create mode 100644 bin/k8s/values-aws.yaml diff --git a/bin/k8s/Chart.yaml b/bin/k8s/Chart.yaml index 548d9e531e0..8cda7ee2276 100644 --- a/bin/k8s/Chart.yaml +++ b/bin/k8s/Chart.yaml @@ -49,6 +49,7 @@ dependencies: - name: minio version: 15.0.7 repository: https://charts.bitnami.com/bitnami + condition: minio.enabled - name: lakefs version: 1.8.1 diff --git a/bin/k8s/templates/base/external-names/external-names.yaml b/bin/k8s/templates/base/external-names/external-names.yaml index c79ce40f33d..6879b8d7f7e 100644 --- a/bin/k8s/templates/base/external-names/external-names.yaml +++ b/bin/k8s/templates/base/external-names/external-names.yaml @@ -73,13 +73,16 @@ to access services in the main namespace using the same service names. "externalName" (printf "%s-svc.%s.svc.cluster.local" .Values.webserver.name $namespace) ) | nindent 0 }} +{{- if .Values.minio.enabled }} --- -{{/* MinIO ExternalName */}} -{{- include "external-name-service" (dict +{{/* MinIO ExternalName -- only when the in-cluster MinIO is enabled; with an + external S3 store the CU pods reach it directly via STORAGE_S3_ENDPOINT. */}} +{{- include "external-name-service" (dict "name" (printf "%s-minio" .Release.Name) "namespace" $workflowComputingUnitPoolNamespace "externalName" (printf "%s-minio.%s.svc.cluster.local" .Release.Name $namespace) ) | nindent 0 }} +{{- end }} --- {{/* Lakekeeper ExternalName */}} diff --git a/bin/k8s/templates/base/lakekeeper/lakekeeper-init-job.yaml b/bin/k8s/templates/base/lakekeeper/lakekeeper-init-job.yaml index 0a3540b6055..5b35a3e46a8 100644 --- a/bin/k8s/templates/base/lakekeeper/lakekeeper-init-job.yaml +++ b/bin/k8s/templates/base/lakekeeper/lakekeeper-init-job.yaml @@ -33,17 +33,17 @@ spec: image: alpine:3.19 env: - name: STORAGE_S3_ENDPOINT - value: http://{{ .Release.Name }}-minio:9000 + value: {{ include "texera.s3.endpoint" . }} - name: STORAGE_S3_AUTH_USERNAME valueFrom: secretKeyRef: - name: {{ .Release.Name }}-minio - key: root-user + name: {{ include "texera.s3.secretName" . }} + key: {{ include "texera.s3.accessKeyIdKey" . }} - name: STORAGE_S3_AUTH_PASSWORD valueFrom: secretKeyRef: - name: {{ .Release.Name }}-minio - key: root-password + name: {{ include "texera.s3.secretName" . }} + key: {{ include "texera.s3.secretAccessKeyKey" . }} - name: STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET value: {{ .Values.lakekeeperInit.warehouse.s3Bucket | quote }} - name: STORAGE_ICEBERG_CATALOG_REST_REGION @@ -85,14 +85,18 @@ spec: sleep 3 done - echo "Step 1: Initializing MinIO bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}'..." +{{- if .Values.lakekeeperInit.createBucket }} + echo "Step 1: Initializing object-store bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}'..." mc alias set minio "${STORAGE_S3_ENDPOINT}" "${STORAGE_S3_AUTH_USERNAME}" "${STORAGE_S3_AUTH_PASSWORD}" || true if mc ls minio/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET} > /dev/null 2>&1; then - echo "MinIO bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' already exists." + echo "Bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' already exists." else - mc mb minio/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET} - echo "MinIO bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' created successfully." + mc mb minio/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET} || true + echo "Bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' created successfully." fi +{{- else }} + echo "Step 1: skipping bucket creation (createBucket=false; using pre-existing external S3 bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}')." +{{- end }} echo "Step 2: Initializing default project..." PROJECT_PAYLOAD="{\"project-id\":\"${LAKEKEEPER_PROJECT_ID}\",\"project-name\":\"${LAKEKEEPER_PROJECT_NAME}\"}" @@ -112,9 +116,14 @@ spec: "type": "s3", "bucket": "${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}", "region": "${STORAGE_ICEBERG_CATALOG_REST_REGION}", +{{- if .Values.lakekeeperInit.warehouse.keyPrefix }} + "key-prefix": {{ .Values.lakekeeperInit.warehouse.keyPrefix | quote }}, +{{- end }} +{{- if eq .Values.lakekeeperInit.warehouse.flavor "s3-compat" }} "endpoint": "${STORAGE_S3_ENDPOINT}", - "flavor": "s3-compat", "path-style-access": true, +{{- end }} + "flavor": {{ .Values.lakekeeperInit.warehouse.flavor | quote }}, "sts-enabled": false }, "storage-credential": { diff --git a/bin/k8s/templates/on-prem/minio-persistence.yaml b/bin/k8s/templates/on-prem/minio-persistence.yaml index 94d22e5e3a8..4073c32582c 100644 --- a/bin/k8s/templates/on-prem/minio-persistence.yaml +++ b/bin/k8s/templates/on-prem/minio-persistence.yaml @@ -19,7 +19,7 @@ {{/* This path only works for local-path storage class */}} {{- $hostBasePath := .Values.persistence.minioHostLocalPath }} -{{- if .Values.minio.persistence.enabled }} +{{- if and .Values.minio.enabled .Values.minio.persistence.enabled }} {{- $name := "minio" }} {{- $persistence := .Values.minio.persistence }} {{- $volumeName := printf "%s-data-pv" $name }} diff --git a/bin/k8s/values-aws.yaml b/bin/k8s/values-aws.yaml new file mode 100644 index 00000000000..e46726de5b5 --- /dev/null +++ b/bin/k8s/values-aws.yaml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# --------------------------------------------------------------------------- +# Example overlay for running Texera against an external S3 store (e.g. AWS S3) +# instead of the bundled in-cluster MinIO. +# +# helm install texera bin/k8s -f bin/k8s/values-aws.yaml +# +# Replace the placeholder bucket names, region, and credentials below with your +# own. The buckets must already exist (the init job does not create them). +# --------------------------------------------------------------------------- + +# Turn off the bundled MinIO object store; the services talk to external S3. +minio: + enabled: false + persistence: + enabled: false + +# External S3 object store shared by the app tier, LakeFS, and Lakekeeper. +storage: + s3: + # Regional S3 endpoint. For AWS S3 use https://s3..amazonaws.com. + endpoint: "https://s3.us-west-2.amazonaws.com" + region: "us-west-2" + # Recommended: reference a pre-created Secret with keys access-key-id / + # secret-access-key (e.g. one synced from AWS Secrets Manager), and leave + # accessKeyId / secretAccessKey empty. Otherwise the chart creates a Secret + # named "-s3-credentials" from the inline values below. + existingSecret: "" + accessKeyId: "REPLACE_WITH_ACCESS_KEY_ID" + secretAccessKey: "REPLACE_WITH_SECRET_ACCESS_KEY" + +# Point LakeFS at external S3: a region-only blockstore (the AWS SDK resolves +# the endpoint from the region) with credentials injected via the same Secret +# the chart/app uses. Override the secretKeyRef name if you set +# storage.s3.existingSecret above. +lakefs: + lakefsConfig: | + database: + type: postgres + blockstore: + type: s3 + s3: + region: us-west-2 + pre_signed_expiry: 15m + extraEnvVars: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: texera-s3-credentials + key: access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: texera-s3-credentials + key: secret-access-key + +# Lakekeeper Iceberg warehouse on external S3: skip bucket creation (the bucket +# pre-exists and the IAM principal may lack CreateBucket) and use the native +# AWS S3 storage-profile flavor (no custom endpoint / path-style access). +lakekeeperInit: + createBucket: false + warehouse: + region: us-west-2 + s3Bucket: texera-iceberg + flavor: aws diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index 64642b517a9..4651bae6947 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -78,6 +78,10 @@ storage: secretAccessKey: "" minio: + # Set to false to disable the in-cluster MinIO and point the services at an + # external S3 store instead (configure storage.s3 above and the lakefs/ + # lakekeeperInit blocks below). See values-aws.yaml for a complete example. + enabled: true mode: standalone image: repository: bitnamilegacy/minio @@ -102,6 +106,9 @@ minio: existingClaim: "minio-data-pvc" lakefs: + # The blockstore below points LakeFS at the in-cluster MinIO. To use an + # external S3 store, override lakefsConfig (region-only blockstore) and inject + # the S3 credentials via extraEnvVars -- see values-aws.yaml for an example. secrets: authEncryptSecretKey: random_string_for_lakefs databaseConnectionString: postgres://postgres:root_password@texera-postgresql:5432/texera_lakefs?sslmode=disable @@ -149,6 +156,10 @@ lakekeeper: lakekeeperInit: enabled: true + # Create the object-store bucket from the init job (true for in-cluster MinIO). + # Set false when pointing at a pre-existing external bucket (e.g. AWS S3) whose + # IAM principal can't CreateBucket. + createBucket: true defaultProject: id: "00000000-0000-0000-0000-000000000000" name: default @@ -156,6 +167,12 @@ lakekeeperInit: name: texera region: us-west-2 s3Bucket: texera-iceberg + # Lakekeeper S3 storage-profile flavor: "s3-compat" (MinIO/other; adds + # endpoint + path-style-access) or "aws" (real AWS S3; omits both). + flavor: s3-compat + # Optional object-key prefix to isolate the Iceberg warehouse within a + # shared bucket. Empty = bucket root. + keyPrefix: "" # Part2: configurations of Texera-related micro services texeraImages: