diff --git a/.gitignore b/.gitignore index 167d8389022f3..e01a728d18640 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ clusters/app.ci/.actual_diff hack/gcp-secret-manager/gcp-secrets-venv hack/gcp-secret-manager/.secret-manager-gcloud CLAUDE.local.md +.bob/ ### macOS ### .DS_Store cluster-profile-set-details.json diff --git a/ci-operator/config/openshift/secrets-store-csi-driver-operator/openshift-secrets-store-csi-driver-operator-main.yaml b/ci-operator/config/openshift/secrets-store-csi-driver-operator/openshift-secrets-store-csi-driver-operator-main.yaml index b44638d2df26e..f53be970ea5ba 100644 --- a/ci-operator/config/openshift/secrets-store-csi-driver-operator/openshift-secrets-store-csi-driver-operator-main.yaml +++ b/ci-operator/config/openshift/secrets-store-csi-driver-operator/openshift-secrets-store-csi-driver-operator-main.yaml @@ -293,6 +293,22 @@ tests: test: - ref: openshift-e2e-aws-csi-secrets-store-test workflow: openshift-e2e-aws-csi-secrets-store +- as: operator-e2e-s390x-vault-weekly + capabilities: + - intranet + cron: '@weekly' + steps: + allow_best_effort_post_steps: true + cluster_profile: libvirt-s390x-vpn + env: + ARCH: s390x + BRANCH: "5.0" + ETCD_DISK_SPEED: slow + USE_EXTERNAL_DNS: "true" + test: + - ref: openshift-e2e-libvirt-s390x-csi-secrets-store-setup-fbc + - ref: openshift-e2e-libvirt-s390x-csi-secrets-store-test-vault + workflow: openshift-e2e-libvirt-vpn - as: verify-deps steps: test: diff --git a/ci-operator/jobs/openshift/secrets-store-csi-driver-operator/openshift-secrets-store-csi-driver-operator-main-periodics.yaml b/ci-operator/jobs/openshift/secrets-store-csi-driver-operator/openshift-secrets-store-csi-driver-operator-main-periodics.yaml index e9df144af9bc1..66fc7d4d4226b 100644 --- a/ci-operator/jobs/openshift/secrets-store-csi-driver-operator/openshift-secrets-store-csi-driver-operator-main-periodics.yaml +++ b/ci-operator/jobs/openshift/secrets-store-csi-driver-operator/openshift-secrets-store-csi-driver-operator-main-periodics.yaml @@ -1,6 +1,6 @@ periodics: - agent: kubernetes - cluster: build11 + cluster: build12 cron: '@weekly' decorate: true decoration_config: @@ -86,3 +86,91 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build12 + cron: '@weekly' + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile.mustgather + - Dockerfile.openshift + extra_refs: + - base_ref: main + org: openshift + repo: secrets-store-csi-driver-operator + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile.mustgather + - Dockerfile.openshift + labels: + capability/intranet: intranet + ci-operator.openshift.io/cloud: libvirt-s390x-vpn + ci-operator.openshift.io/cloud-cluster-profile: libvirt-s390x-vpn + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-secrets-store-csi-driver-operator-main-operator-e2e-s390x-vault-weekly + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=operator-e2e-s390x-vault-weekly + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator diff --git a/ci-operator/step-registry/openshift/e2e/libvirt-s390x/OWNERS b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/OWNERS new file mode 100644 index 0000000000000..8bacff29c16f8 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/OWNERS @@ -0,0 +1,11 @@ +approvers: + - chiragkyal + - mytreya-rh + - sandisamp + - v78singh + +reviewers: + - chiragkyal + - mytreya-rh + - sandisamp + - v78singh diff --git a/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/OWNERS b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/OWNERS new file mode 100644 index 0000000000000..8bacff29c16f8 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/OWNERS @@ -0,0 +1,11 @@ +approvers: + - chiragkyal + - mytreya-rh + - sandisamp + - v78singh + +reviewers: + - chiragkyal + - mytreya-rh + - sandisamp + - v78singh diff --git a/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/OWNERS b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/OWNERS new file mode 100644 index 0000000000000..8bacff29c16f8 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/OWNERS @@ -0,0 +1,11 @@ +approvers: + - chiragkyal + - mytreya-rh + - sandisamp + - v78singh + +reviewers: + - chiragkyal + - mytreya-rh + - sandisamp + - v78singh diff --git a/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/setup-fbc/OWNERS b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/setup-fbc/OWNERS new file mode 100644 index 0000000000000..8bacff29c16f8 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/setup-fbc/OWNERS @@ -0,0 +1,11 @@ +approvers: + - chiragkyal + - mytreya-rh + - sandisamp + - v78singh + +reviewers: + - chiragkyal + - mytreya-rh + - sandisamp + - v78singh diff --git a/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/setup-fbc/openshift-e2e-libvirt-s390x-csi-secrets-store-setup-fbc-commands.sh b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/setup-fbc/openshift-e2e-libvirt-s390x-csi-secrets-store-setup-fbc-commands.sh new file mode 100644 index 0000000000000..e54fa3e78f78a --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/setup-fbc/openshift-e2e-libvirt-s390x-csi-secrets-store-setup-fbc-commands.sh @@ -0,0 +1,577 @@ +#!/bin/bash +# ============================================================================== +# Script to setup a custom FileBasedCatalog (FBC) CatalogSource on OpenShift +# for testing the secrets-store-csi-driver operator on s390x. +# +# This is adapted for CI environment with vault credentials. +# ============================================================================== + +set -euo pipefail + +# Setup unprivileged tool installation path +export PATH="/tmp/bin:${PATH}" +mkdir -p /tmp/bin + +# --- Configuration --- +# Check if TEST_GA_BUILD is set to true (case-insensitive) +TEST_GA_BUILD="${TEST_GA_BUILD:-false}" +if [[ "${TEST_GA_BUILD,,}" == "true" ]]; then + USE_GA_BUILD=true + echo "INFO: TEST_GA_BUILD is set to true - will use default catalog sources" +else + USE_GA_BUILD=false + echo "INFO: TEST_GA_BUILD is not set or false - will use custom FBC catalog" +fi + +OPERATOR_NAME="ose-secrets-store-csi-driver-rhel9-operator" +FBC_IMAGE_REPO="quay.io/redhat-user-workloads/ocp-art-tenant/art-fbc" +FBC_CATALOG_NAME="art-fbc-catalog" +MIRROR_REPO="quay.io/redhat-user-workloads/ocp-art-tenant/art-images-share" + +# Operator installation config +OPERATOR_NAMESPACE="openshift-cluster-csi-drivers" +OPERATORGROUP_NAME="secrets-store-csi-driver-og" +SUBSCRIPTION_NAME="secrets-store-csi-driver-operator" +SUBSCRIPTION_PACKAGE="secrets-store-csi-driver-operator" +SUBSCRIPTION_CHANNEL="stable" +CLUSTER_CSI_DRIVER_NAME="secrets-store.csi.k8s.io" +CSV_READY_TIMEOUT=300 # seconds to wait for CSV to reach Succeeded + +# Pull secret from deploy-konflux credential (same as AWS weekly test) +PULL_SECRET_FILE="/var/run/secrets/pull-secret/.dockerconfigjson" +CLUSTER_PULL_SECRET_NAME="pull-secret" +CLUSTER_PULL_SECRET_NAMESPACE="openshift-config" + +# Temp directory for oras pull output +WORK_DIR=$(mktemp -d) +trap 'rm -rf "${WORK_DIR}"' EXIT + +# --- Helper: print a section header --- +section() { + echo + echo "======================================================================" + echo " $*" + echo "======================================================================" +} + +# ============================================================================== +# STEP 0: Architecture + Dependency Check +# ============================================================================== +check_arch_and_deps() { + section "STEP 0: Architecture and Dependency Check" + + CLUSTER_ARCH=$(oc get nodes -o jsonpath='{.items[0].status.nodeInfo.architecture}' 2>/dev/null || echo "unknown") + echo "Detected cluster architecture: ${CLUSTER_ARCH}" + if [ "${CLUSTER_ARCH}" != "s390x" ]; then + echo "ERROR: This script is intended for s390x (IBM Z) clusters." + echo " Detected cluster architecture: ${CLUSTER_ARCH}. Exiting." + exit 1 + fi + echo "Architecture check passed: s390x cluster confirmed." + + echo + echo "Checking required tools..." + + # Verify tools already present in cli image + for tool in oc podman curl tar; do + if ! command -v "${tool}" &>/dev/null; then + echo "ERROR: Required tool '${tool}' not found in PATH." + exit 1 + fi + echo " [OK] ${tool}" + done + + # Install jq to /tmp/bin if not present + if ! command -v jq &>/dev/null; then + echo "Installing jq (amd64 for build farm pod)..." + JQ_VERSION="1.7.1" + JQ_URL="https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64" + curl -fsSL "${JQ_URL}" -o /tmp/bin/jq || { + echo "ERROR: Failed to download jq from ${JQ_URL}." + exit 1 + } + chmod +x /tmp/bin/jq + echo " [OK] jq $(jq --version)" + else + echo " [OK] jq $(jq --version)" + fi + + # Install oras to /tmp/bin if not present + if ! command -v oras &>/dev/null; then + echo "Installing oras (amd64 for build farm pod)..." + ORAS_VERSION="1.3.1" + ORAS_TARBALL="oras_${ORAS_VERSION}_linux_amd64.tar.gz" + ORAS_URL="https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/${ORAS_TARBALL}" + echo " Downloading: ${ORAS_URL}" + curl -fsSL "${ORAS_URL}" -o /tmp/oras.tar.gz || { + echo "ERROR: Failed to download oras from ${ORAS_URL}." + exit 1 + } + tar -xzf /tmp/oras.tar.gz -C /tmp/bin oras + chmod +x /tmp/bin/oras + rm -f /tmp/oras.tar.gz + echo " [OK] oras version: $(oras version 2>/dev/null || echo 'unknown')" + else + echo " [OK] oras version: $(oras version 2>/dev/null || echo 'unknown')" + fi + + echo + echo "All dependency checks passed." +} + +# ============================================================================== +# STEP 1: Extract Cluster Version and Build FBC Image Tag +# ============================================================================== +extract_cluster_version() { + section "STEP 1: Extracting OpenShift Cluster Version" + + OCP_FULL_VERSION=$(oc get clusterversion version \ + -o=jsonpath='{.status.desired.version}' 2>/dev/null) + + if [ -z "${OCP_FULL_VERSION}" ]; then + echo "ERROR: Could not retrieve cluster version." + exit 1 + fi + + # Extract 'X.Y' only (e.g. '4.21' from '4.21.3') + OCP_VERSION=$(echo "${OCP_FULL_VERSION}" | cut -d. -f1,2) + + if [ -z "${OCP_VERSION}" ]; then + echo "ERROR: Could not parse major.minor from version '${OCP_FULL_VERSION}'." + exit 1 + fi + + # Build the final FBC image reference + FINAL_FBC_IMAGE="${FBC_IMAGE_REPO}:ocp__${OCP_VERSION}__${OPERATOR_NAME}" + + echo "Cluster version : ${OCP_FULL_VERSION}" + echo "OCP version used : ${OCP_VERSION}" + echo "FBC image : ${FINAL_FBC_IMAGE}" +} + +# ============================================================================== +# STEP 2: Disable Default CatalogSources +# ============================================================================== +disable_default_catalogs() { + if [ "${USE_GA_BUILD}" = true ]; then + section "STEP 2: Skipping - Using Default CatalogSources (TEST_GA_BUILD=true)" + echo "Default CatalogSources will remain enabled for GA build testing." + return 0 + fi + + section "STEP 2: Disabling All Default CatalogSources via OperatorHub" + + oc patch operatorhub cluster --type=json \ + -p='[{"op":"add","path":"/spec/disableAllDefaultSources","value":true}]' + + echo "Default CatalogSources disabled." +} + +# ============================================================================== +# STEP 3: Merge Pull Secret +# ============================================================================== +merge_pull_secret() { + section "STEP 3: Merging Pull Secret into Cluster Pull Secret" + + if [ ! -f "${PULL_SECRET_FILE}" ]; then + echo "ERROR: Pull secret file not found at: ${PULL_SECRET_FILE}" + exit 1 + fi + + # Validate that the pull secret file is valid JSON + if ! jq empty "${PULL_SECRET_FILE}" 2>/dev/null; then + echo "ERROR: '${PULL_SECRET_FILE}' is not valid JSON." + exit 1 + fi + + echo "1. Fetching existing cluster pull secret..." + EXISTING_SECRET_B64=$(oc get secret "${CLUSTER_PULL_SECRET_NAME}" \ + -n "${CLUSTER_PULL_SECRET_NAMESPACE}" \ + -o=jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null) + + if [ -z "${EXISTING_SECRET_B64}" ]; then + echo "ERROR: Could not read cluster pull secret." + exit 1 + fi + + EXISTING_DOCKERCONFIG=$(echo "${EXISTING_SECRET_B64}" | base64 -d) + + echo "2. Merging new credentials into existing pull secret..." + NEW_DOCKERCONFIG=$(cat "${PULL_SECRET_FILE}") + + # Deep-merge: new values override existing keys + MERGED_DOCKERCONFIG=$(jq -s '.[0] * .[1]' \ + <(echo "${EXISTING_DOCKERCONFIG}") \ + <(echo "${NEW_DOCKERCONFIG}")) + + echo "3. Patching cluster pull secret..." + MERGED_B64=$(echo "${MERGED_DOCKERCONFIG}" | base64 -w 0) + + oc patch secret "${CLUSTER_PULL_SECRET_NAME}" \ + -n "${CLUSTER_PULL_SECRET_NAMESPACE}" \ + --type=json \ + -p="[{\"op\":\"replace\",\"path\":\"/data/.dockerconfigjson\",\"value\":\"${MERGED_B64}\"}]" + + echo "Pull secret merged successfully." +} + +# ============================================================================== +# STEP 4: Apply CatalogSource +# ============================================================================== +apply_catalog_source() { + if [ "${USE_GA_BUILD}" = true ]; then + section "STEP 4: Skipping - Not Creating Custom CatalogSource (TEST_GA_BUILD=true)" + return 0 + fi + + section "STEP 4: Creating CatalogSource '${FBC_CATALOG_NAME}'" + + cat </dev/null || true) + + if [ "${STATUS}" = "READY" ]; then + echo " CatalogSource is READY." + return 0 + fi + + echo " Current state: '${STATUS}' — waiting..." + sleep ${interval} + elapsed=$(( elapsed + interval )) + done + + echo "ERROR: CatalogSource did not reach READY state within ${timeout}s." + exit 1 +} + +# ============================================================================== +# STEP 7: Wait for MachineConfigPool rollout triggered by IDMS +# ============================================================================== +wait_for_mcp_rollout() { + if [ "${USE_GA_BUILD}" = true ]; then + section "STEP 7: Skipping - No MCP Rollout Needed (TEST_GA_BUILD=true)" + return 0 + fi + + section "STEP 7: Waiting for MachineConfigPool Rollout (triggered by IDMS)" + + echo "IDMS changes trigger a MachineConfig update which reboots nodes." + + echo "Waiting for MCP rollout to begin..." + for i in $(seq 1 60); do + MC=$(oc get mcp worker -o jsonpath='{.status.machineCount}' 2>/dev/null || echo "0") + UMC=$(oc get mcp worker -o jsonpath='{.status.updatedMachineCount}' 2>/dev/null || echo "0") + if [ "${MC}" != "${UMC}" ]; then + echo "MCP rollout started (${UMC}/${MC} updated)" + break + fi + sleep 10 + done + + echo "Waiting for all MCPs to finish updating..." + + local timeout=900 # 15 min + local elapsed=0 + local interval=30 + + while [ ${elapsed} -lt ${timeout} ]; do + UPDATING=$(oc get mcp -o=jsonpath='{range .items[*]}{.metadata.name}{" updating="}{.status.conditions[?(@.type=="Updating")].status}{"\n"}{end}' 2>/dev/null \ + | grep -c 'updating=True' || true) + DEGRADED=$(oc get mcp -o=jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Degraded")].status}{"\n"}{end}' 2>/dev/null \ + | grep -c 'True' || true) + + if [ "${DEGRADED}" -gt 0 ]; then + echo "ERROR: One or more MachineConfigPools are Degraded." + oc get mcp + exit 1 + fi + + if [ "${UPDATING}" -eq 0 ]; then + echo " All MachineConfigPools are up to date." + oc get mcp + return 0 + fi + + echo " ${UPDATING} MCP(s) still updating — waiting ${interval}s..." + sleep ${interval} + elapsed=$(( elapsed + interval )) + done + + echo "ERROR: MachineConfigPool rollout did not complete within ${timeout}s." + exit 1 +} + +# ============================================================================== +# STEP 8: Create Namespace + OperatorGroup + Subscription +# ============================================================================== +install_operator() { + section "STEP 8: Installing Operator via Subscription" + + if ! oc get namespace "${OPERATOR_NAMESPACE}" &>/dev/null; then + echo "Creating namespace '${OPERATOR_NAMESPACE}'..." + oc create namespace "${OPERATOR_NAMESPACE}" + else + echo "Namespace '${OPERATOR_NAMESPACE}' already exists." + fi + + echo + echo "1. Applying OperatorGroup..." + oc apply -f - </dev/null || true) + + if [ -n "${CSV_NAME}" ]; then + CSV_PHASE=$(oc get csv "${CSV_NAME}" \ + -n "${OPERATOR_NAMESPACE}" \ + -o=jsonpath='{.status.phase}' 2>/dev/null || true) + + echo " CSV: ${CSV_NAME} Phase: ${CSV_PHASE}" + + if [ "${CSV_PHASE}" = "Succeeded" ]; then + echo + echo "CSV reached Succeeded." + oc get csv "${CSV_NAME}" -n "${OPERATOR_NAMESPACE}" + return 0 + fi + + if [ "${CSV_PHASE}" = "Failed" ]; then + echo "ERROR: CSV '${CSV_NAME}' has Failed." + exit 1 + fi + else + echo " InstallPlan not yet created — waiting..." + fi + + sleep ${interval} + elapsed=$(( elapsed + interval )) + done + + echo "ERROR: CSV did not reach 'Succeeded' within ${CSV_READY_TIMEOUT}s." + exit 1 +} + +# ============================================================================== +# STEP 10: Create ClusterCSIDriver +# ============================================================================== +create_cluster_csi_driver() { + section "STEP 10: Creating ClusterCSIDriver '${CLUSTER_CSI_DRIVER_NAME}'" + + oc apply -f - </dev/null || echo "unknown") + echo "Detected cluster architecture: ${CLUSTER_ARCH}" + if [ "${CLUSTER_ARCH}" != "s390x" ]; then + echo "ERROR: This script is intended for s390x (IBM Z) clusters." + echo " Detected cluster architecture: ${CLUSTER_ARCH}. Exiting." + exit 1 + fi + echo "Architecture check passed: s390x cluster confirmed." + + echo + echo "Checking and installing required tools..." + + # Verify tools already present in cli image + for tool in oc curl tar git; do + if ! command -v "${tool}" &>/dev/null; then + echo "ERROR: Required tool '${tool}' not found in PATH." + exit 1 + fi + echo " [OK] ${tool}" + done + + # Install jq to /tmp/bin if not present + if ! command -v jq &>/dev/null; then + echo "Installing jq (amd64 for build farm pod)..." + JQ_VERSION="1.7.1" + JQ_URL="https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64" + curl -fsSL "${JQ_URL}" -o /tmp/bin/jq || { + echo "ERROR: Failed to download jq from ${JQ_URL}." + exit 1 + } + chmod +x /tmp/bin/jq + echo " [OK] jq $(jq --version)" + else + echo " [OK] jq $(jq --version)" + fi + + # Install bats to /tmp/bin if not present + if ! command -v bats &>/dev/null; then + echo "Installing bats (Bash Automated Testing System)..." + BATS_VERSION="1.11.0" + BATS_URL="https://github.com/bats-core/bats-core/archive/refs/tags/v${BATS_VERSION}.tar.gz" + curl -fsSL "${BATS_URL}" -o /tmp/bats.tar.gz || { + echo "ERROR: Failed to download bats from ${BATS_URL}." + exit 1 + } + tar -xzf /tmp/bats.tar.gz -C /tmp + bash /tmp/bats-core-${BATS_VERSION}/install.sh /tmp/bin + rm -rf /tmp/bats.tar.gz /tmp/bats-core-${BATS_VERSION} + echo " [OK] bats $(bats --version)" + else + echo " [OK] bats $(bats --version)" + fi + + # Install helm to /tmp/bin if not present + if ! command -v helm &>/dev/null; then + echo "Installing helm (amd64 for build farm pod)..." + HELM_VERSION="3.16.4" + HELM_TARBALL="helm-v${HELM_VERSION}-linux-amd64.tar.gz" + HELM_URL="https://get.helm.sh/${HELM_TARBALL}" + curl -fsSL "${HELM_URL}" -o /tmp/helm.tar.gz || { + echo "ERROR: Failed to download helm from ${HELM_URL}." + exit 1 + } + tar -xzf /tmp/helm.tar.gz -C /tmp + mv /tmp/linux-amd64/helm /tmp/bin/helm + chmod +x /tmp/bin/helm + rm -rf /tmp/helm.tar.gz /tmp/linux-amd64 + echo " [OK] helm $(helm version --short)" + else + echo " [OK] helm $(helm version --short)" + fi + + echo + echo "All dependency checks passed." +} + +# ============================================================================== +# STEP 1: Validate required files exist +# ============================================================================== +validate_files() { + section "STEP 1: Validating Required Files" + + local missing=0 + + if [ ! -f "${VAULT_LICENSE_STRING_FILE}" ]; then + echo "ERROR: Required file not found: ${VAULT_LICENSE_STRING_FILE}" + missing=$(( missing + 1 )) + else + echo " [OK] ${VAULT_LICENSE_STRING_FILE}" + # Validate it's not empty + if [ ! -s "${VAULT_LICENSE_STRING_FILE}" ]; then + echo "ERROR: '${VAULT_LICENSE_STRING_FILE}' is empty." + missing=$(( missing + 1 )) + else + echo " [OK] vault-license file is not empty" + fi + fi + + if [ "${missing}" -gt 0 ]; then + echo + echo "ERROR: ${missing} required file(s) missing or invalid. Exiting." + exit 1 + fi + + echo "All required files present and valid." +} + +# ============================================================================== +# STEP 3: Clone the secrets-store-csi-driver repo +# ============================================================================== +clone_repo() { + section "STEP 3: Cloning secrets-store-csi-driver Repository" + + if [ -d "${REPO_DIR}" ]; then + echo "Repository already exists at '${REPO_DIR}'. Pulling latest changes..." + git -C "${REPO_DIR}" pull --ff-only || { + echo "WARNING: Could not fast-forward pull. Using existing checkout." + } + else + echo "Cloning from ${REPO_URL}..." + git clone "${REPO_URL}" "${REPO_DIR}" + cd "${REPO_DIR}" + echo "Checking out branch for release ${BRANCH}..." + git checkout "release-${BRANCH}" || git checkout main + fi + + echo "Repository ready at: ${REPO_DIR}" +} + +# ============================================================================== +# STEP 4: Replace busybox image in test YAML files +# ============================================================================== +replace_busybox_images() { + section "STEP 4: Replacing Busybox Image in Test YAML Files" + + echo "Replacing:" + echo " FROM: ${OLD_BUSYBOX_IMAGE}" + echo " TO : ${NEW_BUSYBOX_IMAGE}" + echo + + local replaced=0 + local not_found=0 + + for rel_path in "${BUSYBOX_YAML_FILES[@]}"; do + full_path="${REPO_DIR}/${rel_path}" + + if [ ! -f "${full_path}" ]; then + echo " [MISSING] ${rel_path}" + not_found=$(( not_found + 1 )) + continue + fi + + if grep -qF "${OLD_BUSYBOX_IMAGE}" "${full_path}"; then + sed -i "s|${OLD_BUSYBOX_IMAGE}|${NEW_BUSYBOX_IMAGE}|g" "${full_path}" + echo " [REPLACED] ${rel_path}" + replaced=$(( replaced + 1 )) + elif grep -qF "${NEW_BUSYBOX_IMAGE}" "${full_path}"; then + echo " [ALREADY SET] ${rel_path}" + replaced=$(( replaced + 1 )) + else + echo " [WARNING] '${OLD_BUSYBOX_IMAGE}' not found in ${rel_path}" + not_found=$(( not_found + 1 )) + fi + done + + echo + echo "Image replacement complete: ${replaced} file(s) updated, ${not_found} file(s) not matched." + + if [ "${not_found}" -gt 0 ]; then + echo "WARNING: Some files were not patched. Continuing anyway." + fi +} + +# ============================================================================== +# STEP 5: Patch vault.bats — replace the helm install block +# ============================================================================== +patch_vault_bats() { + section "STEP 5: Patching ${BATS_FILE}" + + BATS_FULL_PATH="${REPO_DIR}/${BATS_FILE}" + + if [ ! -f "${BATS_FULL_PATH}" ]; then + echo "ERROR: ${BATS_FULL_PATH} not found." + exit 1 + fi + + # Check if the patch has already been applied + if grep -q "vault-enterprise" "${BATS_FULL_PATH}"; then + echo "vault.bats already contains 'vault-enterprise' — patch already applied, skipping." + return 0 + fi + + # Verify the old block exists + if ! grep -q "helm install vault hashicorp/vault --namespace=vault" "${BATS_FULL_PATH}"; then + echo "ERROR: Expected helm install block not found in ${BATS_FILE}." + exit 1 + fi + + echo "Applying vault license apply line and helm install block replacement..." + + # Use Python for reliable multi-line replacement + python3 - "${BATS_FULL_PATH}" "${VAULT_CREDS_DIR}" << 'PYEOF' +import sys, re + +path = sys.argv[1] +creds_dir = sys.argv[2] +with open(path, 'r') as f: + content = f.read() + +# Patch 1: Insert vault-license secret creation before "# install the vault provider" +license_secret_creation = f''' # Create vault-license secret from license string + oc create namespace vault || true + oc create secret generic vault-license \\ + --from-file=license={creds_dir}/vault-license \\ + -n vault --dry-run=client -o yaml | oc apply -f - + +''' +vault_comment = ' # install the vault provider using the helm charts' + +if 'vault-license' not in content or 'oc create secret' not in content: + content = content.replace( + vault_comment, + license_secret_creation + vault_comment + ) + +# Patch 2: Replace the old helm install block +old_helm_pattern = re.compile( + r' helm install vault hashicorp/vault --namespace=vault\s*\\\s*\n' + r'( --set [^\n]+\s*\\\s*\n)*' + r' --set "csi\.daemonSet\.providersDir=/var/run/secrets-store-csi-providers"', + re.MULTILINE +) + +new_helm_block = ( + ' helm install vault hashicorp/vault \\\n' + ' --namespace vault \\\n' + ' --create-namespace \\\n' + ' --set server.dev.enabled=true \\\n' + ' --set server.image.repository="docker.io/hashicorp/vault-enterprise" \\\n' + ' --set server.image.tag="1.20.9-ent" \\\n' + ' --set server.enterpriseLicense.secretName="vault-license" \\\n' + ' --set server.logLevel=debug \\\n' + ' --set server.serviceAccount.name="vault" \\\n' + ' --set "injector.enabled=false" \\\n' + ' --set global.openshift=true \\\n' + ' --set csi.enabled=true \\\n' + ' --set "csi.daemonSet.providersDir=/var/run/secrets-store-csi-providers" \\\n' + ' --set csi.image.repository="registry.connect.redhat.com/hashicorp/vault-csi-provider" \\\n' + ' --set csi.image.tag="1.7.1-ubi" \\\n' + ' --set csi.agent.enabled=false' +) + +new_content, n = old_helm_pattern.subn(new_helm_block, content) +if n == 0: + print("ERROR: helm install pattern not matched.") + sys.exit(1) + +with open(path, 'w') as f: + f.write(new_content) + +print("vault.bats patched successfully.") +PYEOF + + echo "Patch complete." + + echo + echo "Increasing all timeout values to 300s..." + sed -i 's/--timeout=[0-9]\+[sm]/--timeout=300s/g' "${BATS_FULL_PATH}" + echo "Timeout values updated." +} + +# ============================================================================== +# Logging setup +# ============================================================================== +LOG_FILE="${ARTIFACT_DIR}/vault-test-$(date '+%Y%m%d-%H%M%S').log" +exec > >(tee -a "${LOG_FILE}") 2>&1 + +echo "Log file: ${LOG_FILE}" + +# ============================================================================== +# collect_versions — record cluster + image versions +# ============================================================================== +collect_versions() { + section "Version and Image Information" + + echo "--- Timestamp ---" + date + + echo + echo "--- OpenShift Cluster Version ---" + oc get clusterversion version -o=jsonpath='{.status.desired.version}' 2>/dev/null || echo "(unavailable)" + + echo + echo "--- Node Architecture ---" + oc get nodes -o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.architecture}{"\t"}{.status.nodeInfo.osImage}{"\n"}{end}' 2>/dev/null || echo "(unavailable)" + + echo + echo "--- Secrets Store CSI Driver CSV ---" + oc get csv -n openshift-cluster-csi-drivers --no-headers 2>/dev/null | grep secrets-store || echo "(no CSV found)" + + echo + echo "--- Vault Helm Release ---" + helm list -n "${VAULT_NAMESPACE}" 2>/dev/null || echo "(helm list failed)" + + echo + echo "--- Vault Pod Images ---" + oc get pods -n "${VAULT_NAMESPACE}" -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{range .spec.containers[*]} image: {.image}{"\n"}{end}{end}' 2>/dev/null || echo "(no vault pods)" + + echo + echo "--- Busybox Image Used ---" + echo " ${NEW_BUSYBOX_IMAGE}" + + echo + echo "--- Tool Versions ---" + echo " oc : $(oc version --client 2>/dev/null | head -1)" + echo " helm : $(helm version --short 2>/dev/null)" + echo " bats : $(bats --version 2>/dev/null)" +} + +# ============================================================================== +# collect_failure_logs — gather pod logs for diagnostics +# ============================================================================== +collect_failure_logs() { + section "FAILURE DIAGNOSTICS — Collecting Pod Logs" + + echo "--- Timestamp of failure ---" + date + + for ns in default "${VAULT_NAMESPACE}"; do + echo + echo "=== Namespace: ${ns} ===" + + echo + echo "-- All pods --" + oc get pods -n "${ns}" 2>/dev/null || echo "(could not list pods)" + + echo + echo "-- Events (last 30) --" + oc get events -n "${ns}" --sort-by='.lastTimestamp' 2>/dev/null | tail -30 || echo "(could not retrieve events)" + + PROBLEM_PODS=$(oc get pods -n "${ns}" --no-headers 2>/dev/null \ + | grep -vE '\s(Running|Completed)\s' \ + | awk '{print $1}' || true) + + if [ -n "${PROBLEM_PODS}" ]; then + echo + echo "-- Logs for non-Running/non-Completed pods --" + for pod in ${PROBLEM_PODS}; do + echo + echo " >>> Pod: ${pod} <<<" + echo " -- describe --" + oc describe pod "${pod}" -n "${ns}" 2>/dev/null || true + echo + echo " -- logs (all containers, last 100 lines) --" + CONTAINERS=$(oc get pod "${pod}" -n "${ns}" \ + -o=jsonpath='{range .spec.initContainers[*]}{.name}{"\n"}{end}{range .spec.containers[*]}{.name}{"\n"}{end}' \ + 2>/dev/null || true) + for container in ${CONTAINERS}; do + echo " [container: ${container}]" + oc logs "${pod}" -n "${ns}" -c "${container}" --tail=100 2>/dev/null || echo " (logs unavailable)" + done + done + else + echo + echo " No problem pods found." + fi + done +} + +# ============================================================================== +# cleanup_default_namespace — remove test resources +# ============================================================================== +cleanup_default_namespace() { + section "Cleanup: Removing Test Resources from 'default' Namespace" + + echo "Deleting pods created by vault.bats..." + + for resource in pods deployments secretproviderclasses; do + echo + echo "-- ${resource} --" + ITEMS=$(oc get "${resource}" -n default --no-headers 2>/dev/null \ + | grep -iE 'vault|secrets-store|busybox|synck8s|rotation|inline' \ + | awk '{print $1}' || true) + if [ -n "${ITEMS}" ]; then + for item in ${ITEMS}; do + echo " Deleting ${resource}/${item}..." + oc delete "${resource}" "${item}" -n default --ignore-not-found 2>/dev/null || true + done + else + echo " Nothing to clean up." + fi + done + + echo + echo "Cleanup complete." +} + +# ============================================================================== +# on_exit — trap handler +# ============================================================================== +BATS_EXIT_CODE=0 + +on_exit() { + local exit_code=$? + + [ "${BATS_EXIT_CODE}" -ne 0 ] && exit_code="${BATS_EXIT_CODE}" + + echo + echo "======================================================================" + if [ "${exit_code}" -eq 0 ]; then + echo " EXIT: SUCCESS" + else + echo " EXIT: FAILURE (exit code: ${exit_code})" + fi + echo "======================================================================" + + if [ "${exit_code}" -eq 0 ]; then + cleanup_default_namespace + section "All Tests Passed" + echo "Log file: ${LOG_FILE}" + else + collect_failure_logs + section "Tests FAILED — Resources left for investigation" + echo "Full log: ${LOG_FILE}" + fi +} + +trap on_exit EXIT + +# ============================================================================== +# STEP 6: Run vault.bats tests +# ============================================================================== +run_bats_tests() { + section "STEP 6: Running vault.bats Tests" + + local bats_full="${REPO_DIR}/${BATS_FILE}" + + if [ ! -f "${bats_full}" ]; then + echo "ERROR: ${bats_full} not found." + exit 1 + fi + + echo "Switching to 'default' namespace..." + oc project default + + echo "Working directory: ${REPO_DIR}" + echo "Running: bats --trace --verbose-run --show-output-of-passing-tests ${BATS_FILE}" + echo + + cd "${REPO_DIR}" + + set +e + bats --trace --verbose-run --show-output-of-passing-tests ${BATS_FILE} + BATS_EXIT_CODE=$? + set -e + + if [ "${BATS_EXIT_CODE}" -eq 0 ]; then + echo + echo "All vault.bats tests passed." + else + echo + echo "ERROR: vault.bats exited with code ${BATS_EXIT_CODE}" + fi + + cd - +} + +# ============================================================================== +# Main +# ============================================================================== +collect_versions +check_arch_and_deps +validate_files +clone_repo +replace_busybox_images +patch_vault_bats +run_bats_tests + +exit "${BATS_EXIT_CODE}" diff --git a/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/test-vault/openshift-e2e-libvirt-s390x-csi-secrets-store-test-vault-ref.metadata.json b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/test-vault/openshift-e2e-libvirt-s390x-csi-secrets-store-test-vault-ref.metadata.json new file mode 100644 index 0000000000000..0e5804bffd365 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/test-vault/openshift-e2e-libvirt-s390x-csi-secrets-store-test-vault-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "openshift/e2e/libvirt-s390x/csi/secrets-store/test-vault/openshift-e2e-libvirt-s390x-csi-secrets-store-test-vault-ref.yaml", + "owners": { + "approvers": [ + "chiragkyal", + "mytreya-rh", + "sandisamp", + "v78singh" + ], + "reviewers": [ + "chiragkyal", + "mytreya-rh", + "sandisamp", + "v78singh" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/test-vault/openshift-e2e-libvirt-s390x-csi-secrets-store-test-vault-ref.yaml b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/test-vault/openshift-e2e-libvirt-s390x-csi-secrets-store-test-vault-ref.yaml new file mode 100644 index 0000000000000..e86c8857dc375 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/libvirt-s390x/csi/secrets-store/test-vault/openshift-e2e-libvirt-s390x-csi-secrets-store-test-vault-ref.yaml @@ -0,0 +1,30 @@ +ref: + as: openshift-e2e-libvirt-s390x-csi-secrets-store-test-vault + from: secrets-store-csi-driver-test + cli: latest + commands: openshift-e2e-libvirt-s390x-csi-secrets-store-test-vault-commands.sh + credentials: + - mount_path: /etc/hypershift-agent-ibmz-credentials + name: hypershift-agent-ibmz-credentials + namespace: test-credentials + resources: + requests: + cpu: 100m + memory: 200Mi + timeout: 2h + grace_period: 10m + documentation: |- + This step runs the Vault end-to-end tests for the Secrets Store CSI Driver + on an s390x cluster. It performs the following: + - Validates s390x architecture + - Installs required dependencies (git, curl, jq, tar, helm, bats) + - Clones the secrets-store-csi-driver repository + - Replaces busybox images with s390x-compatible versions (uses docker.io/s390x/busybox) + - Patches vault.bats to use Vault Enterprise with proper configuration + - Applies vault license from vault credentials + - Runs the vault.bats test suite + - Collects logs and diagnostics on failure + - Cleans up test resources on success + + Note: Docker Hub images are accessible via the cluster's pre-configured pull secret. + No additional docker-secret.json is required.