From 78b912fb68d294dba8b068708a7c784b59899e24 Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Mon, 3 Aug 2026 08:25:32 -0400 Subject: [PATCH 01/16] [DEBUG][DO NOT MERGE] Add azure-ipi-peerpods-ga job for candidate422 Add peer-pod GA workload on Azure with 8h wait for debugging. Uses redhat-operators catalog, no RPM install, restrict_network_access: true. Co-Authored-By: Claude Opus 4.6 --- ...erator-devel__downstream-candidate422.yaml | 42 +- ...d-containers-operator-devel-periodics.yaml | 94 +++ .../install-osc-operator/OWNERS | 10 + .../install-osc-operator/README.md | 43 ++ ...-operator-install-osc-operator-commands.sh | 697 ++++++++++++++++++ ...tor-install-osc-operator-ref.metadata.json | 17 + ...ers-operator-install-osc-operator-ref.yaml | 61 ++ ...ndboxed-containers-operator-pre-chain.yaml | 1 + 8 files changed, 963 insertions(+), 2 deletions(-) create mode 100644 ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/OWNERS create mode 100644 ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/README.md create mode 100755 ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh create mode 100644 ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.metadata.json create mode 100644 ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml diff --git a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml index 64b79d5d0857c..f8753ebd8a1bd 100644 --- a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml +++ b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml @@ -34,7 +34,7 @@ tests: report_template: '{{if eq .Status.State "success"}}SUCCESS{{else}}ERROR{{end}} {{trimPrefix "periodic-ci-openshift-sandboxed-containers-operator-" .Spec.Job}} <{{.Status.URL}}|View logs>' - restrict_network_access: false + restrict_network_access: true steps: cluster_profile: azure-qe env: @@ -48,14 +48,52 @@ tests: KATA_RPM_VERSION: 3.31.0-2.rhaos4.22.el9 MUST_GATHER_IMAGE: registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest MUST_GATHER_ON_FAILURE_ONLY: "false" + OSC_INSTALL: "true" SLEEP_DURATION: 0h TEST_FILTERS: ~DisconnectedOnly&;~Disruptive& TEST_RELEASE_TYPE: Pre-GA - TEST_SCENARIOS: sig-kata.*Kata Author + TEST_SCENARIOS: C00102 TEST_TIMEOUT: "90" TRUSTEE_URL: "" workflow: sandboxed-containers-operator-e2e-azure timeout: 24h0m0s +- as: azure-ipi-peerpods-ga + capabilities: + - intranet + cron: 0 0 31 2 1 + reporter_config: + channel: '#kata-ocp-ci-reports' + job_states_to_report: + - success + - failure + - error + report_template: '{{if eq .Status.State "success"}}SUCCESS{{else}}ERROR{{end}} + {{trimPrefix "periodic-ci-openshift-sandboxed-containers-operator-" .Spec.Job}} + <{{.Status.URL}}|View logs>' + restrict_network_access: true + steps: + cluster_profile: azure-qe + env: + BASE_DOMAIN: qe.azure.devcluster.openshift.com + CATALOG_SOURCE_NAME: redhat-operators + CUSTOM_AZURE_REGION: eastus + ENABLE_MUST_GATHER: "true" + ENABLEPEERPODS: "true" + INITDATA: "" + INSTALL_KATA_RPM: "false" + MUST_GATHER_IMAGE: registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest + MUST_GATHER_ON_FAILURE_ONLY: "false" + OSC_INSTALL: "true" + RUNTIMECLASS: kata-remote + SLEEP_DURATION: 8h + TEST_FILTERS: ~DisconnectedOnly&;~Disruptive& + TEST_RELEASE_TYPE: GA + TEST_SCENARIOS: C00102 + TEST_TIMEOUT: "90" + TRUSTEE_URL: "" + WORKLOAD_TO_TEST: peer-pods + workflow: sandboxed-containers-operator-e2e-azure + timeout: 24h0m0s - as: azure-ipi-peerpods capabilities: - intranet diff --git a/ci-operator/jobs/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel-periodics.yaml b/ci-operator/jobs/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel-periodics.yaml index da4bc21152346..c540bd89027b9 100644 --- a/ci-operator/jobs/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel-periodics.yaml +++ b/ci-operator/jobs/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel-periodics.yaml @@ -4605,6 +4605,100 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build09 + cron: 0 0 31 2 1 + decorate: true + decoration_config: + skip_cloning: true + timeout: 24h0m0s + extra_refs: + - base_ref: devel + org: openshift + repo: sandboxed-containers-operator + labels: + capability/intranet: intranet + ci-operator.openshift.io/cloud: azure4 + ci-operator.openshift.io/cloud-cluster-profile: azure-qe + ci-operator.openshift.io/variant: downstream-candidate422 + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-sandboxed-containers-operator-devel-downstream-candidate422-azure-ipi-peerpods-ga + reporter_config: + slack: + channel: '#kata-ocp-ci-reports' + job_states_to_report: + - success + - failure + - error + report_template: '{{if eq .Status.State "success"}}SUCCESS{{else}}ERROR{{end}} + {{trimPrefix "periodic-ci-openshift-sandboxed-containers-operator-" .Spec.Job}} + <{{.Status.URL}}|View logs>' + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=azure-ipi-peerpods-ga + - --variant=downstream-candidate422 + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build09 cron: 0 0 31 2 1 diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/OWNERS b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/OWNERS new file mode 100644 index 0000000000000..5c31fe0ceccfc --- /dev/null +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/OWNERS @@ -0,0 +1,10 @@ +reviewers: + - ldoktor + - tbuskey + - vvoronko + - wainersm +approvers: + - ldoktor + - tbuskey + - vvoronko + - wainersm diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/README.md b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/README.md new file mode 100644 index 0000000000000..5126321c61712 --- /dev/null +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/README.md @@ -0,0 +1,43 @@ +# sandboxed-containers-operator-install-osc-operator + +Installs the OpenShift Sandboxed Containers (OSC) operator and configures the +cluster for the target workload (kata, peer-pods, or coco). + +## Overview + +This step uses Helm charts from the [confidential-devhub/charts](https://github.com/confidential-devhub/charts) +repository to install the OSC operator and configure operands. It follows a +two-phase install: first the operator (namespace, subscription, CSV), then the +operands (KataConfig, feature gates, peer-pods configuration). + +The step is a no-op by default (`OSC_INSTALL=false`) so it can live in the +shared `sandboxed-containers-operator-pre` chain without affecting jobs that +do not need it. + +## Workload Types + +| Workload | ENABLEPEERPODS | WORKLOAD_TO_TEST | What gets configured | +|----------|---------------|-----------------|---------------------| +| kata | false | kata | KataConfig (bare metal kata runtime) | +| peer-pods | true | peer-pods | KataConfig + peer-pods-cm + peer-pods-secret | +| coco | true | coco | KataConfig + peer-pods-cm + peer-pods-secret + confidential feature gate | + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `OSC_INSTALL` | `false` | Set to `true` to enable installation | +| `OSC_CHARTS_REPO` | `https://github.com/confidential-devhub/charts.git` | Git repo URL for Helm charts | +| `OSC_CHARTS_REF` | `main` | Git ref (branch/tag/commit) | +| `CATALOG_SOURCE_IMAGE` | `""` | Custom CatalogSource image for dev/pre-GA | +| `ENABLEPEERPODS` | `false` | Enable peer-pods in KataConfig | +| `WORKLOAD_TO_TEST` | `kata` | Workload type: kata, peer-pods, or coco | +| `OSC_NAMESPACE` | `openshift-sandboxed-containers-operator` | Target namespace | + +## Prerequisites + +This step expects the following to be available (created by earlier steps in the chain): + +- `osc-config` ConfigMap in default namespace (created by `env-cm` step) +- `peerpods-param-cm` ConfigMap in default namespace (created by `peerpods-param-cm` step, when peer-pods enabled) +- `peerpods-param-secret` Secret in default namespace (created by `peerpods-param-cm` step, when peer-pods enabled) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh new file mode 100755 index 0000000000000..7325a820efe1c --- /dev/null +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh @@ -0,0 +1,697 @@ +#!/usr/bin/env bash +# +# Install OSC (OpenShift Sandboxed Containers) Operator +# +# This script installs and configures the OSC operator and operands using +# helm charts cloned from OSC_CHARTS_REPO. +# Requires ci/rhdh-e2e-runner base image (provides helm, oc, git, jq). +# +# Environment Variables: +# OSC_INSTALL - "true" to install, "false" to skip (default: false) +# OSC_NAMESPACE - Namespace for operator (default: openshift-sandboxed-containers-operator) +# CATALOG_SOURCE_IMAGE - Custom catalog image (optional) +# OSC_CHARTS_REPO - Charts repo URL +# OSC_CHARTS_REF - Charts git ref (default: main) +# ENABLEPEERPODS - "true" to enable peer-pods (default: false) +# WORKLOAD_TO_TEST - "kata", "peer-pods", or "coco" (default: kata) +# +# Outputs: +# Patches osc-config ConfigMap in default namespace to indicate installation complete. +# + +set -euo pipefail + +#======================================== +# Configuration +#======================================== + +export SHARED_DIR=${SHARED_DIR:-/tmp} +export KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig} + +OSC_INSTALL=${OSC_INSTALL:-false} +OSC_NAMESPACE=${OSC_NAMESPACE:-openshift-sandboxed-containers-operator} +CATALOG_SOURCE_IMAGE=${CATALOG_SOURCE_IMAGE:-} +# Pick up the resolved catalog image from env-cm step (resolves :latest to a specific tag) +if [[ -f "${SHARED_DIR}/catalog-source-image.env" ]]; then + # shellcheck disable=SC1091 + source "${SHARED_DIR}/catalog-source-image.env" +fi +OSC_CHARTS_REPO=${OSC_CHARTS_REPO:-https://github.com/confidential-devhub/charts.git} +OSC_CHARTS_REF=${OSC_CHARTS_REF:-main} +ENABLEPEERPODS=${ENABLEPEERPODS:-false} +WORKLOAD_TO_TEST=${WORKLOAD_TO_TEST:-kata} + +# Early exit if installation disabled +if [[ "${OSC_INSTALL}" != "true" ]]; then + echo ">>> Skipping OSC operator installation (OSC_INSTALL=${OSC_INSTALL})" + exit 0 +fi + +# Verify helm is available (pre-installed in base image) +if ! command -v helm &> /dev/null; then + echo ">>> ERROR: helm not found in base image" + exit 1 +fi + +# Show configuration +echo ">>> OSC charts: ${OSC_CHARTS_REPO} (ref: ${OSC_CHARTS_REF})" +echo ">>> Namespace: ${OSC_NAMESPACE}" +echo ">>> Workload: ${WORKLOAD_TO_TEST}" +echo ">>> Peer-pods: ${ENABLEPEERPODS}" +if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then + echo ">>> Catalog source: osc-operator-dev-catalog (image: ${CATALOG_SOURCE_IMAGE})" +else + echo ">>> Catalog source: redhat-operators (using existing catalog)" +fi + +#======================================== +# Cleanup Handler +#======================================== + +SCRATCH=$(mktemp -d) +cd "${SCRATCH}" + +function exit_handler() { + local exitcode=$? + set +e + rm -rf "${SCRATCH}" + + if [[ ${exitcode} -ne 0 ]]; then + echo ">>> ERROR: OSC operator installation failed" + echo ">>> Namespace status:" + oc get all -n "${OSC_NAMESPACE}" || true + echo ">>> Operator logs:" + oc logs -n "${OSC_NAMESPACE}" deployment/controller-manager --tail=50 || true + fi +} +trap 'exit_handler' EXIT + +#======================================== +# Helper Functions +#======================================== + +function retry() { + "$@" && return 0 + for (( i = 0; i < 9; i++ )); do + sleep 30 + "$@" && return 0 + done + return 1 +} + +function wait_until() { + local description="$1" + local timeout_seconds="$2" + local check_interval="$3" + local condition_command="$4" + + local max_iterations=$((timeout_seconds / check_interval)) + local progress_interval=$((60 / check_interval)) + [[ ${progress_interval} -lt 1 ]] && progress_interval=1 + + echo ">>> Waiting for ${description} (timeout: ${timeout_seconds}s, interval: ${check_interval}s)..." >&2 + + for (( i = 1; i <= max_iterations; i++ )); do + if eval "${condition_command}" 2>/dev/null; then + echo ">>> ${description} - SUCCESS (after $((i * check_interval))s)" >&2 + return 0 + fi + + if [[ $((i % progress_interval)) -eq 0 ]]; then + echo ">>> Still waiting for ${description} (${i}/${max_iterations}, $((i * check_interval))s elapsed)..." >&2 + fi + + [[ ${i} -lt ${max_iterations} ]] && sleep "${check_interval}" + done + + echo ">>> ERROR: ${description} - TIMEOUT after ${timeout_seconds}s" >&2 + return 1 +} + +#======================================== +# Chart Fetching +#======================================== + +function fetch_osc_charts() { + local charts_dir="${SCRATCH}/charts" + + echo ">>> Fetching OSC charts from: ${OSC_CHARTS_REPO} (ref: ${OSC_CHARTS_REF})" >&2 + + mkdir -p "${charts_dir}" + rm -rf "${charts_dir}" + git clone --depth 1 --branch "${OSC_CHARTS_REF}" "${OSC_CHARTS_REPO}" "${charts_dir}" + + if [[ ! -d "${charts_dir}" ]]; then + echo ">>> ERROR: Failed to clone charts repository" >&2 + exit 1 + fi + + echo ">>> Charts fetched" >&2 + local result_dir + if [[ -d "${charts_dir}/charts" ]]; then + result_dir="${charts_dir}/charts" + else + result_dir="${charts_dir}" + fi + + # DEBUG until https://github.com/confidential-devhub/charts/pull/3 merges + if grep -q 'startingCSV' "${result_dir}/osc-operator/templates/subscription.yaml" 2>/dev/null; then + sed -i '/startingCSV/d' "${result_dir}/osc-operator/templates/subscription.yaml" + echo ">>> DEBUG: subscription.yaml patched (removed startingCSV line). PR 3 not merged yet." >&2 + fi + + echo "${result_dir}" +} + +#======================================== +# Cloud Provider Detection +#======================================== + +function get_cloud_provider() { + local provider + provider=$(oc get infrastructure/cluster -o jsonpath='{.status.platformStatus.type}' 2>/dev/null | tr '[:upper:]' '[:lower:]') + if [[ "${provider}" == "none" ]]; then + provider="libvirt" + fi + echo "${provider}" +} + +#======================================== +# Helm Chart Rendering +#======================================== + +function render_osc_operator_chart() { + local charts_dir="$1" + local operator_chart="${charts_dir}/osc-operator" + + if [[ ! -d "${operator_chart}" ]]; then + echo ">>> ERROR: Operator chart not found at ${operator_chart}" >&2 + return 1 + fi + + echo ">>> Rendering osc-operator chart from: ${operator_chart}" >&2 + + local helm_args=( + "osc-operator" + "${operator_chart}" + "--namespace" "${OSC_NAMESPACE}" + "--set" "namespaceOverride=${OSC_NAMESPACE}" + ) + + if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then + helm_args+=( + "--set" "dev.enabled=true" + "--set" "dev.image=${CATALOG_SOURCE_IMAGE}" + ) + echo ">>> Helm: dev.enabled=true, dev.image=${CATALOG_SOURCE_IMAGE}" >&2 + else + helm_args+=( + "--set" "dev.enabled=false" + ) + echo ">>> Helm: dev.enabled=false (using redhat-operators)" >&2 + fi + + local helm_output + if ! helm_output=$(helm template "${helm_args[@]}" 2>&1); then + echo ">>> ERROR: helm template failed" >&2 + echo "$helm_output" >&2 + return 1 + fi + + echo "$helm_output" +} + +function render_osc_operands_chart() { + local charts_dir="$1" + local operands_chart="${charts_dir}/osc-operands" + + if [[ ! -d "${operands_chart}" ]]; then + echo ">>> ERROR: Operands chart not found at ${operands_chart}" >&2 + return 1 + fi + + echo ">>> Rendering osc-operands chart from: ${operands_chart}" >&2 + + local helm_args=( + "osc-operands" + "${operands_chart}" + "--namespace" "${OSC_NAMESPACE}" + "--set" "namespaceOverride=${OSC_NAMESPACE}" + ) + + # Workload-specific settings + if [[ "${WORKLOAD_TO_TEST}" == "coco" ]]; then + helm_args+=("--set" "confidential.enabled=true") + echo ">>> Helm: confidential.enabled=true" >&2 + else + helm_args+=("--set" "confidential.enabled=false") + fi + + if [[ "${ENABLEPEERPODS}" == "true" ]]; then + helm_args+=("--set" "peerpods.enabled=true") + + local provider + provider=$(get_cloud_provider) + helm_args+=("--set" "peerpods.provider=${provider}") + echo ">>> Helm: peerpods.enabled=true, provider=${provider}" >&2 + + # Generate SSH keys via the chart's Makefile (ed25519, into files/ for .Files.Get) + make -C "${operands_chart}" ssh-keys >&2 + + # Read cloud config from peerpods-param-cm (created by peerpods-param-cm step) + local cm_data + cm_data=$(oc get configmap peerpods-param-cm -n default -o json 2>/dev/null || echo "") + if [[ -n "${cm_data}" ]]; then + echo ">>> Reading cloud config from peerpods-param-cm" >&2 + + # Extract common values + local vxlan_port proxy_timeout + vxlan_port=$(echo "${cm_data}" | jq -r '.data.VXLAN_PORT // ""') + proxy_timeout=$(echo "${cm_data}" | jq -r '.data.PROXY_TIMEOUT // ""') + [[ -n "${vxlan_port}" ]] && helm_args+=("--set" "peerpods.providersConfigs.all.VXLAN_PORT=${vxlan_port}") + [[ -n "${proxy_timeout}" ]] && helm_args+=("--set" "peerpods.providersConfigs.all.PROXY_TIMEOUT=${proxy_timeout}") + + case "${provider}" in + azure) + local azure_subnet_id azure_nsg_id azure_resource_group azure_region azure_instance_size + azure_subnet_id=$(echo "${cm_data}" | jq -r '.data.AZURE_SUBNET_ID // ""') + azure_nsg_id=$(echo "${cm_data}" | jq -r '.data.AZURE_NSG_ID // ""') + azure_resource_group=$(echo "${cm_data}" | jq -r '.data.AZURE_RESOURCE_GROUP // ""') + azure_region=$(echo "${cm_data}" | jq -r '.data.AZURE_REGION // ""') + azure_instance_size=$(echo "${cm_data}" | jq -r '.data.AZURE_INSTANCE_SIZE // ""') + [[ -n "${azure_subnet_id}" ]] && helm_args+=("--set" "peerpods.providersConfigs.azure.AZURE_SUBNET_ID=${azure_subnet_id}") + [[ -n "${azure_nsg_id}" ]] && helm_args+=("--set" "peerpods.providersConfigs.azure.AZURE_NSG_ID=${azure_nsg_id}") + [[ -n "${azure_resource_group}" ]] && helm_args+=("--set" "peerpods.providersConfigs.azure.AZURE_RESOURCE_GROUP=${azure_resource_group}") + [[ -n "${azure_region}" ]] && helm_args+=("--set" "peerpods.providersConfigs.azure.AZURE_REGION=${azure_region}") + # Let chart auto-set AZURE_INSTANCE_SIZE based on confidential mode if not explicitly set + [[ -n "${azure_instance_size}" ]] && helm_args+=("--set" "peerpods.providersConfigs.azure.AZURE_INSTANCE_SIZE=${azure_instance_size}") + ;; + aws) + local aws_region aws_subnet_id aws_vpc_id aws_sg_ids podvm_instance_type + aws_region=$(echo "${cm_data}" | jq -r '.data.AWS_REGION // ""') + aws_subnet_id=$(echo "${cm_data}" | jq -r '.data.AWS_SUBNET_ID // ""') + aws_vpc_id=$(echo "${cm_data}" | jq -r '.data.AWS_VPC_ID // ""') + aws_sg_ids=$(echo "${cm_data}" | jq -r '.data.AWS_SG_IDS // ""') + podvm_instance_type=$(echo "${cm_data}" | jq -r '.data.PODVM_INSTANCE_TYPE // ""') + [[ -n "${aws_region}" ]] && helm_args+=("--set" "peerpods.providersConfigs.aws.AWS_REGION=${aws_region}") + [[ -n "${aws_subnet_id}" ]] && helm_args+=("--set" "peerpods.providersConfigs.aws.AWS_SUBNET_ID=${aws_subnet_id}") + [[ -n "${aws_vpc_id}" ]] && helm_args+=("--set" "peerpods.providersConfigs.aws.AWS_VPC_ID=${aws_vpc_id}") + [[ -n "${aws_sg_ids}" ]] && helm_args+=("--set" "peerpods.providersConfigs.aws.AWS_SG_IDS=${aws_sg_ids}") + [[ -n "${podvm_instance_type}" ]] && helm_args+=("--set" "peerpods.providersConfigs.aws.PODVM_INSTANCE_TYPE=${podvm_instance_type}") + ;; + gcp) + local gcp_project_id gcp_zone gcp_network gcp_machine_type + gcp_project_id=$(echo "${cm_data}" | jq -r '.data.GCP_PROJECT_ID // ""') + gcp_zone=$(echo "${cm_data}" | jq -r '.data.GCP_ZONE // ""') + gcp_network=$(echo "${cm_data}" | jq -r '.data.GCP_NETWORK // ""') + gcp_machine_type=$(echo "${cm_data}" | jq -r '.data.GCP_MACHINE_TYPE // ""') + [[ -n "${gcp_project_id}" ]] && helm_args+=("--set" "peerpods.providersConfigs.gcp.GCP_PROJECT_ID=${gcp_project_id}") + [[ -n "${gcp_zone}" ]] && helm_args+=("--set" "peerpods.providersConfigs.gcp.GCP_ZONE=${gcp_zone}") + [[ -n "${gcp_network}" ]] && helm_args+=("--set" "peerpods.providersConfigs.gcp.GCP_NETWORK=${gcp_network}") + [[ -n "${gcp_machine_type}" ]] && helm_args+=("--set" "peerpods.providersConfigs.gcp.GCP_MACHINE_TYPE=${gcp_machine_type}") + ;; + esac + else + echo ">>> WARNING: peerpods-param-cm not found in default namespace" >&2 + fi + else + helm_args+=("--set" "peerpods.enabled=false") + fi + + local helm_output + if ! helm_output=$(helm template "${helm_args[@]}" 2>&1); then + echo ">>> ERROR: helm template failed" >&2 + echo "$helm_output" >&2 + return 1 + fi + + echo "$helm_output" +} + +#======================================== +# Installation Functions +#======================================== + +function install_osc_operator() { + local charts_dir="$1" + + echo ">>> Installing OSC operator" + + echo ">>> Creating namespace ${OSC_NAMESPACE}" + oc create namespace "${OSC_NAMESPACE}" 2>/dev/null || true + + local operator_yaml="${SCRATCH}/operator-manifests.yaml" + if ! render_osc_operator_chart "${charts_dir}" > "${operator_yaml}"; then + echo ">>> ERROR: Failed to render operator chart" + return 1 + fi + + echo ">>> Rendered operator YAML:" + cat "${operator_yaml}" + + local apply_output + if ! apply_output=$(oc apply -f "${operator_yaml}" 2>&1); then + echo ">>> ERROR: Failed to apply operator manifests" + echo "$apply_output" + return 1 + fi + + echo ">>> Apply output:" + echo "$apply_output" +} + +function wait_for_operator() { + # Stage 0: Wait for ALL CatalogSources to be READY (600s) + echo ">>> Waiting for all CatalogSources to be READY..." + local all_catalogs_ready=false + for i in {1..120}; do + local catalog_states + catalog_states=$(oc get catalogsource -n openshift-marketplace -o jsonpath='{range .items[*]}{.metadata.name}={.status.connectionState.lastObservedState}{"\n"}{end}' 2>/dev/null || echo "") + + if [[ -z "${catalog_states}" ]]; then + [[ ${i} -lt 120 ]] && sleep 5 + continue + fi + + local total_catalogs ready_catalogs + total_catalogs=$(echo "${catalog_states}" | wc -l) + ready_catalogs=$(echo "${catalog_states}" | grep -c "=READY" || echo "0") + + if [[ ${ready_catalogs} -eq ${total_catalogs} && ${ready_catalogs} -gt 0 ]]; then + echo ">>> All CatalogSources are READY (${ready_catalogs}/${total_catalogs})" + all_catalogs_ready=true + break + fi + + if [[ $((i % 6)) -eq 0 ]]; then + echo ">>> CatalogSources ready: ${ready_catalogs}/${total_catalogs} ($((i*5))s elapsed)..." + fi + + [[ ${i} -lt 120 ]] && sleep 5 + done + + if [[ "${all_catalogs_ready}" != "true" ]]; then + echo ">>> ERROR: Not all CatalogSources are READY after 600s" + oc get catalogsource -n openshift-marketplace -o custom-columns=NAME:.metadata.name,STATE:.status.connectionState.lastObservedState || true + return 1 + fi + + # Stage 1: Wait for custom CatalogSource if configured + if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then + if ! wait_until "OSC CatalogSource osc-operator-dev-catalog READY" 60 5 \ + "[[ \"\$(oc get catalogsource -n openshift-marketplace osc-operator-dev-catalog -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null)\" == \"READY\" ]]"; then + oc get catalogsource -n openshift-marketplace || true + oc describe catalogsource -n openshift-marketplace osc-operator-dev-catalog || true + return 1 + fi + fi + + # Stage 2: Wait for Subscription to reference an InstallPlan (300s) + if ! wait_until "Subscription to reference InstallPlan" 300 5 \ + "oc get subscription -n '${OSC_NAMESPACE}' sandboxed-containers-operator -o jsonpath='{.status.installplan.name}' 2>/dev/null | grep -q '^install-'"; then + echo ">>> ERROR: Subscription has no InstallPlan reference" >&2 + oc get subscription -n "${OSC_NAMESPACE}" sandboxed-containers-operator -o yaml || true + return 1 + fi + + local installplan_ref + installplan_ref=$(oc get subscription -n "${OSC_NAMESPACE}" sandboxed-containers-operator -o jsonpath='{.status.installplan.name}' 2>/dev/null || echo "") + echo ">>> InstallPlan: ${installplan_ref}" + + # Stage 3: Wait for InstallPlan to be Complete (300s) + if ! wait_until "InstallPlan ${installplan_ref} Complete" 300 5 \ + "[[ \"\$(oc get installplan -n '${OSC_NAMESPACE}' '${installplan_ref}' -o jsonpath='{.status.phase}' 2>/dev/null)\" == \"Complete\" ]]"; then + oc get installplan -n "${OSC_NAMESPACE}" "${installplan_ref}" -o yaml || true + return 1 + fi + + # Stage 4: Wait for CSV to be Succeeded (600s) + if ! wait_until "CSV Succeeded" 600 5 \ + "[[ \"\$(oc get csv -n '${OSC_NAMESPACE}' -o jsonpath='{.items[0].status.phase}' 2>/dev/null)\" == \"Succeeded\" ]]"; then + oc get csv -n "${OSC_NAMESPACE}" -o yaml || true + return 1 + fi + + local csv_name + csv_name=$(oc get csv -n "${OSC_NAMESPACE}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") + echo ">>> CSV ${csv_name} is Succeeded" + + # Stage 5: Wait for controller-manager Deployment to be Available (900s) + if ! wait_until "controller-manager deployment Available" 900 5 \ + "oc get deployment -n '${OSC_NAMESPACE}' controller-manager -o jsonpath='{.status.conditions[?(@.type==\"Available\")].status}' 2>/dev/null | grep -q 'True'"; then + oc get deployment -n "${OSC_NAMESPACE}" || true + oc get pods -n "${OSC_NAMESPACE}" || true + return 1 + fi + + # Stage 6: Wait for controller-manager rollout to complete (900s) + if ! wait_until "controller-manager rollout complete" 900 5 \ + "oc rollout status deployment/controller-manager -n '${OSC_NAMESPACE}' --timeout=0 2>/dev/null | grep -q 'successfully rolled out'"; then + oc get pods -n "${OSC_NAMESPACE}" || true + oc describe deployment -n "${OSC_NAMESPACE}" controller-manager | tail -30 || true + return 1 + fi + + oc get pods -n "${OSC_NAMESPACE}" || true + echo ">>> OSC operator installation complete" +} + +function install_osc_operands() { + local charts_dir="$1" + + echo ">>> Installing OSC operands (workload: ${WORKLOAD_TO_TEST}, peerpods: ${ENABLEPEERPODS})" + + local operands_yaml="${SCRATCH}/operands-manifests.yaml" + if ! render_osc_operands_chart "${charts_dir}" > "${operands_yaml}"; then + echo ">>> ERROR: Failed to render operands chart" + return 1 + fi + + # DEBUG until https://github.com/confidential-devhub/charts/pull/4 merges + # openshift-tests-private expects kataconfig name to be "example-kataconfig". Rename to match the test expectation. + if grep -q 'name: kataconfig' "${operands_yaml}" 2>/dev/null; then + sed -i 's/name:.*$/name: example-kataconfig/' "${operands_yaml}" + echo ">>> DEBUG: kataconfig.yaml patched (renamed to example-kataconfig). PR 4 not merged yet." >&2 + fi + + echo ">>> Rendered operands YAML:" + cat "${operands_yaml}" + + local apply_output + if ! apply_output=$(oc apply -f "${operands_yaml}" 2>&1); then + echo ">>> ERROR: Failed to apply operands manifests" + echo "$apply_output" + return 1 + fi + + echo ">>> Apply output:" + echo "$apply_output" +} + +#======================================== +# Peer-Pods Secret Handling +#======================================== + +function create_peer_pods_secret() { + echo ">>> Creating peer-pods-secret in ${OSC_NAMESPACE}" + + # Check if secret already exists + if oc get secret peer-pods-secret -n "${OSC_NAMESPACE}" &>/dev/null; then + echo ">>> peer-pods-secret already exists, skipping" + return 0 + fi + + # Detect identity mode from osc-config or default to manual + local identity_mode + identity_mode=$(oc get configmap osc-config -n default -o jsonpath='{.data.identityMode}' 2>/dev/null || echo "manual") + + case "${identity_mode}" in + cco) + echo ">>> Identity mode: cco (Cloud Credential Operator handles credentials)" + return 0 + ;; + sts) + echo ">>> Identity mode: sts (credentials via subscription environment)" + return 0 + ;; + manual|*) + echo ">>> Identity mode: manual (copying credentials from peerpods-param-secret)" + ;; + esac + + # Read peerpods-param-secret from default namespace + if ! oc get secret peerpods-param-secret -n default &>/dev/null; then + echo ">>> WARNING: peerpods-param-secret not found in default namespace" + return 0 + fi + + local provider + provider=$(get_cloud_provider) + + case "${provider}" in + azure) + # Extract Azure service principal credentials + local sp_json + sp_json=$(oc get secret peerpods-param-secret -n default -o jsonpath='{.data.osServicePrincipal\.json}' 2>/dev/null || echo "") + if [[ -z "${sp_json}" ]]; then + # Try auth.json format + sp_json=$(oc get secret peerpods-param-secret -n default -o jsonpath='{.data.auth\.json}' 2>/dev/null || echo "") + fi + + if [[ -n "${sp_json}" ]]; then + local decoded + decoded=$(echo "${sp_json}" | base64 -d) + local client_id client_secret tenant_id + client_id=$(echo "${decoded}" | jq -r '.clientId // .azure.azure_client_id // ""') + client_secret=$(echo "${decoded}" | jq -r '.clientSecret // .azure.azure_client_secret // ""') + tenant_id=$(echo "${decoded}" | jq -r '.tenantId // .azure.azure_tenant_id // ""') + + local subscription_id + subscription_id=$(oc get secret azure-credentials -n kube-system -o jsonpath='{.data.azure_subscription_id}' 2>/dev/null | base64 -d || echo "") + + cat > "${SCRATCH}/peer-pods-secret.json" <>> WARNING: Could not extract Azure credentials from peerpods-param-secret" + fi + ;; + aws) + # Extract AWS credentials + local auth_json + auth_json=$(oc get secret peerpods-param-secret -n default -o jsonpath='{.data.auth\.json}' 2>/dev/null || echo "") + if [[ -n "${auth_json}" ]]; then + echo "${auth_json}" | base64 -d > "${SCRATCH}/auth.json" + oc create secret generic peer-pods-secret \ + -n "${OSC_NAMESPACE}" \ + --from-file="${SCRATCH}/auth.json" + rm -f "${SCRATCH}/auth.json" + else + echo ">>> WARNING: Could not extract AWS credentials from peerpods-param-secret" + fi + ;; + *) + echo ">>> WARNING: peer-pods-secret creation not implemented for provider: ${provider}" + ;; + esac +} + +#======================================== +# KataConfig Wait +#======================================== + +function wait_for_kataconfig() { + echo ">>> Waiting for KataConfig to be ready (this may take up to 2 hours for node reboots)" + + # Wait for KataConfig CR to exist + if ! wait_until "KataConfig CR to exist" 60 5 \ + "oc get kataconfig -o name 2>/dev/null | grep -q 'kataconfig'"; then + echo ">>> ERROR: KataConfig not found" + return 1 + fi + + local kataconfig_name + kataconfig_name=$(oc get kataconfig -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") + echo ">>> KataConfig name: ${kataconfig_name}" + + # Wait for KataConfig InProgress condition to be False (7200s / 2h) + if ! wait_until "KataConfig ready (InProgress=False)" 7200 30 \ + "[[ \"\$(oc get kataconfig '${kataconfig_name}' -o jsonpath='{.status.conditions[?(@.type==\"InProgress\")].status}' 2>/dev/null)\" == \"False\" ]]"; then + echo ">>> ERROR: KataConfig not ready after 2 hours" + oc get kataconfig "${kataconfig_name}" -o yaml || true + oc get nodes || true + oc get mcp || true + return 1 + fi + + echo ">>> KataConfig is ready" + oc get kataconfig "${kataconfig_name}" -o jsonpath='{.status}' 2>/dev/null | jq . || true +} + +function verify_peerpod_pods() { + echo ">>> Verifying peer-pod control pods" + + # Wait for cloud-api-adaptor daemonset pods + if ! wait_until "cloud-api-adaptor pods running" 600 10 \ + "oc get daemonset -n '${OSC_NAMESPACE}' -l app=cloud-api-adaptor -o jsonpath='{.items[0].status.numberReady}' 2>/dev/null | grep -qE '^[1-9]'"; then + echo ">>> WARNING: cloud-api-adaptor daemonset not ready" + oc get daemonset -n "${OSC_NAMESPACE}" || true + oc get pods -n "${OSC_NAMESPACE}" -l app=cloud-api-adaptor || true + fi + + # Wait for peerpodconfig-ctrl-caa-daemon + if ! wait_until "peerpodconfig webhook pod running" 300 10 \ + "oc get pods -n '${OSC_NAMESPACE}' -l app=peerpodconfig-ctrl-caa-daemon -o jsonpath='{.items[0].status.phase}' 2>/dev/null | grep -q 'Running'"; then + echo ">>> WARNING: peerpodconfig webhook pod not running" + oc get pods -n "${OSC_NAMESPACE}" || true + fi + + echo ">>> Peer-pod pods:" + oc get pods -n "${OSC_NAMESPACE}" || true +} + +#======================================== +# Update Shared State +#======================================== + +function update_osc_config() { + echo ">>> Patching osc-config ConfigMap to indicate operator is installed" + + if ! oc get configmap osc-config -n default &>/dev/null; then + echo ">>> WARNING: osc-config ConfigMap not found in default namespace" + return 0 + fi + + oc patch configmap osc-config -n default --type merge \ + -p '{"data":{"oscInstalled":"true"}}' || true + + echo ">>> osc-config patched with oscInstalled=true" +} + +#======================================== +# Main Execution +#======================================== + +echo "=========================================" +echo ">>> OSC Operator Installation" +echo ">>> Workload: ${WORKLOAD_TO_TEST}" +echo ">>> Peer-pods: ${ENABLEPEERPODS}" +echo "=========================================" + +# Phase 2: Fetch charts +CHARTS_DIR=$(fetch_osc_charts) +echo ">>> Charts directory: ${CHARTS_DIR}" + +# Phase 3: Install operator +install_osc_operator "${CHARTS_DIR}" +wait_for_operator + +# Phase 4: Install operands +if [[ "${ENABLEPEERPODS}" == "true" ]]; then + create_peer_pods_secret +fi + +install_osc_operands "${CHARTS_DIR}" +wait_for_kataconfig + +if [[ "${ENABLEPEERPODS}" == "true" ]]; then + verify_peerpod_pods +fi + +# Phase 5: Update shared state +update_osc_config + +echo "=========================================" +echo ">>> OSC operator installation complete" +echo ">>> Workload: ${WORKLOAD_TO_TEST}" +echo ">>> Namespace: ${OSC_NAMESPACE}" +echo "=========================================" diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.metadata.json b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.metadata.json new file mode 100644 index 0000000000000..5fe8f601aa553 --- /dev/null +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml", + "owners": { + "approvers": [ + "ldoktor", + "tbuskey", + "vvoronko", + "wainersm" + ], + "reviewers": [ + "ldoktor", + "tbuskey", + "vvoronko", + "wainersm" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml new file mode 100644 index 0000000000000..afe52a0a94137 --- /dev/null +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml @@ -0,0 +1,61 @@ +ref: + as: sandboxed-containers-operator-install-osc-operator + from_image: + name: rhdh-e2e-runner + namespace: ci + tag: main + grace_period: 10m + commands: sandboxed-containers-operator-install-osc-operator-commands.sh + resources: + requests: + cpu: 1000m + memory: 2000Mi + env: + - name: OSC_INSTALL + default: "false" + documentation: |- + Whether to install the OSC operator. Set to "true" to enable installation. + If not true, the step is skipped and no further actions are taken. + - name: OSC_CHARTS_REPO + default: "https://github.com/confidential-devhub/charts.git" + documentation: |- + The git repository URL for OSC Helm charts. + - name: OSC_CHARTS_REF + default: "main" + documentation: |- + The git ref (branch/tag/commit) to use from the charts repository. + - name: CATALOG_SOURCE_IMAGE + default: "" + documentation: |- + The container image for a custom OSC operator CatalogSource. + If empty (default), uses existing "redhat-operators" catalog. + If set, helm chart creates a new CatalogSource named "osc-operator-dev-catalog" with this image. + - name: ENABLEPEERPODS + default: "false" + documentation: |- + When "true", enables peer-pods in KataConfig and creates peer-pods-cm + ConfigMap with cloud provider configuration. Reads cloud parameters from + peerpods-param-cm in the default namespace (created by the peerpods-param-cm step). + - name: WORKLOAD_TO_TEST + default: "kata" + documentation: |- + Selects the workload type to configure. Supported values: "kata", "peer-pods", "coco". + When "coco", sets confidential mode in feature gates. + - name: OSC_NAMESPACE + default: "openshift-sandboxed-containers-operator" + documentation: |- + The namespace where the OSC operator will be installed. + documentation: |- + If OSC_INSTALL=true, installs the OSC operator and configures the cluster for + the target workload (kata, peer-pods, or coco) using Helm charts from OSC_CHARTS_REPO. + If OSC_INSTALL=false, the step is skipped and no further actions are taken. + + Charts are cloned from OSC_CHARTS_REPO. Phase 1 installs the operator (namespace, + operatorgroup, subscription, optional dev catalog). Phase 2 installs operands + (KataConfig, feature gates, peer-pods-cm) and waits for KataConfig readiness. + + For peer-pods and coco workloads, reads cloud config from peerpods-param-cm and + creates peer-pods-secret in the operator namespace. For coco, enables confidential + mode in osc-feature-gates. + + When complete, patches osc-config ConfigMap so tests can detect the pre-installed operator. diff --git a/ci-operator/step-registry/sandboxed-containers-operator/pre/sandboxed-containers-operator-pre-chain.yaml b/ci-operator/step-registry/sandboxed-containers-operator/pre/sandboxed-containers-operator-pre-chain.yaml index 3cb7b9fc097a3..517e792404215 100644 --- a/ci-operator/step-registry/sandboxed-containers-operator/pre/sandboxed-containers-operator-pre-chain.yaml +++ b/ci-operator/step-registry/sandboxed-containers-operator/pre/sandboxed-containers-operator-pre-chain.yaml @@ -4,6 +4,7 @@ chain: - ref: sandboxed-containers-operator-get-kata-rpm - ref: sandboxed-containers-operator-peerpods-param-cm - ref: sandboxed-containers-operator-env-cm + - ref: sandboxed-containers-operator-install-osc-operator - ref: sandboxed-containers-operator-install-trustee-operator - ref: sandboxed-containers-operator-record-metadata documentation: |- From 9a2066597c461fce461cf576946529b5d55652db Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Mon, 3 Aug 2026 09:57:52 -0400 Subject: [PATCH 02/16] Fix grep -c fallback, sed anchoring, secret leak, and pre-chain order - Fix grep -c "=READY" || echo "0" producing two lines causing arithmetic syntax error when no CatalogSources are ready yet - Anchor sed pattern for kataconfig rename to avoid rewriting all name: fields - Replace cat of rendered YAML with dry-run object list to prevent leaking SSH keys into CI logs - Reorder pre-chain: install-trustee-operator before install-osc-operator Co-Authored-By: Claude Opus 4.6 --- ...ners-operator-install-osc-operator-commands.sh | 15 ++++++++------- .../sandboxed-containers-operator-pre-chain.yaml | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh index 7325a820efe1c..fab9c00e3b288 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh @@ -346,8 +346,8 @@ function install_osc_operator() { return 1 fi - echo ">>> Rendered operator YAML:" - cat "${operator_yaml}" + echo ">>> Rendered operator objects:" + oc apply -f "${operator_yaml}" --dry-run=client -o name || true local apply_output if ! apply_output=$(oc apply -f "${operator_yaml}" 2>&1); then @@ -375,7 +375,8 @@ function wait_for_operator() { local total_catalogs ready_catalogs total_catalogs=$(echo "${catalog_states}" | wc -l) - ready_catalogs=$(echo "${catalog_states}" | grep -c "=READY" || echo "0") + ready_catalogs=$(echo "${catalog_states}" | grep -c "=READY" || true) + ready_catalogs=${ready_catalogs:-0} if [[ ${ready_catalogs} -eq ${total_catalogs} && ${ready_catalogs} -gt 0 ]]; then echo ">>> All CatalogSources are READY (${ready_catalogs}/${total_catalogs})" @@ -469,13 +470,13 @@ function install_osc_operands() { # DEBUG until https://github.com/confidential-devhub/charts/pull/4 merges # openshift-tests-private expects kataconfig name to be "example-kataconfig". Rename to match the test expectation. - if grep -q 'name: kataconfig' "${operands_yaml}" 2>/dev/null; then - sed -i 's/name:.*$/name: example-kataconfig/' "${operands_yaml}" + if grep -qE '^\s+name: kataconfig$' "${operands_yaml}" 2>/dev/null; then + sed -i -E 's/^([[:space:]]*)name: kataconfig$/\1name: example-kataconfig/' "${operands_yaml}" echo ">>> DEBUG: kataconfig.yaml patched (renamed to example-kataconfig). PR 4 not merged yet." >&2 fi - echo ">>> Rendered operands YAML:" - cat "${operands_yaml}" + echo ">>> Rendered operands objects:" + oc apply -f "${operands_yaml}" --dry-run=client -o name || true local apply_output if ! apply_output=$(oc apply -f "${operands_yaml}" 2>&1); then diff --git a/ci-operator/step-registry/sandboxed-containers-operator/pre/sandboxed-containers-operator-pre-chain.yaml b/ci-operator/step-registry/sandboxed-containers-operator/pre/sandboxed-containers-operator-pre-chain.yaml index 517e792404215..14784fc84dd3e 100644 --- a/ci-operator/step-registry/sandboxed-containers-operator/pre/sandboxed-containers-operator-pre-chain.yaml +++ b/ci-operator/step-registry/sandboxed-containers-operator/pre/sandboxed-containers-operator-pre-chain.yaml @@ -4,8 +4,8 @@ chain: - ref: sandboxed-containers-operator-get-kata-rpm - ref: sandboxed-containers-operator-peerpods-param-cm - ref: sandboxed-containers-operator-env-cm - - ref: sandboxed-containers-operator-install-osc-operator - ref: sandboxed-containers-operator-install-trustee-operator + - ref: sandboxed-containers-operator-install-osc-operator - ref: sandboxed-containers-operator-record-metadata documentation: |- The sandboxed containers operator pre-testing chain \ No newline at end of file From 7c991bddb2618444f94447e2fc88ae076fb753c6 Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Mon, 3 Aug 2026 11:30:00 -0400 Subject: [PATCH 03/16] Use sed to patch Subscription catalog source until charts PR #4 merges The helm chart does not yet support dev.enabled/dev.image values for the operator chart. Replace the non-functional helm args with a sed workaround that reads the catalog source name from osc-config ConfigMap (set by env-cm) and patches the rendered Subscription source field. Co-Authored-By: Claude Opus 4.6 --- ...-operator-install-osc-operator-commands.sh | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh index fab9c00e3b288..c41e0b3bad985 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh @@ -198,18 +198,8 @@ function render_osc_operator_chart() { "--set" "namespaceOverride=${OSC_NAMESPACE}" ) - if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then - helm_args+=( - "--set" "dev.enabled=true" - "--set" "dev.image=${CATALOG_SOURCE_IMAGE}" - ) - echo ">>> Helm: dev.enabled=true, dev.image=${CATALOG_SOURCE_IMAGE}" >&2 - else - helm_args+=( - "--set" "dev.enabled=false" - ) - echo ">>> Helm: dev.enabled=false (using redhat-operators)" >&2 - fi + # TODO: use --set dev.enabled=true --set dev.image= once https://github.com/confidential-devhub/charts/pull/4 merges + echo ">>> Helm: using redhat-operators (catalog source patched via sed after render)" >&2 local helm_output if ! helm_output=$(helm template "${helm_args[@]}" 2>&1); then @@ -346,6 +336,18 @@ function install_osc_operator() { return 1 fi + # TODO: remove sed workaround once https://github.com/confidential-devhub/charts/pull/4 merges + # The chart always renders Subscription source as redhat-operators; patch it + # to the catalog created by env-cm when running Pre-GA. + if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then + local catsrc_name + catsrc_name=$(oc get configmap osc-config -n default -o jsonpath='{.data.catalogsourcename}' 2>/dev/null || echo "") + if [[ -n "${catsrc_name}" && "${catsrc_name}" != "redhat-operators" ]]; then + echo ">>> Patching Subscription source: redhat-operators -> ${catsrc_name}" + sed -i "s/source: redhat-operators/source: ${catsrc_name}/" "${operator_yaml}" + fi + fi + echo ">>> Rendered operator objects:" oc apply -f "${operator_yaml}" --dry-run=client -o name || true From 2cdd308a571bc336c2a229c6ed0c444ef32fd1f0 Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Mon, 3 Aug 2026 11:35:30 -0400 Subject: [PATCH 04/16] Switch peerpods-ga to Pre-GA with brew-catalog OSC operator is not available in redhat-operators for 4.22 yet. Use brew-catalog with konflux catalog source image like the other jobs. Co-Authored-By: Claude Opus 4.6 --- ...d-containers-operator-devel__downstream-candidate422.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml index f8753ebd8a1bd..87eed99c39761 100644 --- a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml +++ b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml @@ -75,7 +75,8 @@ tests: cluster_profile: azure-qe env: BASE_DOMAIN: qe.azure.devcluster.openshift.com - CATALOG_SOURCE_NAME: redhat-operators + CATALOG_SOURCE_IMAGE: quay.io/redhat-user-workloads/ose-osc-tenant/osc-test-fbc:latest + CATALOG_SOURCE_NAME: brew-catalog CUSTOM_AZURE_REGION: eastus ENABLE_MUST_GATHER: "true" ENABLEPEERPODS: "true" @@ -87,7 +88,7 @@ tests: RUNTIMECLASS: kata-remote SLEEP_DURATION: 8h TEST_FILTERS: ~DisconnectedOnly&;~Disruptive& - TEST_RELEASE_TYPE: GA + TEST_RELEASE_TYPE: Pre-GA TEST_SCENARIOS: C00102 TEST_TIMEOUT: "90" TRUSTEE_URL: "" From 0dcdb5e4a2f5f8f7a2e432ff166c798987745415 Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Mon, 3 Aug 2026 13:33:09 -0400 Subject: [PATCH 05/16] Fix CatalogSource wait to use actual name from osc-config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 1 was hardcoded to wait for osc-operator-dev-catalog which doesn't exist — env-cm creates the CatalogSource with the name from CATALOG_SOURCE_NAME (e.g. brew-catalog). Read the actual name from osc-config ConfigMap instead. Co-Authored-By: Claude Opus 4.6 --- ...ers-operator-install-osc-operator-commands.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh index c41e0b3bad985..9e6ad99a06d71 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh @@ -399,13 +399,17 @@ function wait_for_operator() { return 1 fi - # Stage 1: Wait for custom CatalogSource if configured + # Stage 1: Wait for the CatalogSource created by env-cm (e.g. brew-catalog) if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then - if ! wait_until "OSC CatalogSource osc-operator-dev-catalog READY" 60 5 \ - "[[ \"\$(oc get catalogsource -n openshift-marketplace osc-operator-dev-catalog -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null)\" == \"READY\" ]]"; then - oc get catalogsource -n openshift-marketplace || true - oc describe catalogsource -n openshift-marketplace osc-operator-dev-catalog || true - return 1 + local catsrc_name + catsrc_name=$(oc get configmap osc-config -n default -o jsonpath='{.data.catalogsourcename}' 2>/dev/null || echo "") + if [[ -n "${catsrc_name}" && "${catsrc_name}" != "redhat-operators" ]]; then + if ! wait_until "CatalogSource ${catsrc_name} READY" 60 5 \ + "[[ \"\$(oc get catalogsource -n openshift-marketplace '${catsrc_name}' -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null)\" == \"READY\" ]]"; then + oc get catalogsource -n openshift-marketplace || true + oc describe catalogsource -n openshift-marketplace "${catsrc_name}" || true + return 1 + fi fi fi From 6458d9caef313e619eafd259a76f4b8b189b2ee2 Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Mon, 3 Aug 2026 16:12:15 -0400 Subject: [PATCH 06/16] Move CatalogSource creation from env-cm into install-osc-operator CatalogSource setup (create, wait, mirror, tag resolution) now lives in install-osc-operator, matching the trustee pattern where each install step manages its own catalog. env-cm is simplified to only create the osc-config ConfigMap. install-osc patches osc-config with the resolved catalogsourcename after creating the CatalogSource. Co-Authored-By: Claude Opus 4.6 --- ...xed-containers-operator-env-cm-commands.sh | 134 -------------- ...dboxed-containers-operator-env-cm-ref.yaml | 17 +- ...-operator-install-osc-operator-commands.sh | 174 +++++++++++++++--- ...ers-operator-install-osc-operator-ref.yaml | 8 +- 4 files changed, 157 insertions(+), 176 deletions(-) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/env-cm/sandboxed-containers-operator-env-cm-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/env-cm/sandboxed-containers-operator-env-cm-commands.sh index 927a70286a81e..e9d96e734c203 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/env-cm/sandboxed-containers-operator-env-cm-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/env-cm/sandboxed-containers-operator-env-cm-commands.sh @@ -6,140 +6,6 @@ configmap_path="${SHARED_DIR:-$(pwd)}/env-cm.yaml" #echo "Giving a 10min stabilization time for AWS fresh 4.18 cluster before applying kataconfig as workaround for KATA-3451" #sleep 600 -create_catsrc() { - local catsrc_name="$1" - local catsrc_image="$2" - local catsrc_path="${SHARED_DIR:-$(pwd)}/catsrc_${catsrc_name}.yaml" - - echo "Create a custom catalogsource named ${catsrc_name} for internal builds" - - cat<<-EOF | tee "${catsrc_path}" - apiVersion: operators.coreos.com/v1alpha1 - kind: CatalogSource - metadata: - name: "${catsrc_name}" - namespace: openshift-marketplace - spec: - displayName: QE - image: "${catsrc_image}" - publisher: QE - sourceType: grpc -EOF - - oc apply -f "${catsrc_path}" -} - -# The "latest" tag does not convey information about the build time, so -# we search in quay.io for an X.Y.Z-unix_epoch (e.g., 1.11.1-1766149846) -# the first X.Y.Z-unix_epoch tag we find is the newest one because Quay -# returns tags sorted by creation time (newest first).and we return it -# if no tag is found, we return "latest" -# -# Optimization: Quay returns tags sorted by creation time (newest first). -# So the first X.Y.Z-unix_epoch tag we find is the newest one - we can stop immediately. -latest_catsrc_image_tag() { - local api_url="https://quay.io/api/v1/repository/redhat-user-workloads/ose-osc-tenant/osc-test-fbc/tag/" - local page=1 - local max_pages=20 # safety limit, typically exits much earlier - - while [ "$page" -le "$max_pages" ]; do - local resp - # Query with onlyActiveTags to skip deleted tags - resp=$(curl -sf "${api_url}?limit=100&page=${page}&onlyActiveTags=true") - - if [ -z "$resp" ] || ! jq -e '.tags | length > 0' <<< "$resp" >/dev/null 2>&1; then - break - fi - - # Find the first matching tag on this page (X.Y.Z-unix_epoch pattern) - # Since Quay returns newest first, the first match is the latest tag - local first_match - first_match=$(echo "$resp" | \ - jq -r '.tags[]? | select(.name | test("^[0-9]+\\.[0-9]+\\.[0-9]+-[0-9]+$")) | .name' | head -1) - - if [ -n "$first_match" ]; then - echo "$first_match" - return 0 - fi - - ((page++)) - done - - # Check if we hit max_pages without finding a tag (potential issue) - if [ "$page" -gt "$max_pages" ]; then - echo "ERROR: Hit max_pages ($max_pages) limit while searching for tags." >&2 - fi - - # Fallback to :latest if no matching tag found - echo "WARNING: No X.Y.Z-unix_epoch tag found, using :latest" >&2 - echo "latest" -} - -mirror_konflux() { - echo "Create mirror for konflux images" - # create the mirror set for the sandboxed-containers-operator and trustee-fbc devel branches - oc apply -f "https://raw.githubusercontent.com/openshift/sandboxed-containers-operator/refs/heads/devel/.tekton/images-mirror-set.yaml" - oc apply -f "https://raw.githubusercontent.com/openshift/trustee-fbc/refs/heads/main/.tekton/images-mirror-set.yaml" -} - -wait_for_catsrc() { - local catsrc_name="$1" - local timeout=300 - echo ">>> Waiting for CatalogSource ${catsrc_name} to be READY..." - - local deadline=$(( SECONDS + timeout )) - while (( SECONDS < deadline )); do - local state - state="$(oc get catalogsource -n openshift-marketplace \ - "${catsrc_name}" -o jsonpath='{.status.connectionState.lastObservedState}' \ - 2>/dev/null || echo "")" - if [[ "${state}" == "READY" ]]; then - echo ">>> CatalogSource ${catsrc_name} is READY" - return 0 - fi - sleep 10 - done - - local state - state="$(oc get catalogsource -n openshift-marketplace "${catsrc_name}" \ - -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null || echo "")" - if [[ "${state}" != "READY" ]]; then - echo "ERROR: CatalogSource ${catsrc_name} not READY after ${timeout}s (state: ${state})" >&2 - oc get catalogsource -n openshift-marketplace "${catsrc_name}" -o yaml || true - exit 1 - fi - echo ">>> CatalogSource ${catsrc_name} is READY" - return 0 -} - - -if [[ "$TEST_RELEASE_TYPE" == "Pre-GA" ]]; then - mirror_konflux - - default_catsrc_image="quay.io/redhat-user-workloads/ose-osc-tenant/osc-test-fbc" - # Only resolve the tag if it's :latest - # Other tags (specific versions like 1.11.1-1766149846 or SHAs) are passed through unchanged - if [[ "${CATALOG_SOURCE_IMAGE}" = "${default_catsrc_image}:latest" ]]; then - catsrc_image_tag=$(latest_catsrc_image_tag) - CATALOG_SOURCE_IMAGE="${default_catsrc_image}:${catsrc_image_tag}" - echo "Resolved :latest to tag: ${catsrc_image_tag}" - else - echo "Using provided catalog image: ${CATALOG_SOURCE_IMAGE}" - fi - - create_catsrc "${CATALOG_SOURCE_NAME}" "${CATALOG_SOURCE_IMAGE}" - wait_for_catsrc "${CATALOG_SOURCE_NAME}" - - # Save resolved CATALOG_SOURCE_IMAGE for subsequent steps - echo "CATALOG_SOURCE_IMAGE=${CATALOG_SOURCE_IMAGE}" > "${SHARED_DIR}/catalog-source-image.env" - echo "Saved resolved CATALOG_SOURCE_IMAGE to ${SHARED_DIR}/catalog-source-image.env" -else - if [[ -n "$CATALOG_SOURCE_IMAGE" ]]; then - echo "CATALOG_SOURCE_IMAGE can only be used when TEST_RELEASE_TYPE==Pre-GA ($CATALOG_SOURCE_IMAGE)" - exit 1 - fi -fi - cat <>> Namespace: ${OSC_NAMESPACE}" echo ">>> Workload: ${WORKLOAD_TO_TEST}" echo ">>> Peer-pods: ${ENABLEPEERPODS}" if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then - echo ">>> Catalog source: osc-operator-dev-catalog (image: ${CATALOG_SOURCE_IMAGE})" + echo ">>> Catalog source: ${CATALOG_SOURCE_NAME} (image: ${CATALOG_SOURCE_IMAGE})" else echo ">>> Catalog source: redhat-operators (using existing catalog)" fi @@ -128,6 +124,135 @@ function wait_until() { return 1 } +#======================================== +# CatalogSource Setup +#======================================== + +function mirror_konflux() { + echo ">>> Create mirror for konflux images" + oc apply -f "https://raw.githubusercontent.com/openshift/sandboxed-containers-operator/refs/heads/devel/.tekton/images-mirror-set.yaml" + oc apply -f "https://raw.githubusercontent.com/openshift/trustee-fbc/refs/heads/main/.tekton/images-mirror-set.yaml" +} + +function latest_catsrc_image_tag() { + local api_url="https://quay.io/api/v1/repository/redhat-user-workloads/ose-osc-tenant/osc-test-fbc/tag/" + local page=1 + local max_pages=20 + + while [ "$page" -le "$max_pages" ]; do + local resp + resp=$(curl -sf "${api_url}?limit=100&page=${page}&onlyActiveTags=true") + + if [ -z "$resp" ] || ! jq -e '.tags | length > 0' <<< "$resp" >/dev/null 2>&1; then + break + fi + + local first_match + first_match=$(echo "$resp" | \ + jq -r '.tags[]? | select(.name | test("^[0-9]+\\.[0-9]+\\.[0-9]+-[0-9]+$")) | .name' | head -1) + + if [ -n "$first_match" ]; then + echo "$first_match" + return 0 + fi + + ((page++)) + done + + if [ "$page" -gt "$max_pages" ]; then + echo "ERROR: Hit max_pages ($max_pages) limit while searching for tags." >&2 + fi + + echo "WARNING: No X.Y.Z-unix_epoch tag found, using :latest" >&2 + echo "latest" +} + +function create_catsrc() { + local catsrc_name="$1" + local catsrc_image="$2" + local catsrc_path="${SHARED_DIR}/catsrc_${catsrc_name}.yaml" + + echo ">>> Create a custom CatalogSource named ${catsrc_name} for internal builds" + + cat<<-EOF | tee "${catsrc_path}" + apiVersion: operators.coreos.com/v1alpha1 + kind: CatalogSource + metadata: + name: "${catsrc_name}" + namespace: openshift-marketplace + spec: + displayName: QE + image: "${catsrc_image}" + publisher: QE + sourceType: grpc +EOF + + oc apply -f "${catsrc_path}" +} + +function wait_for_catsrc() { + local catsrc_name="$1" + local timeout=300 + echo ">>> Waiting for CatalogSource ${catsrc_name} to be READY..." + + local deadline=$(( SECONDS + timeout )) + while (( SECONDS < deadline )); do + local state + state="$(oc get catalogsource -n openshift-marketplace \ + "${catsrc_name}" -o jsonpath='{.status.connectionState.lastObservedState}' \ + 2>/dev/null || echo "")" + if [[ "${state}" == "READY" ]]; then + echo ">>> CatalogSource ${catsrc_name} is READY" + return 0 + fi + sleep 10 + done + + state="$(oc get catalogsource -n openshift-marketplace "${catsrc_name}" \ + -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null || echo "")" + if [[ "${state}" != "READY" ]]; then + echo ">>> ERROR: CatalogSource ${catsrc_name} not READY after ${timeout}s (state: ${state})" >&2 + oc get catalogsource -n openshift-marketplace "${catsrc_name}" -o yaml || true + return 1 + fi + echo ">>> CatalogSource ${catsrc_name} is READY" + return 0 +} + +function setup_catalog_source() { + if [[ -z "${CATALOG_SOURCE_IMAGE}" ]]; then + echo ">>> No CATALOG_SOURCE_IMAGE set, using existing redhat-operators catalog" + return 0 + fi + + echo ">>> Setting up CatalogSource for Pre-GA install" + + mirror_konflux + + local default_catsrc_image="quay.io/redhat-user-workloads/ose-osc-tenant/osc-test-fbc" + if [[ "${CATALOG_SOURCE_IMAGE}" = "${default_catsrc_image}:latest" ]]; then + local catsrc_image_tag + catsrc_image_tag=$(latest_catsrc_image_tag) + CATALOG_SOURCE_IMAGE="${default_catsrc_image}:${catsrc_image_tag}" + echo ">>> Resolved :latest to tag: ${catsrc_image_tag}" + else + echo ">>> Using provided catalog image: ${CATALOG_SOURCE_IMAGE}" + fi + + create_catsrc "${CATALOG_SOURCE_NAME}" "${CATALOG_SOURCE_IMAGE}" + wait_for_catsrc "${CATALOG_SOURCE_NAME}" + + echo "CATALOG_SOURCE_IMAGE=${CATALOG_SOURCE_IMAGE}" > "${SHARED_DIR}/catalog-source-image.env" + echo ">>> Saved resolved CATALOG_SOURCE_IMAGE to ${SHARED_DIR}/catalog-source-image.env" + + # Update osc-config ConfigMap with the resolved catalog source name + if oc get configmap osc-config -n default &>/dev/null; then + oc patch configmap osc-config -n default --type merge \ + -p "{\"data\":{\"catalogsourcename\":\"${CATALOG_SOURCE_NAME}\"}}" || true + echo ">>> Patched osc-config with catalogsourcename=${CATALOG_SOURCE_NAME}" + fi +} + #======================================== # Chart Fetching #======================================== @@ -198,8 +323,8 @@ function render_osc_operator_chart() { "--set" "namespaceOverride=${OSC_NAMESPACE}" ) - # TODO: use --set dev.enabled=true --set dev.image= once https://github.com/confidential-devhub/charts/pull/4 merges - echo ">>> Helm: using redhat-operators (catalog source patched via sed after render)" >&2 + # DEBUG until https://github.com/confidential-devhub/charts/pull/4 merges + echo ">>> DEBUG: Helm renders source=redhat-operators (patched via sed after render). PR 4 not merged yet." >&2 local helm_output if ! helm_output=$(helm template "${helm_args[@]}" 2>&1); then @@ -337,15 +462,9 @@ function install_osc_operator() { fi # TODO: remove sed workaround once https://github.com/confidential-devhub/charts/pull/4 merges - # The chart always renders Subscription source as redhat-operators; patch it - # to the catalog created by env-cm when running Pre-GA. - if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then - local catsrc_name - catsrc_name=$(oc get configmap osc-config -n default -o jsonpath='{.data.catalogsourcename}' 2>/dev/null || echo "") - if [[ -n "${catsrc_name}" && "${catsrc_name}" != "redhat-operators" ]]; then - echo ">>> Patching Subscription source: redhat-operators -> ${catsrc_name}" - sed -i "s/source: redhat-operators/source: ${catsrc_name}/" "${operator_yaml}" - fi + if [[ -n "${CATALOG_SOURCE_IMAGE}" && "${CATALOG_SOURCE_NAME}" != "redhat-operators" ]]; then + echo ">>> Patching Subscription source: redhat-operators -> ${CATALOG_SOURCE_NAME}" + sed -i "s/source: redhat-operators/source: ${CATALOG_SOURCE_NAME}/" "${operator_yaml}" fi echo ">>> Rendered operator objects:" @@ -399,17 +518,13 @@ function wait_for_operator() { return 1 fi - # Stage 1: Wait for the CatalogSource created by env-cm (e.g. brew-catalog) - if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then - local catsrc_name - catsrc_name=$(oc get configmap osc-config -n default -o jsonpath='{.data.catalogsourcename}' 2>/dev/null || echo "") - if [[ -n "${catsrc_name}" && "${catsrc_name}" != "redhat-operators" ]]; then - if ! wait_until "CatalogSource ${catsrc_name} READY" 60 5 \ - "[[ \"\$(oc get catalogsource -n openshift-marketplace '${catsrc_name}' -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null)\" == \"READY\" ]]"; then - oc get catalogsource -n openshift-marketplace || true - oc describe catalogsource -n openshift-marketplace "${catsrc_name}" || true - return 1 - fi + # Stage 1: Wait for the custom CatalogSource (e.g. brew-catalog) + if [[ -n "${CATALOG_SOURCE_IMAGE}" && "${CATALOG_SOURCE_NAME}" != "redhat-operators" ]]; then + if ! wait_until "CatalogSource ${CATALOG_SOURCE_NAME} READY" 60 5 \ + "[[ \"\$(oc get catalogsource -n openshift-marketplace '${CATALOG_SOURCE_NAME}' -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null)\" == \"READY\" ]]"; then + oc get catalogsource -n openshift-marketplace || true + oc describe catalogsource -n openshift-marketplace "${CATALOG_SOURCE_NAME}" || true + return 1 fi fi @@ -674,6 +789,9 @@ echo ">>> Workload: ${WORKLOAD_TO_TEST}" echo ">>> Peer-pods: ${ENABLEPEERPODS}" echo "=========================================" +# Phase 1: Set up CatalogSource (if Pre-GA) +setup_catalog_source + # Phase 2: Fetch charts CHARTS_DIR=$(fetch_osc_charts) echo ">>> Charts directory: ${CHARTS_DIR}" diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml index afe52a0a94137..8fc186d0ae5e6 100644 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml @@ -29,7 +29,13 @@ ref: documentation: |- The container image for a custom OSC operator CatalogSource. If empty (default), uses existing "redhat-operators" catalog. - If set, helm chart creates a new CatalogSource named "osc-operator-dev-catalog" with this image. + If set, creates a CatalogSource named by CATALOG_SOURCE_NAME with this image. + - name: CATALOG_SOURCE_NAME + default: "redhat-operators" + documentation: |- + The name for the CatalogSource to create when CATALOG_SOURCE_IMAGE is set. + Defaults to "redhat-operators" (no custom catalog created). + Set to e.g. "brew-catalog" for Pre-GA testing with a custom catalog image. - name: ENABLEPEERPODS default: "false" documentation: |- From dcda050b4bc4eb6fdab2876a663bb2ebf5af9f0e Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Mon, 3 Aug 2026 16:26:55 -0400 Subject: [PATCH 07/16] Bump peerpods-ga TEST_TIMEOUT to 120 for peer-pods test duration Co-Authored-By: Claude Opus 4.6 --- ...oxed-containers-operator-devel__downstream-candidate422.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml index 87eed99c39761..f19cb14ceef67 100644 --- a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml +++ b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml @@ -90,7 +90,7 @@ tests: TEST_FILTERS: ~DisconnectedOnly&;~Disruptive& TEST_RELEASE_TYPE: Pre-GA TEST_SCENARIOS: C00102 - TEST_TIMEOUT: "90" + TEST_TIMEOUT: "120" TRUSTEE_URL: "" WORKLOAD_TO_TEST: peer-pods workflow: sandboxed-containers-operator-e2e-azure From 42d19f272720e164ab19dd3f174b5c4250fc5992 Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Tue, 4 Aug 2026 08:04:01 -0400 Subject: [PATCH 08/16] Use helm chart's osc-operator-dev-catalog instead of brew-catalog Let the helm chart create its own CatalogSource (osc-operator-dev-catalog) by passing --set dev.enabled=true --set dev.image=, matching the trustee install pattern. Remove manual create_catsrc/wait_for_catsrc functions. Keep mirror_konflux and tag resolution in setup_catalog_source. Co-Authored-By: Claude Opus 4.6 --- ...-operator-install-osc-operator-commands.sh | 89 +++++-------------- ...ers-operator-install-osc-operator-ref.yaml | 8 +- 2 files changed, 22 insertions(+), 75 deletions(-) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh index e51e5204c8691..0a1e638fc5a71 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh @@ -31,7 +31,6 @@ export KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig} OSC_INSTALL=${OSC_INSTALL:-false} OSC_NAMESPACE=${OSC_NAMESPACE:-openshift-sandboxed-containers-operator} CATALOG_SOURCE_IMAGE=${CATALOG_SOURCE_IMAGE:-} -CATALOG_SOURCE_NAME=${CATALOG_SOURCE_NAME:-redhat-operators} OSC_CHARTS_REPO=${OSC_CHARTS_REPO:-https://github.com/confidential-devhub/charts.git} OSC_CHARTS_REF=${OSC_CHARTS_REF:-main} ENABLEPEERPODS=${ENABLEPEERPODS:-false} @@ -55,7 +54,7 @@ echo ">>> Namespace: ${OSC_NAMESPACE}" echo ">>> Workload: ${WORKLOAD_TO_TEST}" echo ">>> Peer-pods: ${ENABLEPEERPODS}" if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then - echo ">>> Catalog source: ${CATALOG_SOURCE_NAME} (image: ${CATALOG_SOURCE_IMAGE})" + echo ">>> Catalog source: ${OSC_DEV_CATALOG_NAME} (image: ${CATALOG_SOURCE_IMAGE})" else echo ">>> Catalog source: redhat-operators (using existing catalog)" fi @@ -167,57 +166,7 @@ function latest_catsrc_image_tag() { echo "latest" } -function create_catsrc() { - local catsrc_name="$1" - local catsrc_image="$2" - local catsrc_path="${SHARED_DIR}/catsrc_${catsrc_name}.yaml" - - echo ">>> Create a custom CatalogSource named ${catsrc_name} for internal builds" - - cat<<-EOF | tee "${catsrc_path}" - apiVersion: operators.coreos.com/v1alpha1 - kind: CatalogSource - metadata: - name: "${catsrc_name}" - namespace: openshift-marketplace - spec: - displayName: QE - image: "${catsrc_image}" - publisher: QE - sourceType: grpc -EOF - - oc apply -f "${catsrc_path}" -} - -function wait_for_catsrc() { - local catsrc_name="$1" - local timeout=300 - echo ">>> Waiting for CatalogSource ${catsrc_name} to be READY..." - - local deadline=$(( SECONDS + timeout )) - while (( SECONDS < deadline )); do - local state - state="$(oc get catalogsource -n openshift-marketplace \ - "${catsrc_name}" -o jsonpath='{.status.connectionState.lastObservedState}' \ - 2>/dev/null || echo "")" - if [[ "${state}" == "READY" ]]; then - echo ">>> CatalogSource ${catsrc_name} is READY" - return 0 - fi - sleep 10 - done - - state="$(oc get catalogsource -n openshift-marketplace "${catsrc_name}" \ - -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null || echo "")" - if [[ "${state}" != "READY" ]]; then - echo ">>> ERROR: CatalogSource ${catsrc_name} not READY after ${timeout}s (state: ${state})" >&2 - oc get catalogsource -n openshift-marketplace "${catsrc_name}" -o yaml || true - return 1 - fi - echo ">>> CatalogSource ${catsrc_name} is READY" - return 0 -} +OSC_DEV_CATALOG_NAME="osc-operator-dev-catalog" function setup_catalog_source() { if [[ -z "${CATALOG_SOURCE_IMAGE}" ]]; then @@ -225,7 +174,7 @@ function setup_catalog_source() { return 0 fi - echo ">>> Setting up CatalogSource for Pre-GA install" + echo ">>> Setting up catalog source image for Pre-GA install" mirror_konflux @@ -239,17 +188,15 @@ function setup_catalog_source() { echo ">>> Using provided catalog image: ${CATALOG_SOURCE_IMAGE}" fi - create_catsrc "${CATALOG_SOURCE_NAME}" "${CATALOG_SOURCE_IMAGE}" - wait_for_catsrc "${CATALOG_SOURCE_NAME}" - echo "CATALOG_SOURCE_IMAGE=${CATALOG_SOURCE_IMAGE}" > "${SHARED_DIR}/catalog-source-image.env" echo ">>> Saved resolved CATALOG_SOURCE_IMAGE to ${SHARED_DIR}/catalog-source-image.env" + echo ">>> Helm chart will create CatalogSource ${OSC_DEV_CATALOG_NAME} with this image" - # Update osc-config ConfigMap with the resolved catalog source name + # Update osc-config ConfigMap with the chart's catalog name if oc get configmap osc-config -n default &>/dev/null; then oc patch configmap osc-config -n default --type merge \ - -p "{\"data\":{\"catalogsourcename\":\"${CATALOG_SOURCE_NAME}\"}}" || true - echo ">>> Patched osc-config with catalogsourcename=${CATALOG_SOURCE_NAME}" + -p "{\"data\":{\"catalogsourcename\":\"${OSC_DEV_CATALOG_NAME}\"}}" || true + echo ">>> Patched osc-config with catalogsourcename=${OSC_DEV_CATALOG_NAME}" fi } @@ -324,6 +271,12 @@ function render_osc_operator_chart() { ) # DEBUG until https://github.com/confidential-devhub/charts/pull/4 merges + if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then + helm_args+=("--set" "dev.enabled=true" "--set" "dev.image=${CATALOG_SOURCE_IMAGE}") + echo ">>> Helm: dev.enabled=true, dev.image=${CATALOG_SOURCE_IMAGE}" >&2 + else + helm_args+=("--set" "dev.enabled=false") + fi echo ">>> DEBUG: Helm renders source=redhat-operators (patched via sed after render). PR 4 not merged yet." >&2 local helm_output @@ -462,9 +415,9 @@ function install_osc_operator() { fi # TODO: remove sed workaround once https://github.com/confidential-devhub/charts/pull/4 merges - if [[ -n "${CATALOG_SOURCE_IMAGE}" && "${CATALOG_SOURCE_NAME}" != "redhat-operators" ]]; then - echo ">>> Patching Subscription source: redhat-operators -> ${CATALOG_SOURCE_NAME}" - sed -i "s/source: redhat-operators/source: ${CATALOG_SOURCE_NAME}/" "${operator_yaml}" + if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then + echo ">>> Patching Subscription source: redhat-operators -> ${OSC_DEV_CATALOG_NAME}" + sed -i "s/source: redhat-operators/source: ${OSC_DEV_CATALOG_NAME}/" "${operator_yaml}" fi echo ">>> Rendered operator objects:" @@ -518,12 +471,12 @@ function wait_for_operator() { return 1 fi - # Stage 1: Wait for the custom CatalogSource (e.g. brew-catalog) - if [[ -n "${CATALOG_SOURCE_IMAGE}" && "${CATALOG_SOURCE_NAME}" != "redhat-operators" ]]; then - if ! wait_until "CatalogSource ${CATALOG_SOURCE_NAME} READY" 60 5 \ - "[[ \"\$(oc get catalogsource -n openshift-marketplace '${CATALOG_SOURCE_NAME}' -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null)\" == \"READY\" ]]"; then + # Stage 1: Wait for the dev CatalogSource created by the helm chart + if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then + if ! wait_until "CatalogSource ${OSC_DEV_CATALOG_NAME} READY" 300 5 \ + "[[ \"\$(oc get catalogsource -n openshift-marketplace '${OSC_DEV_CATALOG_NAME}' -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null)\" == \"READY\" ]]"; then oc get catalogsource -n openshift-marketplace || true - oc describe catalogsource -n openshift-marketplace "${CATALOG_SOURCE_NAME}" || true + oc describe catalogsource -n openshift-marketplace "${OSC_DEV_CATALOG_NAME}" || true return 1 fi fi diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml index 8fc186d0ae5e6..53b42aecd223d 100644 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-ref.yaml @@ -29,13 +29,7 @@ ref: documentation: |- The container image for a custom OSC operator CatalogSource. If empty (default), uses existing "redhat-operators" catalog. - If set, creates a CatalogSource named by CATALOG_SOURCE_NAME with this image. - - name: CATALOG_SOURCE_NAME - default: "redhat-operators" - documentation: |- - The name for the CatalogSource to create when CATALOG_SOURCE_IMAGE is set. - Defaults to "redhat-operators" (no custom catalog created). - Set to e.g. "brew-catalog" for Pre-GA testing with a custom catalog image. + If set, the helm chart creates "osc-operator-dev-catalog" with this image. - name: ENABLEPEERPODS default: "false" documentation: |- From ac39ded42dc1a8f4f93cc1b3bb3ad00ef45122a9 Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Tue, 4 Aug 2026 09:29:53 -0400 Subject: [PATCH 09/16] Move OSC_DEV_CATALOG_NAME to config section to fix unbound variable Co-Authored-By: Claude Opus 4.6 --- ...dboxed-containers-operator-install-osc-operator-commands.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh index 0a1e638fc5a71..b7ffb51944505 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh @@ -35,6 +35,7 @@ OSC_CHARTS_REPO=${OSC_CHARTS_REPO:-https://github.com/confidential-devhub/charts OSC_CHARTS_REF=${OSC_CHARTS_REF:-main} ENABLEPEERPODS=${ENABLEPEERPODS:-false} WORKLOAD_TO_TEST=${WORKLOAD_TO_TEST:-kata} +OSC_DEV_CATALOG_NAME="osc-operator-dev-catalog" # Early exit if installation disabled if [[ "${OSC_INSTALL}" != "true" ]]; then @@ -166,8 +167,6 @@ function latest_catsrc_image_tag() { echo "latest" } -OSC_DEV_CATALOG_NAME="osc-operator-dev-catalog" - function setup_catalog_source() { if [[ -z "${CATALOG_SOURCE_IMAGE}" ]]; then echo ">>> No CATALOG_SOURCE_IMAGE set, using existing redhat-operators catalog" From e6e163bfd0d1b4c2b08e7a42af0d7faf06e8d593 Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Tue, 4 Aug 2026 09:32:09 -0400 Subject: [PATCH 10/16] Remove DEBUG workarounds for merged charts PRs #3 and #4 Charts PRs #3 (startingCSV removal) and #4 (dev.enabled/dev.image, kataconfig naming) have merged. Remove all sed workarounds. Co-Authored-By: Claude Opus 4.6 --- ...-operator-install-osc-operator-commands.sh | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh index b7ffb51944505..b46f8e6ff6cda 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh @@ -225,12 +225,6 @@ function fetch_osc_charts() { result_dir="${charts_dir}" fi - # DEBUG until https://github.com/confidential-devhub/charts/pull/3 merges - if grep -q 'startingCSV' "${result_dir}/osc-operator/templates/subscription.yaml" 2>/dev/null; then - sed -i '/startingCSV/d' "${result_dir}/osc-operator/templates/subscription.yaml" - echo ">>> DEBUG: subscription.yaml patched (removed startingCSV line). PR 3 not merged yet." >&2 - fi - echo "${result_dir}" } @@ -269,14 +263,12 @@ function render_osc_operator_chart() { "--set" "namespaceOverride=${OSC_NAMESPACE}" ) - # DEBUG until https://github.com/confidential-devhub/charts/pull/4 merges if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then helm_args+=("--set" "dev.enabled=true" "--set" "dev.image=${CATALOG_SOURCE_IMAGE}") echo ">>> Helm: dev.enabled=true, dev.image=${CATALOG_SOURCE_IMAGE}" >&2 else helm_args+=("--set" "dev.enabled=false") fi - echo ">>> DEBUG: Helm renders source=redhat-operators (patched via sed after render). PR 4 not merged yet." >&2 local helm_output if ! helm_output=$(helm template "${helm_args[@]}" 2>&1); then @@ -413,12 +405,6 @@ function install_osc_operator() { return 1 fi - # TODO: remove sed workaround once https://github.com/confidential-devhub/charts/pull/4 merges - if [[ -n "${CATALOG_SOURCE_IMAGE}" ]]; then - echo ">>> Patching Subscription source: redhat-operators -> ${OSC_DEV_CATALOG_NAME}" - sed -i "s/source: redhat-operators/source: ${OSC_DEV_CATALOG_NAME}/" "${operator_yaml}" - fi - echo ">>> Rendered operator objects:" oc apply -f "${operator_yaml}" --dry-run=client -o name || true @@ -541,13 +527,6 @@ function install_osc_operands() { return 1 fi - # DEBUG until https://github.com/confidential-devhub/charts/pull/4 merges - # openshift-tests-private expects kataconfig name to be "example-kataconfig". Rename to match the test expectation. - if grep -qE '^\s+name: kataconfig$' "${operands_yaml}" 2>/dev/null; then - sed -i -E 's/^([[:space:]]*)name: kataconfig$/\1name: example-kataconfig/' "${operands_yaml}" - echo ">>> DEBUG: kataconfig.yaml patched (renamed to example-kataconfig). PR 4 not merged yet." >&2 - fi - echo ">>> Rendered operands objects:" oc apply -f "${operands_yaml}" --dry-run=client -o name || true From f9a049abed5d20c10f7df4bde7a5d55e2921ce4b Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Tue, 4 Aug 2026 12:15:37 -0400 Subject: [PATCH 11/16] Use --set-string for helm providersConfigs to fix type mismatch Helm --set parses bare numbers (e.g. VXLAN_PORT=9000) as integers, but the chart template uses ne/hasPrefix which require strings. Also add --set kataconfig.name=example-kataconfig for test compat. Co-Authored-By: Claude Opus 4.6 --- ...-operator-install-osc-operator-commands.sh | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh index b46f8e6ff6cda..c53cb0a33da56 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh @@ -298,6 +298,9 @@ function render_osc_operands_chart() { "--set" "namespaceOverride=${OSC_NAMESPACE}" ) + # openshift-tests-private expects kataconfig named "example-kataconfig" + helm_args+=("--set" "kataconfig.name=example-kataconfig") + # Workload-specific settings if [[ "${WORKLOAD_TO_TEST}" == "coco" ]]; then helm_args+=("--set" "confidential.enabled=true") @@ -327,8 +330,8 @@ function render_osc_operands_chart() { local vxlan_port proxy_timeout vxlan_port=$(echo "${cm_data}" | jq -r '.data.VXLAN_PORT // ""') proxy_timeout=$(echo "${cm_data}" | jq -r '.data.PROXY_TIMEOUT // ""') - [[ -n "${vxlan_port}" ]] && helm_args+=("--set" "peerpods.providersConfigs.all.VXLAN_PORT=${vxlan_port}") - [[ -n "${proxy_timeout}" ]] && helm_args+=("--set" "peerpods.providersConfigs.all.PROXY_TIMEOUT=${proxy_timeout}") + [[ -n "${vxlan_port}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.all.VXLAN_PORT=${vxlan_port}") + [[ -n "${proxy_timeout}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.all.PROXY_TIMEOUT=${proxy_timeout}") case "${provider}" in azure) @@ -338,12 +341,11 @@ function render_osc_operands_chart() { azure_resource_group=$(echo "${cm_data}" | jq -r '.data.AZURE_RESOURCE_GROUP // ""') azure_region=$(echo "${cm_data}" | jq -r '.data.AZURE_REGION // ""') azure_instance_size=$(echo "${cm_data}" | jq -r '.data.AZURE_INSTANCE_SIZE // ""') - [[ -n "${azure_subnet_id}" ]] && helm_args+=("--set" "peerpods.providersConfigs.azure.AZURE_SUBNET_ID=${azure_subnet_id}") - [[ -n "${azure_nsg_id}" ]] && helm_args+=("--set" "peerpods.providersConfigs.azure.AZURE_NSG_ID=${azure_nsg_id}") - [[ -n "${azure_resource_group}" ]] && helm_args+=("--set" "peerpods.providersConfigs.azure.AZURE_RESOURCE_GROUP=${azure_resource_group}") - [[ -n "${azure_region}" ]] && helm_args+=("--set" "peerpods.providersConfigs.azure.AZURE_REGION=${azure_region}") - # Let chart auto-set AZURE_INSTANCE_SIZE based on confidential mode if not explicitly set - [[ -n "${azure_instance_size}" ]] && helm_args+=("--set" "peerpods.providersConfigs.azure.AZURE_INSTANCE_SIZE=${azure_instance_size}") + [[ -n "${azure_subnet_id}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.azure.AZURE_SUBNET_ID=${azure_subnet_id}") + [[ -n "${azure_nsg_id}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.azure.AZURE_NSG_ID=${azure_nsg_id}") + [[ -n "${azure_resource_group}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.azure.AZURE_RESOURCE_GROUP=${azure_resource_group}") + [[ -n "${azure_region}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.azure.AZURE_REGION=${azure_region}") + [[ -n "${azure_instance_size}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.azure.AZURE_INSTANCE_SIZE=${azure_instance_size}") ;; aws) local aws_region aws_subnet_id aws_vpc_id aws_sg_ids podvm_instance_type @@ -352,11 +354,11 @@ function render_osc_operands_chart() { aws_vpc_id=$(echo "${cm_data}" | jq -r '.data.AWS_VPC_ID // ""') aws_sg_ids=$(echo "${cm_data}" | jq -r '.data.AWS_SG_IDS // ""') podvm_instance_type=$(echo "${cm_data}" | jq -r '.data.PODVM_INSTANCE_TYPE // ""') - [[ -n "${aws_region}" ]] && helm_args+=("--set" "peerpods.providersConfigs.aws.AWS_REGION=${aws_region}") - [[ -n "${aws_subnet_id}" ]] && helm_args+=("--set" "peerpods.providersConfigs.aws.AWS_SUBNET_ID=${aws_subnet_id}") - [[ -n "${aws_vpc_id}" ]] && helm_args+=("--set" "peerpods.providersConfigs.aws.AWS_VPC_ID=${aws_vpc_id}") - [[ -n "${aws_sg_ids}" ]] && helm_args+=("--set" "peerpods.providersConfigs.aws.AWS_SG_IDS=${aws_sg_ids}") - [[ -n "${podvm_instance_type}" ]] && helm_args+=("--set" "peerpods.providersConfigs.aws.PODVM_INSTANCE_TYPE=${podvm_instance_type}") + [[ -n "${aws_region}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.aws.AWS_REGION=${aws_region}") + [[ -n "${aws_subnet_id}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.aws.AWS_SUBNET_ID=${aws_subnet_id}") + [[ -n "${aws_vpc_id}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.aws.AWS_VPC_ID=${aws_vpc_id}") + [[ -n "${aws_sg_ids}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.aws.AWS_SG_IDS=${aws_sg_ids}") + [[ -n "${podvm_instance_type}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.aws.PODVM_INSTANCE_TYPE=${podvm_instance_type}") ;; gcp) local gcp_project_id gcp_zone gcp_network gcp_machine_type @@ -364,10 +366,10 @@ function render_osc_operands_chart() { gcp_zone=$(echo "${cm_data}" | jq -r '.data.GCP_ZONE // ""') gcp_network=$(echo "${cm_data}" | jq -r '.data.GCP_NETWORK // ""') gcp_machine_type=$(echo "${cm_data}" | jq -r '.data.GCP_MACHINE_TYPE // ""') - [[ -n "${gcp_project_id}" ]] && helm_args+=("--set" "peerpods.providersConfigs.gcp.GCP_PROJECT_ID=${gcp_project_id}") - [[ -n "${gcp_zone}" ]] && helm_args+=("--set" "peerpods.providersConfigs.gcp.GCP_ZONE=${gcp_zone}") - [[ -n "${gcp_network}" ]] && helm_args+=("--set" "peerpods.providersConfigs.gcp.GCP_NETWORK=${gcp_network}") - [[ -n "${gcp_machine_type}" ]] && helm_args+=("--set" "peerpods.providersConfigs.gcp.GCP_MACHINE_TYPE=${gcp_machine_type}") + [[ -n "${gcp_project_id}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.gcp.GCP_PROJECT_ID=${gcp_project_id}") + [[ -n "${gcp_zone}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.gcp.GCP_ZONE=${gcp_zone}") + [[ -n "${gcp_network}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.gcp.GCP_NETWORK=${gcp_network}") + [[ -n "${gcp_machine_type}" ]] && helm_args+=("--set-string" "peerpods.providersConfigs.gcp.GCP_MACHINE_TYPE=${gcp_machine_type}") ;; esac else From 4428999027a2965bf4b61aa66dbe381131fc770b Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Wed, 5 Aug 2026 08:06:29 -0400 Subject: [PATCH 12/16] Fix peer-pods-secret key format for Azure credentials The operator expects individual keys (AZURE_CLIENT_ID, AZURE_TENANT_ID, etc.) not a JSON file. Use --from-literal instead of --from-file. Co-Authored-By: Claude Opus 4.6 --- ...rs-operator-install-osc-operator-commands.sh | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh index c53cb0a33da56..026783af0be34 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh @@ -604,18 +604,15 @@ function create_peer_pods_secret() { local subscription_id subscription_id=$(oc get secret azure-credentials -n kube-system -o jsonpath='{.data.azure_subscription_id}' 2>/dev/null | base64 -d || echo "") - cat > "${SCRATCH}/peer-pods-secret.json" <>> WARNING: Could not extract Azure credentials from peerpods-param-secret" fi From f81a82640e91435908431c4211cf6414f15dbc65 Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Wed, 5 Aug 2026 10:59:51 -0400 Subject: [PATCH 13/16] Fix set -e exit after peer-pods-secret creation The WAS_TRACING pattern ($false && set -x) returns exit code 1, which set -e catches. Since the script never enables tracing, remove the save/restore entirely. Co-Authored-By: Claude Opus 4.6 --- ...ndboxed-containers-operator-install-osc-operator-commands.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh index 026783af0be34..186eb916c2d85 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh @@ -604,7 +604,6 @@ function create_peer_pods_secret() { local subscription_id subscription_id=$(oc get secret azure-credentials -n kube-system -o jsonpath='{.data.azure_subscription_id}' 2>/dev/null | base64 -d || echo "") - [[ $- == *x* ]] && WAS_TRACING=true || WAS_TRACING=false set +x oc create secret generic peer-pods-secret \ -n "${OSC_NAMESPACE}" \ @@ -612,7 +611,6 @@ function create_peer_pods_secret() { --from-literal="AZURE_CLIENT_SECRET=${client_secret}" \ --from-literal="AZURE_TENANT_ID=${tenant_id}" \ --from-literal="AZURE_SUBSCRIPTION_ID=${subscription_id}" - $WAS_TRACING && set -x else echo ">>> WARNING: Could not extract Azure credentials from peerpods-param-secret" fi From bba55895d2efce993accaf300f2f9d930b1b9098 Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Wed, 5 Aug 2026 14:11:01 -0400 Subject: [PATCH 14/16] =?UTF-8?q?Remove=20verify=5Fpeerpod=5Fpods=20?= =?UTF-8?q?=E2=80=94=20KataConfig=20status=20is=20sufficient?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verify_peerpod_pods() function used stale label selectors (app=cloud-api-adaptor, app=peerpodconfig-ctrl-caa-daemon) that don't match current OSC pod labels (name=osc-caa-ds, peer-pods-webhook). Rather than fix the labels, remove the check entirely — KataConfig InProgress=False already confirms the operator finished reconciling nodes, runtime classes, and all managed pods. Co-Authored-By: Claude Opus 4.6 --- ...-operator-install-osc-operator-commands.sh | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh index 186eb916c2d85..7b33d79d30314 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/install-osc-operator/sandboxed-containers-operator-install-osc-operator-commands.sh @@ -667,28 +667,6 @@ function wait_for_kataconfig() { oc get kataconfig "${kataconfig_name}" -o jsonpath='{.status}' 2>/dev/null | jq . || true } -function verify_peerpod_pods() { - echo ">>> Verifying peer-pod control pods" - - # Wait for cloud-api-adaptor daemonset pods - if ! wait_until "cloud-api-adaptor pods running" 600 10 \ - "oc get daemonset -n '${OSC_NAMESPACE}' -l app=cloud-api-adaptor -o jsonpath='{.items[0].status.numberReady}' 2>/dev/null | grep -qE '^[1-9]'"; then - echo ">>> WARNING: cloud-api-adaptor daemonset not ready" - oc get daemonset -n "${OSC_NAMESPACE}" || true - oc get pods -n "${OSC_NAMESPACE}" -l app=cloud-api-adaptor || true - fi - - # Wait for peerpodconfig-ctrl-caa-daemon - if ! wait_until "peerpodconfig webhook pod running" 300 10 \ - "oc get pods -n '${OSC_NAMESPACE}' -l app=peerpodconfig-ctrl-caa-daemon -o jsonpath='{.items[0].status.phase}' 2>/dev/null | grep -q 'Running'"; then - echo ">>> WARNING: peerpodconfig webhook pod not running" - oc get pods -n "${OSC_NAMESPACE}" || true - fi - - echo ">>> Peer-pod pods:" - oc get pods -n "${OSC_NAMESPACE}" || true -} - #======================================== # Update Shared State #======================================== @@ -736,10 +714,6 @@ fi install_osc_operands "${CHARTS_DIR}" wait_for_kataconfig -if [[ "${ENABLEPEERPODS}" == "true" ]]; then - verify_peerpod_pods -fi - # Phase 5: Update shared state update_osc_config From 178ce3142e4c1e4f9ed2540a82ba4178b37d14fe Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Wed, 5 Aug 2026 15:21:46 -0400 Subject: [PATCH 15/16] Set peerpods-ga SLEEP_DURATION to 0h for rehearsal Co-Authored-By: Claude Opus 4.6 --- ...oxed-containers-operator-devel__downstream-candidate422.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml index f19cb14ceef67..e0563043ff58d 100644 --- a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml +++ b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate422.yaml @@ -86,7 +86,7 @@ tests: MUST_GATHER_ON_FAILURE_ONLY: "false" OSC_INSTALL: "true" RUNTIMECLASS: kata-remote - SLEEP_DURATION: 8h + SLEEP_DURATION: 0h TEST_FILTERS: ~DisconnectedOnly&;~Disruptive& TEST_RELEASE_TYPE: Pre-GA TEST_SCENARIOS: C00102 From a2d669b8979f2afe5c06513872c30c27e841c44c Mon Sep 17 00:00:00 2001 From: Tom Buskey Date: Thu, 6 Aug 2026 09:18:55 -0400 Subject: [PATCH 16/16] [DEBUG][DO NOT MERGE] Signed-off-by: Tom Buskey --- ...perator-devel__downstream-candidate421.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate421.yaml b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate421.yaml index d81946bb9bc24..4b7b413f87482 100644 --- a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate421.yaml +++ b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate421.yaml @@ -45,10 +45,10 @@ tests: ENABLE_MUST_GATHER: "true" INITDATA: "" INSTALL_KATA_RPM: "true" - KATA_RPM_VERSION: 3.31.0-2.rhaos4.19.el9 + KATA_RPM_VERSION: 3.31.0-4.rhaos4.19.el9 MUST_GATHER_IMAGE: registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest MUST_GATHER_ON_FAILURE_ONLY: "false" - SLEEP_DURATION: 0h + SLEEP_DURATION: 8h TEST_FILTERS: ~DisconnectedOnly&;~Disruptive& TEST_RELEASE_TYPE: Pre-GA TEST_SCENARIOS: sig-kata.*Kata Author @@ -81,11 +81,11 @@ tests: ENABLEPEERPODS: "true" INITDATA: "" INSTALL_KATA_RPM: "true" - KATA_RPM_VERSION: 3.31.0-2.rhaos4.19.el9 + KATA_RPM_VERSION: 3.31.0-4.rhaos4.19.el9 MUST_GATHER_IMAGE: registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest MUST_GATHER_ON_FAILURE_ONLY: "false" RUNTIMECLASS: kata-remote - SLEEP_DURATION: 0h + SLEEP_DURATION: 8h TEST_FILTERS: ~DisconnectedOnly&;~Disruptive&;~C00133& TEST_RELEASE_TYPE: Pre-GA TEST_SCENARIOS: sig-kata.*Kata Author @@ -119,7 +119,7 @@ tests: ENABLEPEERPODS: "true" INITDATA: "" INSTALL_KATA_RPM: "true" - KATA_RPM_VERSION: 3.31.0-2.rhaos4.19.el9 + KATA_RPM_VERSION: 3.31.0-4.rhaos4.19.el9 MUST_GATHER_IMAGE: registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest MUST_GATHER_ON_FAILURE_ONLY: "false" RUNTIMECLASS: kata-remote @@ -157,7 +157,7 @@ tests: HYPERSHIFT_AZURE_LOCATION: eastus INITDATA: "" INSTALL_KATA_RPM: "true" - KATA_RPM_VERSION: 3.31.0-2.rhaos4.19.el9 + KATA_RPM_VERSION: 3.31.0-4.rhaos4.19.el9 LOCATION: eastus MUST_GATHER_IMAGE: registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest MUST_GATHER_ON_FAILURE_ONLY: "false" @@ -196,7 +196,7 @@ tests: HYPERSHIFT_AZURE_LOCATION: eastus INITDATA: "" INSTALL_KATA_RPM: "true" - KATA_RPM_VERSION: 3.31.0-2.rhaos4.19.el9 + KATA_RPM_VERSION: 3.31.0-4.rhaos4.19.el9 LOCATION: eastus MUST_GATHER_IMAGE: registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest MUST_GATHER_ON_FAILURE_ONLY: "false" @@ -234,7 +234,7 @@ tests: ENABLEPEERPODS: "true" INITDATA: "" INSTALL_KATA_RPM: "true" - KATA_RPM_VERSION: 3.31.0-2.rhaos4.19.el9 + KATA_RPM_VERSION: 3.31.0-4.rhaos4.19.el9 MUST_GATHER_IMAGE: registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest MUST_GATHER_ON_FAILURE_ONLY: "false" RUNTIMECLASS: kata-remote @@ -271,7 +271,7 @@ tests: ENABLEPEERPODS: "true" INITDATA: "" INSTALL_KATA_RPM: "true" - KATA_RPM_VERSION: 3.31.0-2.rhaos4.19.el9 + KATA_RPM_VERSION: 3.31.0-4.rhaos4.19.el9 MUST_GATHER_IMAGE: registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest MUST_GATHER_ON_FAILURE_ONLY: "false" RUNTIMECLASS: kata-remote