diff --git a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-telco-x86-regulus.yaml b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-telco-x86-regulus.yaml new file mode 100644 index 0000000000000..1e4d591934028 --- /dev/null +++ b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-telco-x86-regulus.yaml @@ -0,0 +1,70 @@ +base_images: + cerberus: + name: cerberus + namespace: chaos + tag: cerberus-prow + ocp-qe-perfscale-ci: + name: ocp-qe-perfscale-ci + namespace: ci + tag: latest +build_root: + image_stream_tag: + name: ci-tools-build-root + namespace: ci + tag: latest +releases: + latest: + candidate: + product: ocp + stream: nightly + version: "4.22" +resources: + '*': + limits: + memory: 4Gi + requests: + cpu: 100m + memory: 200Mi +tests: +- as: jetlag + capabilities: + - intranet + cron: 0 17 * * 0 + restrict_network_access: false + steps: + cluster_profile: metal-perfscale-jetlag + env: + CRUCIBLE: "true" + test: + - ref: openshift-qe-installer-bm-regulus + - ref: openshift-qe-orion-regulus + workflow: openshift-qe-installer-bm-deploy + timeout: 8h0m0s +- always_run: false + as: no-jetlag + capabilities: + - intranet + restrict_network_access: false + steps: + cluster_profile: metal-perfscale-jetlag + test: + - ref: openshift-qe-installer-bm-load-kubeconfig + - ref: openshift-qe-cluster-health + - ref: openshift-qe-installer-bm-regulus + - ref: openshift-qe-orion-regulus + timeout: 6h0m0s +- always_run: false + as: orion-only + capabilities: + - intranet + restrict_network_access: false + steps: + cluster_profile: metal-perfscale-jetlag + test: + - ref: openshift-qe-orion-regulus + timeout: 1h0m0s +zz_generated_metadata: + branch: main + org: openshift-eng + repo: ocp-qe-perfscale-ci + variant: metal-telco-x86-regulus diff --git a/ci-operator/jobs/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main-periodics.yaml b/ci-operator/jobs/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main-periodics.yaml index 8024b088db152..269265348aba0 100644 --- a/ci-operator/jobs/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main-periodics.yaml +++ b/ci-operator/jobs/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main-periodics.yaml @@ -2966,3 +2966,88 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build10 + cron: 0 17 * * 0 + decorate: true + decoration_config: + skip_cloning: true + timeout: 8h0m0s + extra_refs: + - base_ref: main + org: openshift-eng + repo: ocp-qe-perfscale-ci + labels: + capability/intranet: intranet + ci-operator.openshift.io/cloud: metal-perfscale-jetlag + ci-operator.openshift.io/cloud-cluster-profile: metal-perfscale-jetlag + ci-operator.openshift.io/variant: metal-telco-x86-regulus + ci.openshift.io/generator: prowgen + job-release: "4.22" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-eng-ocp-qe-perfscale-ci-main-metal-telco-x86-regulus-jetlag + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=jetlag + - --variant=metal-telco-x86-regulus + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator diff --git a/ci-operator/jobs/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main-presubmits.yaml b/ci-operator/jobs/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main-presubmits.yaml index 09c9f1b6a423d..69bed6de75ac9 100644 --- a/ci-operator/jobs/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main-presubmits.yaml @@ -4991,6 +4991,180 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )(metal-rhoso-x86-weekly-compact-6nodes|remaining-required),?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build11 + context: ci/prow/metal-telco-x86-regulus-no-jetlag + decorate: true + decoration_config: + skip_cloning: true + timeout: 6h0m0s + labels: + capability/intranet: intranet + ci-operator.openshift.io/cloud: metal-perfscale-jetlag + ci-operator.openshift.io/cloud-cluster-profile: metal-perfscale-jetlag + ci-operator.openshift.io/variant: metal-telco-x86-regulus + ci.openshift.io/generator: prowgen + job-release: "4.22" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-ocp-qe-perfscale-ci-main-metal-telco-x86-regulus-no-jetlag + rerun_command: /test metal-telco-x86-regulus-no-jetlag + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=no-jetlag + - --variant=metal-telco-x86-regulus + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )(metal-telco-x86-regulus-no-jetlag|remaining-required),?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build11 + context: ci/prow/metal-telco-x86-regulus-orion-only + decorate: true + decoration_config: + skip_cloning: true + timeout: 1h0m0s + labels: + capability/intranet: intranet + ci-operator.openshift.io/cloud: metal-perfscale-jetlag + ci-operator.openshift.io/cloud-cluster-profile: metal-perfscale-jetlag + ci-operator.openshift.io/variant: metal-telco-x86-regulus + ci.openshift.io/generator: prowgen + job-release: "4.22" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-ocp-qe-perfscale-ci-main-metal-telco-x86-regulus-orion-only + rerun_command: /test metal-telco-x86-regulus-orion-only + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=orion-only + - --variant=metal-telco-x86-regulus + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )(metal-telco-x86-regulus-orion-only|remaining-required),?($|\s.*) - agent: kubernetes always_run: false branches: diff --git a/ci-operator/step-registry/openshift-qe/installer/bm/regulus/openshift-qe-installer-bm-regulus-commands.sh b/ci-operator/step-registry/openshift-qe/installer/bm/regulus/openshift-qe-installer-bm-regulus-commands.sh index 31f126b95c73a..823cf3b6d4837 100644 --- a/ci-operator/step-registry/openshift-qe/installer/bm/regulus/openshift-qe-installer-bm-regulus-commands.sh +++ b/ci-operator/step-registry/openshift-qe/installer/bm/regulus/openshift-qe-installer-bm-regulus-commands.sh @@ -159,7 +159,20 @@ install-regulus() { install-regulus # ───────────────────────────────────────────────────────────────────────────── -# Generate Regulus lab.config +# Read ES credentials from mounted secret (optional - will be empty if not present) +# Disable tracing to avoid exposing credentials in CI logs +# ───────────────────────────────────────────────────────────────────────────── +[[ $- == *x* ]] && _WAS_TRACING=true || _WAS_TRACING=false +set +x +ES_PASSWORD=$(cat "/secret/perfscale-prod/password" 2>/dev/null || echo "") +ES_USER=$(cat "/secret/perfscale-prod/username" 2>/dev/null || echo "") +ES_HOST=$(cat "/secret/perfscale-prod/host" 2>/dev/null || echo "") +ES_PROTOCOL="https" +export ES_PASSWORD ES_USER ES_HOST ES_PROTOCOL +$_WAS_TRACING && set -x + +# ───────────────────────────────────────────────────────────────────────────── +# Generate Regulus lab.config # ───────────────────────────────────────────────────────────────────────────── vars=( KUBECONFIG @@ -183,12 +196,20 @@ vars=( TREX_SRIOV_INTERFACE_2 TREX_DPDK_NIC_MODEL REM_DPDK_CONFIG + ES_PROTOCOL + ES_HOST + ES_USER + ES_PASSWORD ) if [ -e /tmp/lab.config ]; then rm /tmp/lab.config fi +# Disable tracing during lab.config generation (contains credentials) +[[ $- == *x* ]] && _WAS_TRACING=true || _WAS_TRACING=false +set +x +umask 077 cat > /tmp/lab.config <> /tmp/lab.config done +$_WAS_TRACING && set -x # ─────────────────────────────────────────────────────────────────────────── # Launch Regulus (tests are listed in regulus_repo/jobs.config) diff --git a/ci-operator/step-registry/openshift-qe/installer/bm/regulus/openshift-qe-installer-bm-regulus-ref.yaml b/ci-operator/step-registry/openshift-qe/installer/bm/regulus/openshift-qe-installer-bm-regulus-ref.yaml index d5be2a506328a..6c48f01515691 100644 --- a/ci-operator/step-registry/openshift-qe/installer/bm/regulus/openshift-qe-installer-bm-regulus-ref.yaml +++ b/ci-operator/step-registry/openshift-qe/installer/bm/regulus/openshift-qe-installer-bm-regulus-ref.yaml @@ -105,4 +105,8 @@ ref: default: "false" documentation: |- Automatic initialize Trex. Else user must config Trex DPDK. + credentials: + - namespace: test-credentials + name: ocp-perfscale-prod-es-creds + mount_path: /secret/perfscale-prod diff --git a/ci-operator/step-registry/openshift-qe/orion/regulus/OWNERS b/ci-operator/step-registry/openshift-qe/orion/regulus/OWNERS new file mode 120000 index 0000000000000..8c272259fbba9 --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/orion/regulus/OWNERS @@ -0,0 +1 @@ +../../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/openshift-qe/orion/regulus/openshift-qe-orion-regulus-commands.sh b/ci-operator/step-registry/openshift-qe/orion/regulus/openshift-qe-orion-regulus-commands.sh new file mode 100755 index 0000000000000..811335d4fccfb --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/orion/regulus/openshift-qe-orion-regulus-commands.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# Orion Regulus - Dynamic fingerprint-based regression detection for Regulus tests +# +# This step clones the Regulus repo and uses its ORION/analyze-batch.py directly. +# The ORION directory is the single source of truth for the batch analysis logic. +# +# Workflow: +# 1. Set up Python virtualenv +# 2. Clone and install Orion (cloud-bulldozer/orion CLI tool) +# 3. Clone Regulus repo (use ORION/ subdirectory) +# 4. Install ORION dependencies +# 5. Run prow-entry.sh (bridges Prow env vars to analyze-batch.py) +# +set -o errexit +set -o nounset +set -o pipefail + +MAX_RETRIES=5 + +echo "==================================" +echo "Orion Regulus - Dynamic Regression Detection" +echo "==================================" +echo "" + +python --version +pushd /tmp || exit 1 + +# ── Set up Python virtual environment ───────────────────────────────────────── +echo "Setting up Python virtual environment..." +python -m virtualenv ./venv_orion_regulus +source ./venv_orion_regulus/bin/activate + +# ── Clone and install Orion CLI ─────────────────────────────────────────────── +if [[ "$ORION_TAG" == "latest" ]]; then + LATEST_TAG=$(git ls-remote --tags "${ORION_REPO}" | awk -F'refs/tags/' '{print $2}' | grep -v '\^{}' | sort -V | tail -n1) +else + LATEST_TAG="$ORION_TAG" +fi + +echo "Cloning Orion from ${ORION_REPO} (tag: ${LATEST_TAG})..." +for attempt in $(seq 1 "$MAX_RETRIES"); do + rm -rf orion + if git clone -q --branch "$LATEST_TAG" "$ORION_REPO" --depth 1; then + echo "Successfully cloned Orion" + break + fi + if [[ "$attempt" -eq "$MAX_RETRIES" ]]; then + echo "ERROR: git clone orion failed after $MAX_RETRIES attempts" >&2 + exit 1 + fi + echo "git clone failed (attempt $attempt/$MAX_RETRIES), retrying in 10s..." >&2 + sleep 10 +done + +pushd orion || exit 1 +pip install -q --retries "$MAX_RETRIES" -r requirements.txt +pip install -q --retries "$MAX_RETRIES" . +popd || exit 1 + +# ── Clone Regulus repo ────────────────────────────────────────────── +echo "Cloning Regulus from ${REGULUS_REPO} (branch: ${REGULUS_BRANCH})..." +for attempt in $(seq 1 "$MAX_RETRIES"); do + rm -rf regulus + if git clone -q --branch "${REGULUS_BRANCH}" "${REGULUS_REPO}" --depth 1 regulus; then + echo "Successfully cloned Regulus" + break + fi + if [[ "$attempt" -eq "$MAX_RETRIES" ]]; then + echo "ERROR: git clone Regulus failed after $MAX_RETRIES attempts" >&2 + exit 1 + fi + echo "git clone failed (attempt $attempt/$MAX_RETRIES), retrying in 10s..." >&2 + sleep 10 +done + +# ── Install ORION dependencies ──────────────────────────────────── +pushd regulus/ORION || exit 1 +# Orion CLI already installed above; this covers ORION/ script deps (requests, pyyaml) +pip install -q --retries "$MAX_RETRIES" -r requirements.txt +echo "✅ Dependencies installed" + +# ── Run prow-entry.sh ───────────────────────────────────────────────────────── +echo "" +echo "==================================" +echo "Running Orion Regulus Analysis" +echo "==================================" +echo "" + +if ./scripts/prow-entry.sh; then + analysis_exit_status=0 +else + analysis_exit_status=$? +fi + +popd || true +popd || true + +exit $analysis_exit_status diff --git a/ci-operator/step-registry/openshift-qe/orion/regulus/openshift-qe-orion-regulus-ref.metadata.json b/ci-operator/step-registry/openshift-qe/orion/regulus/openshift-qe-orion-regulus-ref.metadata.json new file mode 100644 index 0000000000000..1657633cd60d7 --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/orion/regulus/openshift-qe-orion-regulus-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "openshift-qe/orion/regulus/openshift-qe-orion-regulus-ref.yaml", + "owners": { + "approvers": [ + "perfscale-ocp-approvers" + ], + "reviewers": [ + "perfscale-ocp-reviewers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift-qe/orion/regulus/openshift-qe-orion-regulus-ref.yaml b/ci-operator/step-registry/openshift-qe/orion/regulus/openshift-qe-orion-regulus-ref.yaml new file mode 100644 index 0000000000000..0dd238ff8771c --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/orion/regulus/openshift-qe-orion-regulus-ref.yaml @@ -0,0 +1,51 @@ +ref: + as: openshift-qe-orion-regulus + from_image: + namespace: ci + name: ocp-qe-perfscale-ci + tag: latest + cli: latest + env: + - name: REGULUS_REPO + default: "https://github.com/redhat-performance/regulus.git" + documentation: Regulus repo to clone (uses ORION/ subdirectory) + - name: REGULUS_BRANCH + default: "cpt" + documentation: Branch/tag of Regulus repo to use + - name: ORION_REPO + default: "https://github.com/cloud-bulldozer/orion.git" + documentation: Orion repo to clone + - name: ORION_TAG + default: "latest" + documentation: Branch/tag of orion + - name: BATCH_ID + default: "" + documentation: Specific batch ID to analyze. If empty, auto-discovers latest batch + - name: MATCH + default: "" + documentation: Filter to specific tests within batch (e.g., "threads=128") + - name: IGNORE + default: "rcos kernel" + documentation: Fields to exclude from fingerprint for cross-version analysis (e.g., "rcos kernel") + - name: ES_BENCHMARK_INDEX + default: "regulus-results-*" + documentation: Elasticsearch index pattern containing Regulus benchmark results + - name: LOOKBACK + default: "15" + documentation: Lookback for historical data. Plain integer for sample count (e.g., 15), or duration string (e.g., 90d, 30d2h) + - name: DEBUG + default: "false" + documentation: Enable debug output in analyze-batch.py + commands: openshift-qe-orion-regulus-commands.sh + timeout: 6h + credentials: + - namespace: test-credentials + name: ocp-perfscale-prod-es-creds + mount_path: /secret/perfscale-prod + resources: + requests: + cpu: 100m + memory: 100Mi + documentation: >- + This step runs Orion change point detection on regulus benchmark results +