From bd99722f0e5063b598bacda13a28fbf9f3e17f19 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Tue, 4 Aug 2026 10:47:00 -0400 Subject: [PATCH 1/3] feat: add ad-hoc testing integration and configs --- .kokoro/system.sh | 65 +++++++++++++++++++++++++-- ci/adhoc/.package_groups.txt | 4 ++ ci/adhoc/.standalone_package_list.txt | 3 ++ ci/adhoc/README.md | 47 +++++++++++++++++++ ci/adhoc/adhoc_test_runner.sh | 51 +++++++++++++++++++++ 5 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 ci/adhoc/.package_groups.txt create mode 100644 ci/adhoc/.standalone_package_list.txt create mode 100644 ci/adhoc/README.md create mode 100755 ci/adhoc/adhoc_test_runner.sh diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 7635c0be17ea..1baaa22c2f4b 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -161,6 +161,8 @@ reap_parallel_results() { fi done + + if [ "$failed_count" -gt 0 ]; then echo "==================================================" echo "@FAILED - DETAILED LOGS FOR FAILED PACKAGES" @@ -183,6 +185,7 @@ reap_parallel_results() { cat "$LOG_DIR/$pkg.log" else echo "Warning: No log file found for failed package $pkg" + fi echo "" fi @@ -276,6 +279,60 @@ for path in `find 'packages' \ fi done +# --- Ad-hoc Testing Integration --- +TRIGGER_ADHOC="false" +if [[ -n "${KOKORO_GITHUB_PULL_REQUEST_NUMBER}" ]]; then + echo "Checking for adhoc test label on PR #${KOKORO_GITHUB_PULL_REQUEST_NUMBER}..." + headers=(-H "User-Agent: Kokoro") + if [[ -n "${GITHUB_TOKEN:-${GH_TOKEN}}" ]]; then + headers+=(-H "Authorization: token ${GITHUB_TOKEN:-${GH_TOKEN}}") + fi + # Hardened curl call with || true to prevent script termination if network fails + LABELS_JSON=$(curl -s "${headers[@]}" "https://api.github.com/repos/googleapis/google-cloud-python/issues/${KOKORO_GITHUB_PULL_REQUEST_NUMBER}/labels" || echo "[]") + + # Use jq to parse github labels (works as long as jq is available in python-multi image). + IS_ADHOC=$(echo "$LABELS_JSON" | jq -r 'if type == "array" then any(.name == "test:adhoc") else false end' 2>/dev/null) + + + if [[ "$IS_ADHOC" == "true" ]]; then + TRIGGER_ADHOC="true" + echo "Adhoc test label 'test:adhoc' found!" + else + if [[ "$LABELS_JSON" != "["* ]]; then + API_ERR_MSG=$(echo "$LABELS_JSON" | jq -r '.message // "Unknown error"' 2>/dev/null) + echo "================================================================" + echo "WARNING: Failed to fetch PR labels from GitHub API!" + echo "Error Message: $API_ERR_MSG" + echo "This might be due to API Rate Limiting." + echo "Ad-hoc tests will NOT be triggered." + echo "================================================================" + else + echo "Adhoc test label 'test:adhoc' not found." + fi + fi + +fi + +if [[ "$TRIGGER_ADHOC" == "true" ]]; then + echo "Running ad-hoc package selection..." + source ci/adhoc/adhoc_test_runner.sh + + echo "Deduplicating packages..." + # Deduplication using Associative Arrays (Requires Bash 4+) + declare -A unique_packages + for pkg in "${PACKAGES_TO_TEST[@]}"; do + [[ -n "$pkg" ]] && unique_packages["$pkg"]=1 + done + for pkg in $ADHOC_PACKAGES; do + [[ -n "$pkg" ]] && unique_packages["$pkg"]=1 + done + + PACKAGES_TO_TEST=("${!unique_packages[@]}") + + echo "Combined packages to test: ${PACKAGES_TO_TEST[*]}" +fi +# --- End Ad-hoc Testing Integration --- + # Parallel Execution Logic MAX_JOBS=${MAX_JOBS:-4} @@ -301,18 +358,20 @@ export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR # Stream package names to xargs for parallel execution # -P "$MAX_JOBS" controls concurrency # -I {} replaces {} with the package name -printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ - | xargs -n 1 -P "$MAX_JOBS" \ +printf '%s\0' "${PACKAGES_TO_TEST[@]}" \ + | xargs -0 -n 1 -P "$MAX_JOBS" \ bash -c ' pkg="$0" + # Determine log location: prefer Sponge artifacts directory if available if [ -n "$KOKORO_ARTIFACTS_DIR" ]; then pkg_log_dir="$KOKORO_ARTIFACTS_DIR/$pkg" - mkdir -p "$pkg_log_dir" || { touch "$LOG_DIR/$pkg.failed"; exit 1; } + mkdir -p "$pkg_log_dir" || { echo "Failed to mkdir $pkg_log_dir"; touch "$LOG_DIR/$pkg.failed"; exit 1; } log_file="$pkg_log_dir/sponge_log.log" else log_file="$LOG_DIR/$pkg.log" fi + echo "Log file for $pkg: $log_file" # Run test; if it fails, create a .failed file to signal failure to the reaper run_package_test "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" diff --git a/ci/adhoc/.package_groups.txt b/ci/adhoc/.package_groups.txt new file mode 100644 index 000000000000..fe24e6ba4c96 --- /dev/null +++ b/ci/adhoc/.package_groups.txt @@ -0,0 +1,4 @@ +handwritten: google-cloud-translate +handwritten: google-cloud-logging +core: google-api-core +core: google-cloud-core diff --git a/ci/adhoc/.standalone_package_list.txt b/ci/adhoc/.standalone_package_list.txt new file mode 100644 index 000000000000..31760352449c --- /dev/null +++ b/ci/adhoc/.standalone_package_list.txt @@ -0,0 +1,3 @@ +package: google-cloud-logging +package: google-cloud-dns +group: handwritten diff --git a/ci/adhoc/README.md b/ci/adhoc/README.md new file mode 100644 index 000000000000..10a2aefbfc02 --- /dev/null +++ b/ci/adhoc/README.md @@ -0,0 +1,47 @@ +# Ad-Hoc Package Testing + +## Overview +Ad-hoc package testing allows you to run CI tests for a specific subset of packages or predefined package groups without the need for intrusive and/or temporary mods to the package code to trigger a CI job run. This is useful for verifying: +* local changes or remote changes (i.e. on GitHub) +* debugging specific package failures +* testing CI infrastructure updates without wasting resources. + +## How It Works +The ad-hoc testing system reads configuration files in the `/ci/adhoc` directory to determine which packages to test. It can be triggered via CI labels (e.g., `test:adhoc`). + +## Configuration Files + +### 1. `.standalone_package_list.txt` +This file lists the specific packages or groups you want to test. + +* **To test an individual package:** Add a line starting with `package: ` (be sure to include the colon and space) followed by the package directory name. + * *Example:* `package: google-cloud-dns` +* **To test a group of packages:** Add a line starting with `group: ` (be sure to include the colon and space) followed by the group name. NOTE: groups are defined in the file: `.package_groups.txt` + * *Example:* `group: handwritten` + +### 2. `.package_groups.txt` +This file defines groups of commonly tested packages for convenience of the team. Groups such as all handwritten, all core, all hybrids, etc. can be defined here. + +#### 💡 Pro Tip +You can mix packages and groups in `.standalone_package_list.txt`. The system will automatically deduplicate the list so each package is tested only once! + +* **Format:** Each package in a group should be on its own line, prefixed by the group name, colon, and a space. + * *Example:* + ```text + handwritten: google-cloud-translate + handwritten: google-cloud-logging + core: google-api-core + ``` + +## Usage + +1. **Edit Configuration:** Open `.standalone_package_list.txt` and add the packages or groups you want to test. +2. **Trigger Tests:** + * Issue a Pull Request (PR) with the updated `.standalone_package_list.txt` file. + * **In GitHub UI:** Add the `test:adhoc` label to your PR. + +## ⚠️ Best Practices & Warnings + +* **Do Not Pollute Packages:** Do not modify package code just to trigger tests. Use the ad-hoc configuration files instead. +* **Revert Temporary Changes:** If you add temporary print statements or dummy tests for debugging, ensure they are reverted before merging. +* **Clean Up:** Keep the configuration files clean and remove unused entries. diff --git a/ci/adhoc/adhoc_test_runner.sh b/ci/adhoc/adhoc_test_runner.sh new file mode 100755 index 000000000000..a0a3d3b15e0f --- /dev/null +++ b/ci/adhoc/adhoc_test_runner.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to determine ad-hoc packages to test. +# This script is intended to be sourced from main test scripts. +# +# Precondition: This script assumes it is sourced from the project root (as set by system.sh). + + +ADHOC_DIR="ci/adhoc" +STANDALONE_LIST="${ADHOC_DIR}/.standalone_package_list.txt" +GROUPS_FILE="${ADHOC_DIR}/.package_groups.txt" + +if [[ ! -f "$STANDALONE_LIST" ]]; then + echo "Warning: $STANDALONE_LIST not found." + return 0 2>/dev/null || exit 0 +fi + +if [[ ! -f "$GROUPS_FILE" ]]; then + echo "Warning: $GROUPS_FILE not found." + return 0 2>/dev/null || exit 0 +fi + +# Grab individual packages +adhoc_packages=$(grep "^package:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs || true) + +# Grab requested groups +requested_groups=$(grep "^group:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs || true) + +# Expand groups +for group in $requested_groups; do + group_pkgs=$(grep "^$group:" "$GROUPS_FILE" | cut -d':' -f2 | xargs || true) + adhoc_packages="$adhoc_packages $group_pkgs" +done + +# Convert to unique list (deduplicate our adhoc packages) +ADHOC_PACKAGES=$(echo "$adhoc_packages" | tr ' ' '\n' | sort -u | xargs) + +export ADHOC_PACKAGES From 2a6ec65af25710ebc8f68eefe6f36dcbedd5418c Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 4 Aug 2026 11:23:10 -0400 Subject: [PATCH 2/3] Removes blank lines --- .kokoro/system.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 1baaa22c2f4b..40d9b738683e 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -161,8 +161,6 @@ reap_parallel_results() { fi done - - if [ "$failed_count" -gt 0 ]; then echo "==================================================" echo "@FAILED - DETAILED LOGS FOR FAILED PACKAGES" From 1548f290d8828d7549d490d256b66b1356df1b59 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Tue, 4 Aug 2026 11:24:28 -0400 Subject: [PATCH 3/3] chore: trigger ad-hoc tests