diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 69402439d08a..9ae0a8392f94 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -57,7 +57,7 @@ jobs: TEST_ALL_PACKAGES: ${{ steps.check-label.outputs.is_full_run }} run: | if [ -n "$TARGET_BRANCH" ]; then - git fetch origin "$TARGET_BRANCH" --depth=1 || true + git fetch origin "$TARGET_BRANCH" --deepen=200 || true fi python3 ci/get_package_shards.py diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 971a9ad84003..d6a3d462e677 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -27,6 +27,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} + packages: ${{ steps.set-matrix.outputs.packages }} is_full_run: ${{ steps.check-label.outputs.is_full_run }} env: MAX_SHARDS: 16 @@ -140,7 +141,7 @@ jobs: echo "All unit tests passed or were skipped" cover: - if: always() && !cancelled() && needs.all-tests.result == 'success' && needs.unit.result != 'skipped' + if: always() && !cancelled() && needs.all-tests.result == 'success' runs-on: ubuntu-latest needs: - all-tests @@ -158,42 +159,24 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" - - name: Determine if coverage evaluation is required - id: packages - env: - TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }} - TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }} - run: | - if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then - echo "should_evaluate_coverage=true" >> "$GITHUB_OUTPUT" - else - TARGET_BRANCH="${TARGET_BRANCH:-main}" - git fetch origin "$TARGET_BRANCH" --depth=1 || true - num_files_changed=$(git diff --name-only "origin/${TARGET_BRANCH}" -- ${PACKAGE_DIRS} | wc -l | tr -d ' ') - if [[ "${num_files_changed}" -gt 0 ]]; then - echo "should_evaluate_coverage=true" >> "$GITHUB_OUTPUT" - else - echo "should_evaluate_coverage=false" >> "$GITHUB_OUTPUT" - fi - fi - name: Install coverage - if: ${{ steps.packages.outputs.should_evaluate_coverage == 'true' }} + if: ${{ needs.initialize.outputs.packages != '' }} run: | python -m pip install --upgrade setuptools pip wheel python -m pip install coverage - name: Download coverage results - if: ${{ steps.packages.outputs.should_evaluate_coverage == 'true' }} + if: ${{ needs.initialize.outputs.packages != '' }} uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 with: path: /dev/shm/.coverage-results/ merge-multiple: true - name: Report coverage results - if: ${{ steps.packages.outputs.should_evaluate_coverage == 'true' }} env: # TODO: default to 100% coverage after next gapic-generator release # https://github.com/googleapis/google-cloud-python/issues/17459 DEFAULT_FAIL_UNDER: 99 TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }} + PACKAGE_LIST: ${{ needs.initialize.outputs.packages }} TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }} BUILD_TYPE: presubmit run: | diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py index 77cc191cc3ac..015adfc2bacd 100644 --- a/ci/get_package_shards.py +++ b/ci/get_package_shards.py @@ -108,7 +108,7 @@ def get_packages_to_test(): return all_packages if build_type == 'presubmit': - git_diff_arg = f"origin/{target_branch}" + git_diff_arg = f"origin/{target_branch}..." elif build_type == 'continuous': git_diff_arg = "HEAD~1.." else: @@ -227,7 +227,13 @@ def group_packages(packages_map): shards_json = json.dumps(shards) print(shards_json) + all_paths = [] + for paths in packages.values(): + all_paths.extend(paths) + packages_str = " ".join(all_paths) + github_output = os.environ.get("GITHUB_OUTPUT") if github_output: with open(github_output, "a") as f: f.write(f"matrix={shards_json}\n") + f.write(f"packages={packages_str}\n") diff --git a/ci/report_coverage.sh b/ci/report_coverage.sh index f204bc45cdcc..478e082ef5cd 100755 --- a/ci/report_coverage.sh +++ b/ci/report_coverage.sh @@ -26,31 +26,39 @@ MAX_JOBS=$(nproc) mkdir -p "${LOG_DIR}" -if [ ! -d "${RESULTS_DIR}" ]; then - echo "Error: No coverage results found in ${RESULTS_DIR}." - exit 1 -fi - -# Unzip any zipped coverage results -find "$RESULTS_DIR" -type f -name '*.zip' -print0 | xargs -0 -P "${MAX_JOBS}" -I {} unzip -q -o {} -d "$RESULTS_DIR" - -# Identify modified packages BUILD_TYPE="${BUILD_TYPE:-presubmit}" TARGET_BRANCH="${TARGET_BRANCH:-main}" - PACKAGE_DIRS="${PACKAGE_DIRS:-packages preview-packages}" -if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then +if [ "${PACKAGE_LIST+set}" = "set" ]; then + # If pre-determined package list is set, use it + modified_packages="${PACKAGE_LIST}" +elif [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then # Test all packages mode: evaluate coverage for every package in the repository modified_packages=$(for dir in ${PACKAGE_DIRS}; do ls -d ${dir}/*/ 2>/dev/null; done | cut -d/ -f1,2 | sort -u) elif [[ "${BUILD_TYPE}" == "presubmit" ]]; then # Presubmit build: evaluate coverage only for packages modified relative to the target branch - modified_packages=$(git diff --name-only "origin/${TARGET_BRANCH}" -- ${PACKAGE_DIRS} 2>/dev/null | cut -d/ -f1,2 | sort -u) + modified_packages=$(git diff --name-only "origin/${TARGET_BRANCH}..." -- ${PACKAGE_DIRS} 2>/dev/null | cut -d/ -f1,2 | sort -u) else # Continuous build (post-merge on main): evaluate coverage for packages modified in the last commit modified_packages=$(git diff --name-only HEAD~1 -- ${PACKAGE_DIRS} 2>/dev/null | cut -d/ -f1,2 | sort -u) fi +if [ -z "${modified_packages}" ]; then + echo "============================================================" + echo "No modified packages to evaluate coverage for." + echo "============================================================" + exit 0 +fi + +if [ ! -d "${RESULTS_DIR}" ]; then + echo "Error: No coverage results found in ${RESULTS_DIR}." + exit 1 +fi + +# Unzip any zipped coverage results +find "$RESULTS_DIR" -type f -name '*.zip' -print0 | xargs -0 -P "${MAX_JOBS}" -I {} unzip -q -o {} -d "$RESULTS_DIR" + # Function to report coverage for a single package report_package_coverage() { local pkg=$1 diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh index 241de51a4f88..1f8affbc710e 100755 --- a/ci/run_conditional_tests.sh +++ b/ci/run_conditional_tests.sh @@ -54,9 +54,9 @@ elif [[ ${BUILD_TYPE} == "presubmit" ]]; then # For presubmit build, we want to know the difference from the # common commit in the target branch. if [ -n "${TARGET_BRANCH}" ]; then - git fetch origin "${TARGET_BRANCH}" --depth=1 || true + git fetch origin "${TARGET_BRANCH}" --depth=200 || true fi - GIT_DIFF_ARG="origin/${TARGET_BRANCH}" + GIT_DIFF_ARG="origin/${TARGET_BRANCH}..." elif [[ ${BUILD_TYPE} == "continuous" ]]; then # For continuous build, we want to know the difference in the last