From dc7a74e1b22f859bb540cd0b2778c4fa4f55b64f Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Fri, 17 Apr 2026 17:52:32 -0500 Subject: [PATCH 1/9] Add support for pushing osg-htc project images --- .github/workflows/build-containers.yml | 17 +++++++++-------- scripts/build-job-matrix.py | 4 +--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 7809dfe2..a12263cc 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -30,7 +30,7 @@ jobs: - id: image-list run: | - ORG_DIR=opensciencegrid + ORG_DIR_REGEX="^(opensciencegrid|osg-htc)/" # Get the list of files changed based on the type of event # kicking off the GHA: # 1. For the main branch, diff the previous state of main vs @@ -54,7 +54,7 @@ jobs: images=$(git diff --name-only \ "$BASE" \ "$GITHUB_SHA" | - egrep "^$ORG_DIR/" | + grep -E "$ORG_DIR_REGEX" | cut -d/ -f -2 | sort | uniq | @@ -64,10 +64,11 @@ jobs: else # List all image root dirs. Example value: # "opensciencegrid/vo-frontend opensciencegrid/ospool-cm" - images=$(find $ORG_DIR -mindepth 1 \ - -maxdepth 1 \ - -type d \ - -printf "$ORG_DIR/%P\n") + images=$(find . -mindepth 2 \ + -maxdepth 2 \ + -type d \ + -printf "%P\n" | + grep -E "${ORG_DIR_REGEX}") fi image_json=$(echo -n "${images:-dummy}" | jq -Rcs '.|split("\n") | map(select(. != ""))') @@ -124,7 +125,7 @@ jobs: BASE_OS=$(echo $CONFIG | awk -F'-' '{print $1}') OSG_SERIES=$(echo $CONFIG | awk -F'-' '{print $2}') BASE_REPO=$(echo $CONFIG | awk -F'-' '{print $3}') - CONTEXT="opensciencegrid/${{ matrix.name }}" + CONTEXT="${{ matrix.name }}" echo "BASE_OS=${BASE_OS}" >> $GITHUB_ENV echo "OSG_SERIES=${OSG_SERIES}" >> $GITHUB_ENV echo "BASE_REPO=${BASE_REPO}" >> $GITHUB_ENV @@ -176,7 +177,7 @@ jobs: BASE_OS=$(echo $CONFIG | awk -F'-' '{print $1}') OSG_SERIES=$(echo $CONFIG | awk -F'-' '{print $2}') BASE_REPO=$(echo $CONFIG | awk -F'-' '{print $3}') - CONTEXT="opensciencegrid/${{ matrix.name }}" + CONTEXT="${{ matrix.name }}" echo "BASE_OS=${BASE_OS}" >> $GITHUB_ENV echo "OSG_SERIES=${OSG_SERIES}" >> $GITHUB_ENV echo "BASE_REPO=${BASE_REPO}" >> $GITHUB_ENV diff --git a/scripts/build-job-matrix.py b/scripts/build-job-matrix.py index a8b86388..3b544892 100644 --- a/scripts/build-job-matrix.py +++ b/scripts/build-job-matrix.py @@ -35,8 +35,6 @@ def main(image_dirs): config = load_config(build_config_path, default_config) - image_name = os.path.basename(image_dir) - base_os_list = config['base_os'] osg_series_list = config['osg_series'] base_repo_list = config['base_repo'] @@ -56,7 +54,7 @@ def main(image_dirs): # 1. Simplicity: Using a single string to represent configurations is straightforward and easy to understand. # 2. Integration: A single string is easily passed to external tools and systems that manage builds. configuration_string = f"{base_os}-{osg_series}-{base_repo}-{config['standard_build']}-{config['repo_build']}" - include_list.append({"name": image_name, "config": configuration_string}) + include_list.append({"name": image_dir, "config": configuration_string}) sys.stdout.flush() json_output = json.dumps({"include": include_list}, indent=4) From b6bd3073707d8f58bf54f338d83d2852628ca2dc Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Tue, 28 Jul 2026 11:51:39 -0500 Subject: [PATCH 2/9] Get rid of odd 'tail' from image matrix construction pipe line --- .github/workflows/build-containers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index a12263cc..dab8b75f 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -88,7 +88,7 @@ jobs: id: set-matrix run: | # Run Python script and capture JSON output - matrix_json=$(python scripts/build-job-matrix.py ${{ steps.image-list.outputs.images }} | tail -n +2) + matrix_json=$(python scripts/build-job-matrix.py ${{ steps.image-list.outputs.images }}) # Use jq to extract the 'include' part of the JSON matrix=$(echo "$matrix_json" | jq -c '.include') echo "::set-output name=matrix::$matrix" From 40d3826b037d6c34615d3c5411dab50e1f3720c9 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Tue, 28 Jul 2026 11:52:17 -0500 Subject: [PATCH 3/9] Remove extraneous print statement We seemingly drop this on the floor, otherwise jq chokes on this input --- scripts/build-job-matrix.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/build-job-matrix.py b/scripts/build-job-matrix.py index 3b544892..e255c14e 100644 --- a/scripts/build-job-matrix.py +++ b/scripts/build-job-matrix.py @@ -22,8 +22,6 @@ def load_config(config_path, default_config=None): def main(image_dirs): - print("Image directories:", image_dirs) - default_config = load_config(DEFAULT_CONFIG_PATH) include_list = [] From 6725f5e72e4902a354b18f154f86500aaa18f4f9 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Tue, 28 Jul 2026 11:56:12 -0500 Subject: [PATCH 4/9] FIXME: troubleshoot GHA --- .github/workflows/build-containers.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index dab8b75f..8ec8b4e3 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -73,6 +73,7 @@ jobs: image_json=$(echo -n "${images:-dummy}" | jq -Rcs '.|split("\n") | map(select(. != ""))') echo "$image_json" > image_list.json + echo "$images" echo "images=$(echo $images | tr '\n' ' ')" >> $GITHUB_OUTPUT echo "image_list=$image_json" >> $GITHUB_OUTPUT From f6985f0b66f64fcd9d81eac47b2395b707d4d961 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Tue, 28 Jul 2026 12:04:52 -0500 Subject: [PATCH 5/9] Add lost comment explaining the 'dummy' string --- .github/workflows/build-containers.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 8ec8b4e3..05c62c2a 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -71,6 +71,9 @@ jobs: grep -E "${ORG_DIR_REGEX}") fi + # Ensure that the generated JSON array has a member, + # otherwise GHA will throw an error about an empty matrix + # vector in subsequent steps image_json=$(echo -n "${images:-dummy}" | jq -Rcs '.|split("\n") | map(select(. != ""))') echo "$image_json" > image_list.json echo "$images" From 3c6b0be30b5aa015ea237b4c8424db31c57c6c3d Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Tue, 28 Jul 2026 12:05:28 -0500 Subject: [PATCH 6/9] Fix indentation --- .github/workflows/build-containers.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 05c62c2a..29fba4e2 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -71,14 +71,14 @@ jobs: grep -E "${ORG_DIR_REGEX}") fi - # Ensure that the generated JSON array has a member, - # otherwise GHA will throw an error about an empty matrix - # vector in subsequent steps - image_json=$(echo -n "${images:-dummy}" | jq -Rcs '.|split("\n") | map(select(. != ""))') - echo "$image_json" > image_list.json - echo "$images" - echo "images=$(echo $images | tr '\n' ' ')" >> $GITHUB_OUTPUT - echo "image_list=$image_json" >> $GITHUB_OUTPUT + # Ensure that the generated JSON array has a member, + # otherwise GHA will throw an error about an empty matrix + # vector in subsequent steps + image_json=$(echo -n "${images:-dummy}" | jq -Rcs '.|split("\n") | map(select(. != ""))') + echo "$image_json" > image_list.json + echo "$images" + echo "images=$(echo $images | tr '\n' ' ')" >> $GITHUB_OUTPUT + echo "image_list=$image_json" >> $GITHUB_OUTPUT - name: Display image list run: cat image_list.json From dd3acad8b429b1b362f3a7b96fc5b23cc7145fa7 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Tue, 28 Jul 2026 12:11:35 -0500 Subject: [PATCH 7/9] Gracefully handle commits with no image changes --- .github/workflows/build-containers.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 29fba4e2..8363f496 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -91,10 +91,15 @@ jobs: - name: Set matrix output id: set-matrix run: | - # Run Python script and capture JSON output - matrix_json=$(python scripts/build-job-matrix.py ${{ steps.image-list.outputs.images }}) - # Use jq to extract the 'include' part of the JSON - matrix=$(echo "$matrix_json" | jq -c '.include') + # Handle empty images var, i.e. no images were changed + # N.B. an empty GHA output var results in a space + if [[ "${{ steps.image-list.outputs.images }}" != " " ]]; then + matrix_json=$(python scripts/build-job-matrix.py ${{ steps.image-list.outputs.images }}) + # Use jq to extract the 'include' part of the JSON + matrix=$(echo "$matrix_json" | jq -c '.include') + else + matrix="[]" + fi echo "::set-output name=matrix::$matrix" From d1465360ddd051a04b6771e7e0224258aab01504 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Tue, 28 Jul 2026 12:53:09 -0500 Subject: [PATCH 8/9] Remove deprecated use of `::set-output` --- .github/workflows/build-containers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 8363f496..0b8c080a 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -100,7 +100,7 @@ jobs: else matrix="[]" fi - echo "::set-output name=matrix::$matrix" + echo "matrix=$matrix" >> $GITHUB_OUTPUT - name: Verify matrix content From efb1023d29eb427cf2420d9002df1eee80005054 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Wed, 29 Jul 2026 14:18:23 -0500 Subject: [PATCH 9/9] Improve check against empty image lists Co-authored-by: Matyas Selmeci --- .github/workflows/build-containers.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 0b8c080a..7e08a019 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -92,8 +92,9 @@ jobs: id: set-matrix run: | # Handle empty images var, i.e. no images were changed - # N.B. an empty GHA output var results in a space - if [[ "${{ steps.image-list.outputs.images }}" != " " ]]; then + # N.B. an empty GHA output var results in a space, so look for non-space characters instead of just length + IMAGES="${{ steps.image-list.outputs.images }}" + if [[ $IMAGES =~ [^[:space:]] ]]; then matrix_json=$(python scripts/build-job-matrix.py ${{ steps.image-list.outputs.images }}) # Use jq to extract the 'include' part of the JSON matrix=$(echo "$matrix_json" | jq -c '.include')