From 33c7f037183a6e1c5c1f647d756915efdb9a896a Mon Sep 17 00:00:00 2001 From: Spartan322 Date: Wed, 20 May 2026 07:15:14 -0400 Subject: [PATCH] Add nightly releases Pin workflow actions to commits Add a reusable build workflow Add a reusable release workflow --- .github/changed-files.yml | 19 +++ .github/workflows/builds.yml | 207 ++++++++++--------------- .github/workflows/nightly-releases.yml | 88 +++++++++++ .github/workflows/releases.yml | 17 ++ .github/workflows/reusable-build.yml | 189 ++++++++++++++++++++++ .github/workflows/reusable-release.yml | 149 ++++++++++++++++++ 6 files changed, 546 insertions(+), 123 deletions(-) create mode 100644 .github/changed-files.yml create mode 100644 .github/workflows/nightly-releases.yml create mode 100644 .github/workflows/releases.yml create mode 100644 .github/workflows/reusable-build.yml create mode 100644 .github/workflows/reusable-release.yml diff --git a/.github/changed-files.yml b/.github/changed-files.yml new file mode 100644 index 000000000..a532b7f13 --- /dev/null +++ b/.github/changed-files.yml @@ -0,0 +1,19 @@ +# We lack a convenient means of gathering *all* the changes when specializations are passed, so +# a catch-all variable is the easiest workaround. +everything: + - "**" + +# Determines if build actions should occur after static checks are ran. Broadly speaking, these +# files changing would result in SCons rebuilding the engine, or are otherwise pertinent to the +# buildsystem itself. +sources: + - .github/{actions/*,workflows}/*.yml + - "**/{SConstruct,SCsub,*.py}" + - "**/*.{hpp,cpp,inc}" + - scripts + - deps/** + - tests/** + +# Determines which files are appropriate for running clangd-tidy checks on. +clangd: + - "**/*.{hpp,cpp,inc}" \ No newline at end of file diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index d74e650d4..90040aa94 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -1,54 +1,91 @@ name: 🖥️ Builds -on: [push, pull_request, merge_group] - -env: - GH_BASE_BRANCH: master +on: + push: + branches: ['master'] + paths: + - ".github/workflows/{builds, master-builds}.yml" + - "**/{SConstruct,SCsub,*.py}" + - "**/*.{hpp,cpp,inc}" + - "scripts" + - "deps/**" + - "tests/**" + - "AUTHORS.md" + - "COPYRIGHT" + - "LICENSE.md" + - ".pre-commit-config.yaml" + - "pyproject.toml" + - ".clang-format" + - ".clang-tidy" + - ".gitmodules" + pull_request: + branches: ['**'] + paths: + - ".github/workflows/{builds, master-builds}.yml" + - "**/{SConstruct,SCsub,*.py}" + - "**/*.{hpp,cpp,inc}" + - "scripts" + - "deps/**" + - "tests/**" + - "AUTHORS.md" + - "COPYRIGHT" + - "LICENSE.md" + - ".pre-commit-config.yaml" + - "pyproject.toml" + - ".clang-format" + - ".clang-tidy" + - ".gitmodules" + merge_group: + workflow_dispatch: concurrency: - group: ${{ github.workflow }}|${{ github.ref_name }} - cancel-in-progress: true + group: ${{ github.workflow }}|${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: {} jobs: - static-checks: + static-checks-builds: name: Code style, file formatting, and docs - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest + if: github.run_attempt > 1 || github.event_name == 'workflow_dispatch' || !vars.DISABLE_SIM_BUILDS + outputs: + changed-files: '"${{ steps.changed-files.outputs.clangd_all_changed_files }}"' # Wrap with quotes to bookend internal quote separators. + sources-changed: ${{ steps.changed-files.outputs.sources_any_changed }} steps: - name: Checkout project - uses: actions/checkout@v4.1.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - fetch-depth: 2 + fetch-depth: 0 # Treeless clone. Slightly less performant than a shallow clone, but makes finding diffs instantaneous. + filter: tree:0 # See: https://github.blog/open-source/git/get-up-to-speed-with-partial-clone-and-shallow-clone/ - name: Install APT dependencies - uses: awalsh128/cache-apt-pkgs-action@latest + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 with: packages: libxml2-utils - name: Get changed files - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - git config diff.wsErrorHighlight all - if [ "${{ github.event_name }}" == "pull_request" -o "${{ github.event.forced }}" == "true" -o "${{ github.event.created }}" == "true" ]; then - files=$(git diff-tree --no-commit-id --name-only -r HEAD^1..HEAD 2> /dev/null || true) - elif [ "${{ github.event_name }}" == "push" -a "${{ github.event.created }}" == "false" ]; then - files=$(git diff-tree --no-commit-id --name-only -r ${{ github.event.before }}..${{ github.event.after }} 2> /dev/null || true) - fi - echo "$files" >> changed.txt - cat changed.txt - files=$(echo "$files" | xargs -I {} sh -c 'echo "\"./{}\""' | tr '\n' ' ') - echo "CHANGED_FILES=$files" >> $GITHUB_ENV - - - name: Style checks via pre-commit - uses: pre-commit/action@v3.0.1 + id: changed-files + uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 with: - extra_args: --files ${{ env.CHANGED_FILES }} + safe_output: false # Output passed to environment variable to avoid command injection. + separator: '" "' # To account for paths with spaces, ensure our items are split by quotes internally. + skip_initial_fetch: true + files_yaml_from_source_file: .github/changed-files.yml - build: - runs-on: ${{matrix.os}} - name: ${{matrix.name}} - needs: static-checks - permissions: write-all + - name: Style checks via prek + uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 + env: + CHANGED_FILES: '"${{ steps.changed-files.outputs.everything_all_changed_files }}"' # Wrap with quotes to bookend internal quote separators + with: + extra-args: --files ${{ env.CHANGED_FILES }} + + build-master: + name: ${{ matrix.name }} + needs: static-checks-builds + if: needs.static-checks-builds.outputs.sources-changed == 'true' || github.event_name != 'pull_request' + permissions: + contents: read strategy: fail-fast: false matrix: @@ -59,24 +96,28 @@ jobs: target: template_debug platform: windows arch: x86_64 + - identifier: windows-release os: windows-latest name: 🏁 Windows Release target: template_release platform: windows arch: x86_64 + - identifier: macos-debug os: macos-latest name: 🍎 macOS (universal) Debug target: template_debug platform: macos arch: universal + - identifier: macos-release os: macos-latest name: 🍎 macOS (universal) Release target: template_release platform: macos arch: universal + - identifier: linux-debug os: ubuntu-latest name: 🐧 Linux Debug @@ -84,6 +125,7 @@ jobs: target: template_debug platform: linux arch: x86_64 + - identifier: linux-release os: ubuntu-latest name: 🐧 Linux Release @@ -92,93 +134,12 @@ jobs: platform: linux arch: x86_64 - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Setup build cache - uses: OpenVicProject/openvic-cache@master - with: - cache-name: ${{ matrix.identifier }} - base-branch: ${{ env.GH_BASE_BRANCH }} - continue-on-error: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Set up SCons - shell: bash - run: | - python -c "import sys; print(sys.version)" - python -m pip install scons - scons --version - - - name: Linux dependencies - if: ${{ matrix.platform == 'linux' }} - run: | - sudo apt-get update -qq - sudo apt-get install -qqq build-essential pkg-config libtbb-dev - g++ --version - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 12 - sudo update-alternatives --set g++ /usr/bin/g++-12 - g++ --version - - - name: Compile with SCons - uses: OpenVicProject/openvic-build@master - with: - platform: ${{ matrix.platform }} - target: ${{ matrix.target }} - sconsflags: arch=${{ matrix.arch }} build_ovsim_library=yes run_ovsim_tests=yes build_ovsim_benchmarks=yes - - - name: Delete compilation files - if: ${{ matrix.platform == 'windows' }} - run: | - Remove-Item bin/* -Include *.exp,*.pdb -Force - - - name: Upload library artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ github.event.repository.name }}-${{ matrix.identifier }}-library - path: | - ${{ github.workspace }}/bin/libopenvic-simulation.* - - - name: Upload executable artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ github.event.repository.name }}-${{ matrix.identifier }}-executable - path: | - ${{ github.workspace }}/bin/openvic-simulation.headless.* - - - name: Archive Release - uses: thedoctor0/zip-release@0.7.6 - with: - type: "zip" - filename: "../../../libopenvic-simulation.${{ matrix.platform }}.${{ matrix.arch }}.zip" - directory: "${{ github.workspace }}/bin/" - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - - - name: Create and upload asset - uses: ncipollo/release-action@v1.13.0 - with: - allowUpdates: true - artifacts: "libopenvic-simulation.${{ matrix.platform }}.${{ matrix.arch }}.zip" - omitNameDuringUpdate: true - omitBodyDuringUpdate: true - token: ${{ secrets.GITHUB_TOKEN }} - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - - merge-library-files: - runs-on: ubuntu-latest - needs: build - name: 📚 Merge Library Files - steps: - - name: Merge Artifacts - uses: actions/upload-artifact/merge@v4 - with: - delete-merged: true - name: ${{ github.event.repository.name }}-library - pattern: ${{ github.event.repository.name }}-*-library + uses: ./.github/workflows/reusable-build.yml + with: + identifier: ${{ matrix.identifier }} + os: ${{ matrix.runner || matrix.os }} + name: ${{ matrix.name }} + target: ${{ matrix.target }} + platform: ${{ matrix.platform }} + arch: ${{ matrix.arch }} + merge-library-files: true \ No newline at end of file diff --git a/.github/workflows/nightly-releases.yml b/.github/workflows/nightly-releases.yml new file mode 100644 index 000000000..ee8837e29 --- /dev/null +++ b/.github/workflows/nightly-releases.yml @@ -0,0 +1,88 @@ +name: 🌘 Nightly Releases +on: + schedule: + - cron: '23 0 * * *' + workflow_dispatch: + +env: + TARGET_REPO: OpenVic-Simulation-nightly-builds + +permissions: {} + +jobs: + check-nightly: + name: Check for new commits + if: github.event_name == 'workflow_dispatch' || !vars.DISABLE_SIM_NIGHTLYS + permissions: + contents: read + runs-on: ubuntu-latest + outputs: + commit: ${{ steps.check_for_new_commits.outputs.commit }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Retrieve HEAD commit hash + id: head + shell: bash + run: echo "head=$(git rev-parse HEAD)" | tee -a "${GITHUB_OUTPUT}" + + - name: Cache nightly commit hash + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1 + with: + path: .nightly_commit_hash + key: release-nightly-${{ steps.head.outputs.head }} + restore-keys: | + release-nightly- + + - name: Check for new commits + id: check_for_new_commits + shell: bash + run: | + relevant_files=( + "deps/*" + "misc/*" + "scripts" + "src/*.hpp" + "src/*.cpp" + ':!src/openvic-simulation/pch.hpp' + ':!src/openvic-simulation/pch.cpp' + "pyproject.toml" + "SConstruct" + ".github/workflows/builds.yml" + ".github/workflows/nightly-builds.yml" + ) + if [[ -f .nightly_commit_hash ]]; then + limit_args=( + "$(cat .nightly_commit_hash)..HEAD" + ) + else + limit_args=( + --since="24 hours ago" + ) + fi + echo "commit=$(git log --format=%H -1 "${limit_args[@]}" -- "${relevant_files[@]}")" | tee -a "${GITHUB_OUTPUT}" + + - name: Record new nightly commit hash + env: + HEAD: ${{ steps.head.outputs.head }} + shell: bash + run: echo "${HEAD}" | tee .nightly_commit_hash + + nightly-release: + name: Publish GitHub release + needs: check-nightly + if: needs.check-nightly.outputs.commit + permissions: + contents: write # May be needed to publish release + id-token: write # Needed for trusted publishing + uses: ./.github/workflows/reusable-release.yml + with: + prerelease: true + target: ${{ github.env.TARGET_REPO }} + scons-flags: 'use_hot_reload=no build_ovsim_tests=no debug_symbols=yes' + diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml new file mode 100644 index 000000000..3b8062f91 --- /dev/null +++ b/.github/workflows/releases.yml @@ -0,0 +1,17 @@ +name: 🖥️ Releases + +on: + push: + tags: ["**"] + +permissions: {} + +jobs: + release: + name: Publish GitHub release + permissions: + contents: write # May be needed to publish release + id-token: write # Needed for trusted publishing + uses: ./.github/workflows/reusable-release.yml + with: + prerelease: ${{ contains('["alpha", "beta", "rc", "snapshot"]', github.ref) }} \ No newline at end of file diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml new file mode 100644 index 000000000..e1d3d6445 --- /dev/null +++ b/.github/workflows/reusable-build.yml @@ -0,0 +1,189 @@ +name: 🖥️ Reusable Build + +on: + workflow_call: + inputs: + identifier: + required: true + type: string + os: + required: true + type: string + name: + required: true + type: string + target: + required: true + type: string + platform: + required: true + type: string + arch: + required: true + type: string + scons-flags: + required: false + default: 'build_ovsim_library=yes build_ovsim_benchmarks=yes' + type: string + base-branch: + required: false + default: 'master' + type: string + merge-library-files: + required: false + default: false + type: boolean + secrets: + ARCHIVE_REPO_TOKEN: + required: false + GPG_SIGNING_KEY: + required: false + RELEASE_KEY: + required: false + workflow_dispatch: + inputs: + identifier: + description: | + Identifier of this build + required: true + type: string + os: + description: | + Platform to run build on + required: true + type: string + name: + description: | + Human-readable name of this build + required: true + type: string + target: + description: | + Target type to build for + required: true + type: choice + options: + - template_release + - template_debug + - editor + platform: + description: | + Platform to build for + required: true + type: choice + options: + - windows + - linux + - macos + arch: + description: | + Architecture to build for + required: true + type: choice + options: + - x86_64 + - universal + scons-flags: + description: | + Additional flags to send to SCons + required: false + default: 'build_ovsim_library=yes build_ovsim_benchmarks=yes' + type: string + cache-base-branch: + description: | + Branch to base the cache upon + required: false + default: 'master' + type: string + merge-library-files: + description: | + Whether to merge library files into an archive + required: false + default: false + type: boolean + +jobs: + build: + runs-on: ${{ inputs.os }} + name: ${{ inputs.name }} + permissions: + contents: read + artifact-metadata: write + steps: + - name: Checkout project + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: recursive + persist-credentials: false + + - name: Setup build cache + uses: OpenVicProject/openvic-cache@master + with: + cache-name: ${{ inputs.identifier }} + base-branch: ${{ inputs.cache-base-branch }} + continue-on-error: true + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.x" + + - name: Set up SCons + shell: bash + run: | + python -c "import sys; print(sys.version)" + python -m pip install scons + scons --version + + - name: Install APT dependencies + if: ${{ inputs.platform == 'linux' }} + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 + with: + packages: build-essential pkg-config libtbb-dev + + - name: Linux dependencies + if: ${{ inputs.platform == 'linux' }} + run: | + g++ --version + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 12 + sudo update-alternatives --set g++ /usr/bin/g++-12 + g++ --version + + - name: Compile with SCons + uses: OpenVicProject/openvic-build@master + with: + platform: ${{ inputs.platform }} + target: ${{ inputs.target }} + sconsflags: arch=${{ inputs.arch }} ${{ inputs.scons-flags }} + + - name: Delete compilation files + if: ${{ inputs.platform == 'windows' }} + run: | + Remove-Item bin/* -Include *.exp,*.pdb -Force + + - name: Upload library artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ github.event.repository.name }}-${{ inputs.identifier }}-library + path: | + ${{ github.workspace }}/bin/libopenvic-simulation.* + + - name: Upload executable artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ github.event.repository.name }}-${{ inputs.identifier }}-executable + path: | + ${{ github.workspace }}/bin/openvic-simulation.headless.* + + merge-library-files: + name: 📚 Merge Library Files + runs-on: ubuntu-latest + needs: build + if: inputs.merge-library-files + steps: + - name: Merge Artifacts + uses: actions/upload-artifact/merge@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + delete-merged: true + name: ${{ github.event.repository.name }}-library + pattern: ${{ github.event.repository.name }}-*-library diff --git a/.github/workflows/reusable-release.yml b/.github/workflows/reusable-release.yml new file mode 100644 index 000000000..9a6a81223 --- /dev/null +++ b/.github/workflows/reusable-release.yml @@ -0,0 +1,149 @@ +name: 🖥️ Reusable Release + +on: + workflow_call: + inputs: + target: + required: false + default: 'OpenVic-Simulation' + type: string + prerelease: + required: true + type: boolean + scons-flags: + required: false + default: 'use_hot_reload=no build_ovsim_tests=no debug_symbols=no' + type: string + secrets: + ARCHIVE_REPO_TOKEN: + required: false + GPG_SIGNING_KEY: + required: false + RELEASE_KEY: + required: false + workflow_dispatch: + inputs: + target: + description: | + TARGET repo to publish this release to + required: false + default: 'OpenVic-Simulation' + type: string + prerelease: + description: Pre-release + default: false + type: boolean + scons-flags: + description: | + Additional flags to send to SCons + required: false + default: 'use_hot_reload=no build_ovsim_tests=no debug_symbols=no' + type: string + +concurrency: + group: ${{ github.workflow }}|${{ github.ref }} + cancel-in-progress: true + +permissions: {} + +jobs: + static-checks-release: + name: Code style, file formatting, and docs + runs-on: ubuntu-latest + steps: + - name: Checkout project + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 # Treeless clone. Slightly less performant than a shallow clone, but makes finding diffs instantaneous. + filter: tree:0 # See: https://github.blog/open-source/git/get-up-to-speed-with-partial-clone-and-shallow-clone/ + + - name: Install APT dependencies + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 + with: + packages: libxml2-utils + + - name: Style checks via prek + uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 + + build-release: + name: ${{ matrix.name }} + needs: static-checks-release + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - identifier: windows-release + os: windows-latest + name: 🏁 Windows Release + target: template_release + platform: windows + arch: x86_64 + scons-flags: '' + + - identifier: macos-release + os: macos-latest + name: 🍎 macOS (universal) Release + target: template_release + platform: macos + arch: universal + scons-flags: 'lto=full' + + - identifier: linux-release + os: ubuntu-latest + name: 🐧 Linux Release + runner: ubuntu-20.04 + target: template_release + platform: linux + arch: x86_64 + scons-flags: 'lto=full use_static_cpp=yes' + + uses: ./.github/workflows/reusable-build.yml + with: + identifier: ${{ matrix.identifier }} + os: ${{ matrix.runner || matrix.os }} + name: ${{ matrix.name }} + target: ${{ matrix.target }} + platform: ${{ matrix.platform }} + arch: ${{ matrix.arch }} + scons-flags: ${{ matrix.scons-flags }} ${{ inputs.scons-flags }} + + upload-release-assets: + name: Upload Release Assets + runs-on: ubuntu-latest + needs: build-release + permissions: + id-token: write # Needed for trusted publishing + contents: write # Needed by gh to publish release to Github + steps: + - name: Download library assets + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: "${{ github.workspace }}/library" + pattern: "libopenvic-simulation.*" + merge-multiple: true + + - name: Download executable assets + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: "openvic-simulation.headless.*" + merge-multiple: true + + - name: Zip library assets + uses: thedoctor0/zip-release@b57d897cb5d60cb78b51a507f63fa184cfe35554 # v0.7.6 + with: + type: "zip" + filename: "libopenvic-simulation.zip" + directory: "${{ github.workspace }}/library/" + + - name: Create and upload asset + uses: ncipollo/release-action@339a81892b84b4eeb0f6e744e4574d79d0d9b8dd # v1.21.0 + with: + allowUpdates: true + artifacts: "${{ github.workspace }}/*" + omitNameDuringUpdate: true + omitBodyDuringUpdate: true + prerelease: ${{ inputs.prerelease }} + token: ${{ secrets.GITHUB_TOKEN }} + repo: ${{ inputs.target }}