diff --git a/src/incatools/odk/github.py b/src/incatools/odk/github.py new file mode 100644 index 0000000..7f66113 --- /dev/null +++ b/src/incatools/odk/github.py @@ -0,0 +1,98 @@ +# odkcore - Ontology Development Kit Core +# Copyright © 2026 ODK Developers +# +# This file is part of the ODK Core project and distributed under the +# terms of a 3-clause BSD license. See the LICENSE file in that project +# for the detailed conditions. + +from time import sleep +from typing import Any, Dict, Set, Tuple + +import requests +from requests.exceptions import RequestException + +from .download import RETRIABLE_HTTP_ERRORS + + +class GitHubHelper(object): + """Helper class to interact with the GitHub API. + + For now, the only purpose of this class is to automatically obtain + the commit ID of the latest release of a GitHub Action. It may be + expanded for other purposes in the future. + """ + + cache: Dict[str, Tuple[str, str]] + failure_cache: Set[str] + + def __init__(self): + self.cache = {} + self.failure_cache = set() + + def get_latest_release_sha(self, name: str, default: str) -> str: + """Gets the commit ID for the latest release of a GitHub project. + + :param name: The name of a GitHub project, in `owner/repo` form. + :param default: The default tag to fallback to if we can't get the + required information from GitHub. + :returns: A string of the form `XXXX # TAG`, where `XXXX` is the + commit ID of the latest release and `TAG` is the + corresponding tag name; or just the value of the `default` + parameter if the latest commit ID could not be obtained. + """ + latest = self.get_latest_release(name) + if latest: + return f"{latest[1]} # {latest[0]}" + return default + + def get_latest_release(self, name: str) -> Tuple[str, str] | None: + """Gets the tag name and commit ID for the latest release of a GitHub project. + + :param name: The name of a GitHub project, in `owner/repo` form. + :returns: A tuple (TAG,XXXX), where `TAG` is the tag of the + latest release and `XXXX` is the corresponding commit ID; or + None if we could not obtain the information from GitHub. + """ + cached = self.cache.get(name) + if cached or name in self.failure_cache: + return cached + + try: + latest_release = self._query_github_api(f"repos/{name}/releases/latest") + tagname = latest_release["tag_name"] + + release_ref = self._query_github_api(f"repos/{name}/git/ref/tags/{tagname}") + sha = release_ref["object"]["sha"] + if release_ref["object"]["type"] != "commit": + release_tag = self._query_github_api(f"repos/{name}/git/tags/{sha}") + sha = release_tag["object"]["sha"] + + self.cache[name] = (tagname, sha) + return (tagname, sha) + except (KeyError, RequestException): + # We don't really care about what went wrong exactly (e.g. + # network issue or unexpected JSON content). + self.failure_cache.add(name) + return None + + def _query_github_api(self, endpoint: str, max_retry: int = 4) -> Dict[str, Any]: + """Sends a query to the GitHub API and returns the JSON response.""" + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2026-03-10", + } + n_try = 0 + while True: + response = requests.get( + f"https://api.github.com/{endpoint}", timeout=5, headers=headers + ) + if response.status_code == 200: + return response.json() + elif response.status_code in RETRIABLE_HTTP_ERRORS and n_try < max_retry: + n_try += 1 + sleep(1) + else: + response.raise_for_status() + # We could get there upon receiving a non-error HTTP + # status (e.g. 203, 204) + raise RequestException(f"Unexpected status: {response.status_code}") diff --git a/src/incatools/odk/template.py b/src/incatools/odk/template.py index a4910e8..651f25f 100644 --- a/src/incatools/odk/template.py +++ b/src/incatools/odk/template.py @@ -18,6 +18,7 @@ from defusedxml import ElementTree as DefusedElementTree from jinja2 import Template +from .github import GitHubHelper from .model import OntologyProject from .util import runcmd @@ -87,6 +88,7 @@ class Generator(object): project: OntologyProject templatedir: Path + gh_helper: GitHubHelper def __init__(self, project: OntologyProject, templatedir: Optional[str] = None): """Creates a new instance for the specified ontology project. @@ -100,6 +102,7 @@ def __init__(self, project: OntologyProject, templatedir: Optional[str] = None): self.templatedir = Path(templatedir) else: self.templatedir = DEFAULT_TEMPLATE_DIR + self.gh_helper = GitHubHelper() def generate(self, input: Path | str) -> str: """Renders one template file. @@ -112,10 +115,12 @@ def generate(self, input: Path | str) -> str: template = Template(file_.read()) if "ODK_VERSION" in os.environ: return template.render( - project=self.project, env={"ODK_VERSION": os.getenv("ODK_VERSION")} + project=self.project, + gh=self.gh_helper, + env={"ODK_VERSION": os.getenv("ODK_VERSION")}, ) else: - return template.render(project=self.project) + return template.render(project=self.project, gh=self.gh_helper) def generate_from_name(self, name: str) -> str: """Renders one template. diff --git a/src/incatools/odk/templates/_dynamic_workflows.jinja2 b/src/incatools/odk/templates/_dynamic_workflows.jinja2 index d18f8cd..a422605 100644 --- a/src/incatools/odk/templates/_dynamic_workflows.jinja2 +++ b/src/incatools/odk/templates/_dynamic_workflows.jinja2 @@ -80,7 +80,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v6 + - uses: actions/checkout@{{ gh.get_latest_release_sha("actions/checkout", "v7") }} - name: Run ontology QC checks env: @@ -106,16 +106,16 @@ jobs: runs-on: ubuntu-latest container: obolibrary/odklite:{% if env is defined -%}{{env['ODK_VERSION'] or "latest" }}{%- else %}latest{% endif %} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@{{ gh.get_latest_release_sha("actions/checkout", "v7") }} # Checks-out main branch under "main" directory - - uses: actions/checkout@v6 + - uses: actions/checkout@{{ gh.get_latest_release_sha("actions/checkout", "v7") }} with: ref: master path: master - name: Diff classification run: export ROBOT_JAVA_ARGS=-Xmx6G; robot diff --labels True --left master/src/ontology/{{ project.id }}-edit.{{ project.edit_format }} --left-catalog master/src/ontology/catalog-v001.xml --right src/ontology/{{ project.id }}-edit.{{ project.edit_format }} --right-catalog src/ontology/catalog-v001.xml -f markdown -o edit-diff.md - name: Upload diff - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@{{ gh.get_latest_release_sha("actions/upload-artifact", "v7") }} with: name: edit-diff.md path: edit-diff.md @@ -123,11 +123,11 @@ jobs: runs-on: ubuntu-latest container: obolibrary/odklite:{% if env is defined -%}{{env['ODK_VERSION'] or "latest" }}{%- else %}latest{% endif %} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@{{ gh.get_latest_release_sha("actions/checkout", "v7") }} - name: Classify ontology run: cd src/ontology; make IMP=FALSE PAT=FALSE MIR=FALSE {{ project.id }}.owl - name: Upload PR {{ project.id }}.owl - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@{{ gh.get_latest_release_sha("actions/upload-artifact", "v7") }} with: name: {{ project.id }}-pr.owl path: src/ontology/{{ project.id }}.owl @@ -136,13 +136,13 @@ jobs: runs-on: ubuntu-latest container: obolibrary/odklite:{% if env is defined -%}{{env['ODK_VERSION'] or "latest" }}{%- else %}latest{% endif %} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@{{ gh.get_latest_release_sha("actions/checkout", "v7") }} with: ref: master - name: Classify ontology run: cd src/ontology; make IMP=FALSE PAT=FALSE MIR=FALSE {{ project.id }}.owl - name: Upload master {{ project.id }}.owl - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@{{ gh.get_latest_release_sha("actions/upload-artifact", "v7") }} with: name: {{ project.id }}-master.owl path: src/ontology/{{ project.id }}.owl @@ -154,21 +154,21 @@ jobs: runs-on: ubuntu-latest container: obolibrary/odklite:{% if env is defined -%}{{env['ODK_VERSION'] or "latest" }}{%- else %}latest{% endif %} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@{{ gh.get_latest_release_sha("actions/checkout", "v7") }} - name: Download master classification - uses: actions/download-artifact@v8 + uses: actions/download-artifact@{{ gh.get_latest_release_sha("actions/download-artifact", "v8") }} with: name: {{ project.id }}-master.owl path: src/ontology/{{ project.id }}-master.owl - name: Download PR classification - uses: actions/download-artifact@v8 + uses: actions/download-artifact@{{ gh.get_latest_release_sha("actions/download-artifact", "v8") }} with: name: {{ project.id }}-pr.owl path: src/ontology/{{ project.id }}-pr.owl - name: Diff classification run: export ROBOT_JAVA_ARGS=-Xmx6G; cd src/ontology; robot diff --labels True --left {{ project.id }}-master.owl/{{ project.id }}.owl --left-catalog catalog-v001.xml --right {{ project.id }}-pr.owl/{{ project.id }}.owl --right-catalog catalog-v001.xml -f markdown -o classification-diff.md - name: Upload diff - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@{{ gh.get_latest_release_sha("actions/upload-artifact", "v7") }} with: name: classification-diff.md path: src/ontology/classification-diff.md @@ -176,9 +176,9 @@ jobs: needs: [diff_classification, edit_file] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@{{ gh.get_latest_release_sha("actions/checkout", "v7") }} - name: Download reasoned diff - uses: actions/download-artifact@v8 + uses: actions/download-artifact@{{ gh.get_latest_release_sha("actions/download-artifact", "v8") }} with: name: classification-diff.md path: classification-diff.md @@ -187,13 +187,13 @@ jobs: - name: Post reasoned comment env: GITHUB_TOKEN: {% raw %}${{ secrets.GITHUB_TOKEN }}{% endraw %} - uses: NejcZdovc/comment-pr@v2 + uses: NejcZdovc/comment-pr@{{ gh.get_latest_release_sha("NejcZdovc/comment-pr", "v2") }} with: file: "../../comment.md" identifier: "REASONED" - - uses: actions/checkout@v6 + - uses: actions/checkout@{{ gh.get_latest_release_sha("actions/checkout", "v7") }} - name: Download edit diff - uses: actions/download-artifact@v8 + uses: actions/download-artifact@{{ gh.get_latest_release_sha("actions/download-artifact", "v8") }} with: name: edit-diff.md path: edit-diff.md @@ -202,7 +202,7 @@ jobs: - name: Post comment env: GITHUB_TOKEN: {% raw %}${{ secrets.GITHUB_TOKEN }}{% endraw %} - uses: NejcZdovc/comment-pr@v2 + uses: NejcZdovc/comment-pr@{{ gh.get_latest_release_sha("NejcZdovc/comment-pr", "v2") }} with: file: "../../edit-comment.md" identifier: "UNREASONED" @@ -226,7 +226,7 @@ jobs: post_diff: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@{{ gh.get_latest_release_sha("actions/checkout", "v7") }} - name: Prepare release comment env: GITHUB_SHA: {% raw %}${{ github.sha }}{% endraw %} @@ -234,7 +234,7 @@ jobs: - name: Post reasoned comment env: GITHUB_TOKEN: {% raw %}${{ secrets.GITHUB_TOKEN }}{% endraw %} - uses: NejcZdovc/comment-pr@v2 + uses: NejcZdovc/comment-pr@{{ gh.get_latest_release_sha("NejcZdovc/comment-pr", "v2") }} with: github_token: {% raw %}${{ env.GITHUB_TOKEN }}{% endraw %} file: "../../comment.md" @@ -260,10 +260,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout main - uses: actions/checkout@v6 + uses: actions/checkout@{{ gh.get_latest_release_sha("actions/checkout", "v7") }} - name: Deploy docs - uses: mhausenblas/mkdocs-deploy-gh-pages@master + uses: mhausenblas/mkdocs-deploy-gh-pages@a31c6b13a80e4a4fbb525eeb7a2a78253bb15fa5 # master @ 2024-07-19 # Or use mhausenblas/mkdocs-deploy-gh-pages@nomaterial to build without the mkdocs-material theme env: GITHUB_TOKEN: {% raw %}${{ secrets.GITHUB_TOKEN }}{% endraw %}