[AIENG-679] Unify GitHub Actions OIDC onto the generic OIDC flow

smart_tests/commands/verify.py

@@ -18,7 +18,7 @@
 from ..app import Application
 from ..utils.authentication import ensure_org_workspace, get_oidc_token, get_org_workspace
 from ..utils.commands import Command
-from ..utils.env_keys import OIDC_TOKEN_KEY, ORGANIZATION_KEY, TOKEN_KEY, WORKSPACE_KEY
+from ..utils.env_keys import GITHUB_OIDC_KEY, LEGACY_GITHUB_OIDC_KEY, OIDC_TOKEN_KEY, ORGANIZATION_KEY, TOKEN_KEY, WORKSPACE_KEY
 from ..utils.http_client import DEFAULT_GET_TIMEOUT, _HttpClient
 from ..utils.java import get_java_command
 from ..utils.smart_tests_client import SmartTestsClient
@@ -204,20 +204,27 @@ def verify_oidc(app_instance: Application):
     Credential-free OIDC bootstrap. Presents the pipeline's OIDC id-token to Intake's
     /intake/oidc/verify endpoint and translates the 200/403/401 contract into CLI behavior:
 
-      - 200: the subject is registered. Print `export` lines (org/workspace/oidc-token) so the
-             pipeline can `eval "$(smart-tests verify --oidc)"` and authenticate subsequent
-             commands with the same token. Exit 0.
+      - 200: the subject is registered. Print `export` lines (org/workspace, plus the OIDC token
+             when it is a fixed pass-through value) so the pipeline can
+             `eval "$(smart-tests verify --oidc)"` and authenticate subsequent commands. Exit 0.
       - 403: the token verified but its subject isn't registered to any workspace yet. Show the
              normalized `sub` so the user can register it from the WebApp settings. Exit 1.
       - 401: the token is missing/expired/invalid. Exit 1.
     '''
     tracking_client = TrackingClient(Command.VERIFY, app=app_instance)
 
-    token = get_oidc_token()
-    if not token:
-        msg = (f"OIDC authentication requires the {OIDC_TOKEN_KEY} environment variable to hold the "
-               "pipeline's OIDC id-token. In Jenkins, bind an id-token credential to this variable; "
-               "see the OIDC pipeline-authentication setup guide.")
+    # authentication_headers() supplies the bearer for the request below; here we only confirm an OIDC
+    # flow is configured (fail fast) and decide whether the token can be echoed back. A pass-through
+    # token (SMART_TESTS_OIDC_TOKEN, e.g. a Jenkins-bound credential) is a fixed value we re-export;
+    # the GitHub flows mint a short-lived token per request, so exporting one here would only leave a
+    # stale value behind.
+    passthrough_token = get_oidc_token()
+    github_oidc_configured = os.getenv(GITHUB_OIDC_KEY) or os.getenv(LEGACY_GITHUB_OIDC_KEY)
+    if not passthrough_token and not github_oidc_configured:
+        msg = (f"OIDC authentication requires an OIDC flow to be configured: set {OIDC_TOKEN_KEY} to "
+               "the pipeline's OIDC id-token (e.g. a Jenkins-bound credential), or set "
+               f"{GITHUB_OIDC_KEY}=1 in GitHub Actions to fetch one automatically. "
+               "See the OIDC pipeline-authentication setup guide.")
         click.secho(msg, fg='red', err=True)
         tracking_client.send_error_event(
             event_name=Tracking.ErrorEvent.USER_ERROR,
@@ -296,11 +303,13 @@ def verify_oidc(app_instance: Application):
 
     # Emit eval-able export lines so the pipeline can hydrate its environment:
     #   eval "$(smart-tests verify --oidc)"
-    # Subsequent commands then read org/workspace from these vars and present the same OIDC token
-    # (kept in SMART_TESTS_OIDC_TOKEN) as their bearer.
+    # Subsequent commands read org/workspace from these vars. For a pass-through token we also
+    # re-export it as the bearer; the GitHub flows re-mint a fresh short-lived token per request in
+    # authentication_headers(), so exporting one here would only leave a stale value behind.
     click.echo(f'export {ORGANIZATION_KEY}={_shell_quote(org)}')
     click.echo(f'export {WORKSPACE_KEY}={_shell_quote(workspace)}')
-    click.echo(f'export {OIDC_TOKEN_KEY}={_shell_quote(token)}')
+    if passthrough_token:
+        click.echo(f'export {OIDC_TOKEN_KEY}={_shell_quote(passthrough_token)}')
     click.secho(
         f"OIDC authentication verified for organization {org!r}, workspace {workspace!r}" + emoji(" \U0001f389"),
         fg='green', err=True)

smart_tests/utils/authentication.py

@@ -1,12 +1,24 @@
 import os
-from typing import Tuple
+from typing import Optional, Tuple
+from urllib.parse import quote
 
 import click
 import requests
 
 import smart_tests.args4p.typer as typer
 
-from .env_keys import OIDC_TOKEN_KEY, ORGANIZATION_KEY, WORKSPACE_KEY, get_token
+from .env_keys import (GITHUB_OIDC_KEY, LEGACY_GITHUB_OIDC_KEY, OIDC_AUDIENCE_KEY,
+                       OIDC_TOKEN_KEY, ORGANIZATION_KEY, WORKSPACE_KEY, get_token)
+
+# Default audience Intake expects in an OIDC id-token (launchableinc.intake.oidc.audience).
+DEFAULT_OIDC_AUDIENCE = "https://app.cloudbees.io/smart-tests"
+# Header the CLI sends to opt into Intake's deprecated GitHub Actions OIDC path. Absent it, a
+# GitHub-issued token is verified through the generic OIDC path. Mirrors RESTAuthConverter.
+LEGACY_GITHUB_OIDC_HEADER = "GitHub-OIDC-Legacy"
+
+# authentication_headers() runs on every API request, so guard the legacy deprecation notice to
+# print at most once per process instead of once per request.
+_legacy_oidc_warning_shown = False
 
 
 def get_org_workspace():
@@ -60,24 +72,33 @@ def authentication_headers():
     if oidc_token:
         return {'Authorization': f'Bearer {oidc_token}'}
 
-    if os.getenv('EXPERIMENTAL_GITHUB_OIDC_TOKEN_AUTH'):
-        req_url = os.getenv('ACTIONS_ID_TOKEN_REQUEST_URL')
-        rt_token = os.getenv('ACTIONS_ID_TOKEN_REQUEST_TOKEN')
-        if not req_url or not rt_token:
+    # Generic GitHub Actions OIDC: fetch the id-token minted for the Smart Tests audience and present
+    # it like any other OIDC token. Intake routes by `iss` to the generic verifier and matches the
+    # normalized `repo:OWNER/REPO` subject against trusted_oidc_subjects. The audience is required
+    # here because the generic path enforces `aud` for GitHub's issuer.
+    if os.getenv(GITHUB_OIDC_KEY):
+        id_token = _fetch_github_id_token(audience=_expected_oidc_audience())
+        return {'Authorization': f'Bearer {id_token}'}
+
+    # Deprecated legacy GitHub Actions OIDC: Intake matches the `repository` claim against
+    # trusted_github_repositories. The legacy path never checks `aud`, so no audience is requested.
+    # The header tells Intake to take the legacy branch; without it the token would be verified
+    # through the generic path.
+    if os.getenv(LEGACY_GITHUB_OIDC_KEY):
+        global _legacy_oidc_warning_shown
+        if not _legacy_oidc_warning_shown:
+            _legacy_oidc_warning_shown = True
             click.secho(
-                "GitHub Actions OIDC tokens cannot be retrieved."
-                "Confirm that you have added necessary permissions following "
-                "https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers#adding-permissions-settings",  # noqa: E501
-                fg='red', err=True)
-            raise typer.Exit(1)
-        r = requests.get(req_url,
-                         headers={
-                             'Authorization': f'Bearer {rt_token}',
-                             'Accept': 'application/json; api-version=2.0',
-                             'Content-Type': 'application/json',
-                         })
-        r.raise_for_status()
-        return {"Authorization": f"Bearer {r.json()['value']}"}
+                f"{LEGACY_GITHUB_OIDC_KEY} enables the deprecated GitHub Actions OIDC flow. Migrate "
+                f"by registering your repository as a Trusted OIDC subject and switching to "
+                f"{GITHUB_OIDC_KEY}=1. See "
+                "https://docs.cloudbees.com/docs/cloudbees-smart-tests/latest/send-data-to-smart-tests/set-up-smart-tests/migration-to-github-oidc-auth",  # noqa: E501
+                fg='yellow', err=True)
+        id_token = _fetch_github_id_token()
+        return {
+            'Authorization': f'Bearer {id_token}',
+            LEGACY_GITHUB_OIDC_HEADER: '1',
+        }
 
     if os.getenv('GITHUB_ACTIONS'):
         headers = {
@@ -97,3 +118,38 @@ def authentication_headers():
 
         return headers
     return {}
+
+
+def _expected_oidc_audience() -> str:
+    '''Audience the GitHub id-token must carry for Intake's generic OIDC path to accept it.'''
+    return os.getenv(OIDC_AUDIENCE_KEY) or DEFAULT_OIDC_AUDIENCE
+
+
+def _fetch_github_id_token(audience: Optional[str] = None) -> str:
+    '''
+    Retrieve a GitHub Actions OIDC id-token via the runner's token endpoint.
+
+    Requires the `id-token: write` workflow permission, which populates ACTIONS_ID_TOKEN_REQUEST_URL
+    and ACTIONS_ID_TOKEN_REQUEST_TOKEN. When `audience` is given it is requested so the token's `aud`
+    claim matches what Intake expects (the generic OIDC path enforces it); the legacy path omits it.
+    '''
+    req_url = os.getenv('ACTIONS_ID_TOKEN_REQUEST_URL')
+    rt_token = os.getenv('ACTIONS_ID_TOKEN_REQUEST_TOKEN')
+    if not req_url or not rt_token:
+        click.secho(
+            "GitHub Actions OIDC tokens cannot be retrieved."
+            "Confirm that you have added necessary permissions following "
+            "https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers#adding-permissions-settings",  # noqa: E501
+            fg='red', err=True)
+        raise typer.Exit(1)
+    if audience:
+        sep = '&' if '?' in req_url else '?'
+        req_url = f"{req_url}{sep}audience={quote(audience, safe='')}"
+    r = requests.get(req_url,
+                     headers={
+                         'Authorization': f'Bearer {rt_token}',
+                         'Accept': 'application/json; api-version=2.0',
+                         'Content-Type': 'application/json',
+                     })
+    r.raise_for_status()
+    return r.json()['value']

smart_tests/utils/env_keys.py

@@ -9,6 +9,16 @@
 ORGANIZATION_KEY = "SMART_TESTS_ORGANIZATION"
 WORKSPACE_KEY = "SMART_TESTS_WORKSPACE"
 BASE_URL_KEY = "SMART_TESTS_BASE_URL"
+# Opt in to the generic GitHub Actions OIDC flow: the CLI fetches the GitHub id-token and presents
+# it like any other OIDC token. Intake verifies it against trusted_oidc_subjects (self-serve in the
+# webapp) instead of the deprecated trusted_github_repositories path. See authentication_headers().
+GITHUB_OIDC_KEY = "SMART_TESTS_GITHUB_OIDC_TOKEN_AUTH"
+# Deprecated opt in to the legacy GitHub Actions OIDC flow (repository-claim matching). Kept working
+# for backward compatibility; when set, the CLI signals Intake to use the legacy path via a header.
+LEGACY_GITHUB_OIDC_KEY = "EXPERIMENTAL_GITHUB_OIDC_TOKEN_AUTH"
+# Audience the GitHub id-token must be minted for so the generic OIDC path's aud check passes.
+# Overridable for non-production Intake environments.
+OIDC_AUDIENCE_KEY = "SMART_TESTS_OIDC_AUDIENCE"
 SKIP_TIMEOUT_RETRY = "SMART_TESTS_SKIP_TIMEOUT_RETRY"
 COMMIT_TIMEOUT = "SMART_TESTS_COMMIT_TIMEOUT"
 SKIP_CERT_VERIFICATION = "SMART_TESTS_SKIP_CERT_VERIFICATION"

src/main/java/com/launchableinc/ingest/commits/GitHubIdTokenAuthenticator.java

@@ -6,6 +6,9 @@
 import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import org.apache.http.Header;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
@@ -15,10 +18,29 @@
 import org.kohsuke.args4j.CmdLineException;
 
 public class GitHubIdTokenAuthenticator implements Authenticator {
+  // Default audience Intake expects in an OIDC id-token (launchableinc.intake.oidc.audience).
+  // Mirrors the Python CLI's DEFAULT_OIDC_AUDIENCE.
+  static final String DEFAULT_OIDC_AUDIENCE = "https://app.cloudbees.io/smart-tests";
+  // Header the CLI sends to opt into Intake's deprecated GitHub Actions OIDC path. Absent it, a
+  // GitHub-issued token is verified through the generic OIDC path. Mirrors RESTAuthConverter.
+  static final String LEGACY_GITHUB_OIDC_HEADER = "GitHub-OIDC-Legacy";
+
   private static final ObjectMapper objectMapper = new ObjectMapper();
   private final String idToken;
+  private final boolean legacy;
 
-  public GitHubIdTokenAuthenticator() throws CmdLineException {
+  /**
+   * Retrieves a GitHub Actions OIDC id-token via the runner's token endpoint.
+   *
+   * @param audience When non-empty, requested so the token's {@code aud} claim matches what
+   *     Intake's generic OIDC path enforces. Pass {@code null}/empty for the legacy path, which
+   *     never checks {@code aud}.
+   * @param legacy When true, signals Intake to take the deprecated GitHub Actions OIDC path via the
+   *     {@link #LEGACY_GITHUB_OIDC_HEADER} header; without it the token is verified through the
+   *     generic path.
+   */
+  public GitHubIdTokenAuthenticator(String audience, boolean legacy) throws CmdLineException {
+    this.legacy = legacy;
     String reqUrl = System.getenv("ACTIONS_ID_TOKEN_REQUEST_URL");
     String rtToken = System.getenv("ACTIONS_ID_TOKEN_REQUEST_TOKEN");
     if (Strings.isNullOrEmpty(reqUrl) || Strings.isNullOrEmpty(rtToken)) {
@@ -28,6 +50,11 @@ public GitHubIdTokenAuthenticator() throws CmdLineException {
               + "https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers#adding-permissions-settings");
     }
 
+    if (!Strings.isNullOrEmpty(audience)) {
+      String sep = reqUrl.contains("?") ? "&" : "?";
+      reqUrl = reqUrl + sep + "audience=" + encode(audience);
+    }
+
     HttpGet request = new HttpGet(reqUrl);
     request.setHeader("Authorization", "Bearer " + rtToken);
     request.setHeader("Accept", "applicaiton/json; api-version=2.0");
@@ -49,7 +76,21 @@ public GitHubIdTokenAuthenticator() throws CmdLineException {
 
   @Override
   public ImmutableList<Header> getAuthenticationHeaders() {
-    return ImmutableList.of(new BasicHeader("Authorization", "Bearer " + idToken));
+    ImmutableList.Builder<Header> headers = ImmutableList.builder();
+    headers.add(new BasicHeader("Authorization", "Bearer " + idToken));
+    if (legacy) {
+      headers.add(new BasicHeader(LEGACY_GITHUB_OIDC_HEADER, "1"));
+    }
+    return headers.build();
+  }
+
+  private static String encode(String value) {
+    try {
+      return URLEncoder.encode(value, StandardCharsets.UTF_8.name());
+    } catch (UnsupportedEncodingException e) {
+      // UTF-8 is always supported.
+      throw new AssertionError(e);
+    }
   }
 
   @JsonIgnoreProperties(ignoreUnknown = true)

src/main/java/com/launchableinc/ingest/commits/Main.java

@@ -1,6 +1,7 @@
 package com.launchableinc.ingest.commits;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
 import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
@@ -110,8 +111,26 @@ private void parseConfiguration() throws CmdLineException {
           this.ws = w;
         }
 
-        if (System.getenv("EXPERIMENTAL_GITHUB_OIDC_TOKEN_AUTH") != null) {
-          authenticator = new GitHubIdTokenAuthenticator();
+        if (!Strings.isNullOrEmpty(System.getenv("SMART_TESTS_GITHUB_OIDC_TOKEN_AUTH"))) {
+          // Generic GitHub Actions OIDC: fetch the id-token minted for the Smart Tests audience and
+          // present it like any other OIDC token. Intake routes by `iss` to the generic verifier
+          // and matches the normalized `repo:OWNER/REPO` subject against trusted_oidc_subjects. The
+          // audience is required because the generic path enforces `aud` for GitHub's issuer.
+          String audience = System.getenv("SMART_TESTS_OIDC_AUDIENCE");
+          if (Strings.isNullOrEmpty(audience)) {
+            audience = GitHubIdTokenAuthenticator.DEFAULT_OIDC_AUDIENCE;
+          }
+          authenticator = new GitHubIdTokenAuthenticator(audience, false);
+        } else if (!Strings.isNullOrEmpty(System.getenv("EXPERIMENTAL_GITHUB_OIDC_TOKEN_AUTH"))) {
+          // Deprecated legacy GitHub Actions OIDC: Intake matches the `repository` claim against
+          // trusted_github_repositories. The legacy path never checks `aud`, so no audience is
+          // requested; the legacy header tells Intake to take the legacy branch.
+          System.err.println(
+              "EXPERIMENTAL_GITHUB_OIDC_TOKEN_AUTH enables the deprecated GitHub Actions OIDC flow."
+                  + " Migrate by registering your repository as a Trusted OIDC subject and switching"
+                  + " to SMART_TESTS_GITHUB_OIDC_TOKEN_AUTH=1. See "
+                  + "https://docs.cloudbees.com/docs/cloudbees-smart-tests/latest/send-data-to-smart-tests/set-up-smart-tests/migration-to-github-oidc-auth");
+          authenticator = new GitHubIdTokenAuthenticator(null, true);
         } else {
           authenticator = new GitHubActionsAuthenticator();
         }

tests/commands/test_verify.py

@@ -179,11 +179,49 @@ def test_oidc_invalid_token_fails(self):
 
     @patch.dict(os.environ, {}, clear=True)
     def test_oidc_missing_token(self):
-        """No OIDC token in the environment → usage error, exit 2."""
+        """No OIDC flow configured → usage error, exit 2."""
         result = self.cli("verify", "--oidc")
         self.assert_exit_code(result, 2)
         self.assertIn("SMART_TESTS_OIDC_TOKEN", result.output)
 
+    gh_id_token = "gh-header.gh-payload.gh-signature"
+    gh_oidc_env = {
+        "SMART_TESTS_GITHUB_OIDC_TOKEN_AUTH": "1",
+        "SMART_TESTS_BASE_URL": base_url,
+        "ACTIONS_ID_TOKEN_REQUEST_URL": "http://gh-oidc.local/token",
+        "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "runner-rt-token",
+    }
+
+    @responses.activate
+    @patch.dict(os.environ, gh_oidc_env, clear=True)
+    def test_oidc_github_generic_no_token_export(self):
+        """GitHub generic flow (no SMART_TESTS_OIDC_TOKEN): the CLI fetches a fresh id-token, presents
+        it as the bearer, and on 200 exports org/workspace but NOT the short-lived token."""
+        responses.add(
+            responses.GET,
+            "http://gh-oidc.local/token",
+            json={"value": self.gh_id_token},
+            status=200,
+        )
+        responses.add(
+            responses.POST,
+            self.oidc_verify_url,
+            json={"organization": "acme", "workspace": "prod"},
+            status=200,
+        )
+
+        result = self.cli("verify", "--oidc")
+        self.assert_success(result)
+
+        self.assertIn("export SMART_TESTS_ORGANIZATION='acme'", result.output)
+        self.assertIn("export SMART_TESTS_WORKSPACE='prod'", result.output)
+        # The GitHub token is minted per request, so it must not be exported for reuse.
+        self.assertNotIn("export SMART_TESTS_OIDC_TOKEN", result.output)
+
+        # The fetched GitHub id-token is the bearer presented to the verify endpoint.
+        verify_call = next(c for c in responses.calls if "/oidc/verify" in c.request.url)
+        self.assertEqual(verify_call.request.headers["Authorization"], f"Bearer {self.gh_id_token}")
+
 
 def _make_jwt(claims: dict) -> str:
     """Build an unsigned-looking JWT (header.payload.signature) with the given claims payload."""