diff --git a/.github/workflows/shared-reference-app-ui.yml b/.github/workflows/shared-reference-app-ui.yml index d21c55e..25b774e 100644 --- a/.github/workflows/shared-reference-app-ui.yml +++ b/.github/workflows/shared-reference-app-ui.yml @@ -5,12 +5,22 @@ on: branches: [main] paths: - '.github/workflows/shared-reference-app-ui.yml' + - '.github/workflows/publish-portfolio-evidence.yml' - 'docker/shared-app.compose.yaml' - - 'src/test/java/io/github/selenium/saucedemo/tests/SharedReferenceApplicationTests.java' - - 'testng-shared-app.xml' + - 'src/main/java/**' + - 'src/test/java/**' + - 'src/test/resources/**' + - 'docs/shared-reference-app.md' + - 'docs/verification/2026-10-16-selenium-lifecycle-grid.md' + - 'testng*.xml' + - '.mvn/**' + - 'mvnw*' + - 'pom.xml' - 'scripts/emit_evidence.py' - 'scripts/test_emit_evidence.py' - - 'pom.xml' + - 'scripts/validate-evidence.mjs' + - 'scripts/verify_shared_app_controls.py' + - 'scripts/test_shared_app_controls.py' workflow_dispatch: permissions: @@ -33,6 +43,7 @@ jobs: TARGET_ENVIRONMENT: isolated-compose TARGET_PROFILE: chrome-grid-4.43.0 FRESHNESS_POLICY: quarterly_experiment_v1 + SCOPE_REQUIRED: 'false' EVIDENCE_CLASS: controlled steps: - name: Checkout repository @@ -49,6 +60,11 @@ jobs: java-version: '21' distribution: temurin + - name: Check the browser-free scenario coordination controls + run: | + python3 -m unittest discover -s scripts -p 'test_shared_app_controls.py' -v + python3 -m unittest discover -s scripts -p 'test_emit_evidence.py' -v + - name: Record exact runtime versions for evidence shell: bash run: | @@ -104,13 +120,18 @@ jobs: run: | set -euo pipefail mkdir -p target + rm -rf target/shared-app-controls echo "STARTED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_ENV" set +e ./mvnw -B -ntp -C -Dtestng.suite.file=testng-shared-app.xml \ -Dexecution.type=remote -Dremote.url=http://127.0.0.1:4444/wd/hub \ - -Dbrowser=CHROME -Dheadless=true -Dthread.count=2 \ + -Dbrowser=CHROME -Dheadless=true -Dthread.count=1 \ -Dapp.url=http://reference-app:4300/ test 2>&1 | tee target/shared-app-ui.log status=${PIPESTATUS[0]} + if [[ "$status" -eq 0 ]]; then + python3 scripts/verify_shared_app_controls.py target/shared-app-controls + status=$? + fi set -e echo "COMPLETED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_ENV" echo "TEST_OUTCOME=$([[ $status -eq 0 ]] && echo success || echo failure)" >> "$GITHUB_ENV" @@ -134,6 +155,18 @@ jobs: if: always() && hashFiles('target/portfolio-evidence/evidence-v3.json') != '' run: node scripts/validate-evidence.mjs target/portfolio-evidence/evidence-v3.json + - name: Summarize the exact compatibility scope + if: always() + run: | + { + echo '## Shared-app compatibility scope' + echo + echo "- Run: [${GITHUB_RUN_ID}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" + echo "- Tested checkout SHA: ${GITHUB_SHA}" + echo "- Target ID and revision: ${TARGET_ID} / ${TARGET_REVISION}" + echo '- This run applies only to that source and target pair; current compatibility is not established after either changes.' + } >> "$GITHUB_STEP_SUMMARY" + - name: Preserve logs, native reports, and evidence if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -141,6 +174,7 @@ jobs: name: selenium-shared-app-evidence-${{ github.run_attempt }} path: | target/shared-app-ui.log + target/shared-app-controls/*.json target/portfolio-evidence/evidence-v3.json target/surefire-reports/TEST-*.xml target-app-ready.json diff --git a/docs/shared-reference-app.md b/docs/shared-reference-app.md index 61b001f..7c14158 100644 --- a/docs/shared-reference-app.md +++ b/docs/shared-reference-app.md @@ -1,10 +1,21 @@ # Shared reference application profile -This profile runs two deliberately small journeys against the immutable fictional claims target used by the portfolio. It demonstrates the Selenium framework's JVM and remote Grid lifecycle, two parallel browser sessions, and the APP's server-enforced member boundary. Playwright remains the primary browser contract suite; this is not a duplicate of the full UI suite. +This optional, dated experiment runs one complete claims review journey twice against the immutable fictional target. Each invocation gives member A the `BaseTestCase` driver and gives member B a separate driver owned and cleaned by one bounded worker. It checks the target's server-enforced member boundary while the reviewer moves member A's claim through review and approval. Playwright remains the primary browser contract suite; this experiment is not a duplicate of the full UI suite or an always-required compatibility gate. -The test target pins application source `e9f438e06a2b092dccfc6e9ab1820ff2562ddad7`, fixture `v0.1.0`, its multi-architecture GHCR image digest, PostgreSQL 18.6, Selenium Grid 4.43.0, and the Chrome node digest in `docker/shared-app.compose.yaml`. The run creates an isolated namespace and random synthetic database/control credentials, keeps PostgreSQL inside the Compose network, and removes volumes during teardown. The two TestNG methods execute concurrently with separate thread-local drivers: Member A submits a claim and a reviewer moves it through review to approval; Member B verifies that the claim is absent from its own session. +The test target pins application source `e9f438e06a2b092dccfc6e9ab1820ff2562ddad7`, fixture `v0.1.0`, its multi-architecture GHCR image digest, PostgreSQL 18.6, Selenium Grid 4.43.0, and the Chrome node digest in `docker/shared-app.compose.yaml`. Each run creates an isolated namespace and random synthetic database/control credentials, keeps PostgreSQL inside the Compose network, and removes volumes during teardown. The same named TestNG scenario runs twice sequentially. It records only hashes of remote session and claim identifiers, then checks that all four sessions and both claims are distinct and cleaned up. -GitHub Actions runs this profile on pull requests that change its implementation and supports manual dispatch. Required proof includes the readiness endpoint's source/fixture identity, Grid status, Surefire native results, schema-v3 evidence validation, and unconditional Compose cleanup. The evidence scope is `selenium.shared-app`; it is separate from the existing Sauce Demo UI scopes. Anonymous image access is required. If the image is private, the run should fail before browser tests and must not be represented as a product-test failure. +GitHub Actions keeps this profile optional and runs it for pull requests that change its implementation, its runtime/build inputs, or its evidence publisher. Manual dispatch is also available; no schedule is needed. Each successful run is evidence only for its exact tested framework SHA and target revision. It does not guarantee compatibility for every later framework commit, another target revision, or general production use. + +### Last recorded successful experiment + +- Run: [Shared reference app UI #6](https://github.com/qa-test-automation-frameworks/selenium-testng-java-framework/actions/runs/39012550947), completed 2026-10-17 07:10:15 UTC. +- Tested checkout SHA: `18b7cf9b5adfe8ed81e4a6beff344cb8784322e7`. +- Target: `quality-reference-application`, source revision `e9f438e06a2b092dccfc6e9ab1820ff2562ddad7`, fixture `v0.1.0`. +- Current status: **current compatibility not established**. At the 2026-10-21 audit checkpoint, `main` was `528796a0e905420908b863d138c2dabb41a80757`, which differs from the tested checkout SHA. The target remains pinned to the listed source revision. + +Run #6 predates the single-scenario, two-invocation control now in this profile. It is a historical evidence point and does not verify the updated harness; a fresh exact-head run is required for that. + +The workflow's run summary and schema-v3 artifact carry the exact checkout SHA, run ID, completion time, and target identity for each later execution. Evidence timestamps come from the execution itself; editing this page does not refresh experiment evidence. The evidence scope is `selenium.shared-app`, separate from the required Sauce Demo UI scopes. Anonymous image access is required. If the image is private, the run should fail before browser tests and must not be represented as a product-test failure. For local execution, export unique `APP_NAMESPACE`, random `POSTGRES_PASSWORD`, and random `TEST_CONTROL_TOKEN`, then run: diff --git a/docs/verification/2026-10-16-selenium-lifecycle-grid.md b/docs/verification/2026-10-16-selenium-lifecycle-grid.md index b38a231..1662e11 100644 --- a/docs/verification/2026-10-16-selenium-lifecycle-grid.md +++ b/docs/verification/2026-10-16-selenium-lifecycle-grid.md @@ -9,3 +9,5 @@ The `grid-capacity-experiment` job runs a small real Chrome Grid sample using th The exact-head workflow result and uploaded JSON are the evidence for the sample. This checkout has no local Java or Docker runtime, so local verification is limited to source review and workflow/YAML checks until CI executes the suite. The sample records SHA-256 digests of session identifiers only, so independent sessions can be checked without publishing raw session IDs. On successful runs, the workflow summary includes the resource limits and redacted sample JSON. + +The separate shared-reference-app experiment is dated evidence for one exact framework checkout and pinned target. Its last recorded successful run is [#6](https://github.com/qa-test-automation-frameworks/selenium-testng-java-framework/actions/runs/39012550947), completed 2026-10-17 07:10:15 UTC against checkout `18b7cf9b5adfe8ed81e4a6beff344cb8784322e7` and application revision `e9f438e06a2b092dccfc6e9ab1820ff2562ddad7`. That run predates the updated single-scenario harness. Since `main` has changed, current compatibility is not established. See [the shared-app profile](../shared-reference-app.md) for its scope and current evidence record. diff --git a/scripts/emit_evidence.py b/scripts/emit_evidence.py index 12da952..54f6e9c 100644 --- a/scripts/emit_evidence.py +++ b/scripts/emit_evidence.py @@ -53,6 +53,13 @@ def build_record(env, report_dir, output): sums[key] += int(suite.attrib[key]) testcases = suite.findall("testcase") cases += len(testcases) + aggregate_tests = int(suite.attrib["tests"]) + identities = {(case.get("classname"), case.get("name")) for case in testcases} + if aggregate_tests != len(testcases): + if any(not classname or not name for classname, name in identities): + raise ValueError("repeated testcase rows must include class and method identity") + if aggregate_tests != len(identities): + raise ValueError("native test count does not match testcase rows or unique methods") case_failures = sum(any(child.tag == "failure" for child in case) for case in testcases) case_errors = sum(any(child.tag == "error" for child in case) for case in testcases) case_skips = sum(any(child.tag == "skipped" for child in case) for case in testcases) @@ -62,8 +69,8 @@ def build_record(env, report_dir, output): raise ValueError("native aggregate failure/error/skip counters do not match testcase outcomes") retried_cases += sum(any(child.tag in ("flakyFailure", "flakyError") for child in case) for case in testcases) raw_retries += sum(child.tag in ("flakyFailure", "flakyError") for case in testcases for child in case) - selected = sums["tests"] - failed = sums["failures"] + sums["errors"] + selected = cases + failed = sum(sums[key] for key in ("failures", "errors")) skipped = sums["skipped"] passed = selected - failed - skipped retried = retried_cases diff --git a/scripts/test_emit_evidence.py b/scripts/test_emit_evidence.py index 72c5214..57b0fc1 100644 --- a/scripts/test_emit_evidence.py +++ b/scripts/test_emit_evidence.py @@ -40,6 +40,16 @@ def test_fixture_version_is_bound_to_target_when_supplied(self): self.assertEqual(record["target"]["fixtureVersion"], "v0.1.0") self.assertEqual(record["target"]["identity"], "quality-reference-application") + def test_repeated_testng_invocations_count_as_separate_testcases(self): + (self.report / "TEST-suite.xml").write_text( + '' + '' + '') + result = build_record(self.env, self.report, self.root / "repeated.json") + counts = result["execution"]["counts"] + self.assertEqual((counts["selected"], counts["executed"], counts["passed"]), (2, 2, 2)) + self.assertEqual(result["execution"]["integrity"], "complete") + def test_missing_and_inconsistent_reports_never_pass(self): self.assertEqual(build_record(self.env, self.report, self.root / "none.json")["execution"]["integrity"], "unavailable") (self.report / "TEST-suite.xml").write_text('') diff --git a/scripts/test_shared_app_controls.py b/scripts/test_shared_app_controls.py new file mode 100644 index 0000000..83871f2 --- /dev/null +++ b/scripts/test_shared_app_controls.py @@ -0,0 +1,62 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from verify_shared_app_controls import verify + + +class SharedAppControlsTests(unittest.TestCase): + def setUp(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.directory = Path(self.temporary_directory.name) + self.records = [self.record("1", "a"), self.record("2", "b")] + self.write_records() + + def tearDown(self): + self.temporary_directory.cleanup() + + @staticmethod + def record(invocation, prefix): + return { + "scenarioId": invocation, + "memberASessionSha256": prefix * 64, + "memberBSessionSha256": ("c" if prefix == "a" else "d") * 64, + "claimIdSha256": ("e" if prefix == "a" else "f") * 64, + "memberACleaned": True, + "memberBCleaned": True, + } + + def write_records(self): + for index, record in enumerate(self.records, start=1): + (self.directory / f"control-{index}.json").write_text( + json.dumps(record), encoding="utf-8" + ) + + def test_accepts_two_fresh_invocations_with_four_clean_sessions(self): + self.assertEqual( + verify(self.directory), + {"invocations": 2, "uniqueSessions": 4, "cleanedSessions": 4, "uniqueClaims": 2}, + ) + + def test_rejects_reused_session_or_claim_identity(self): + self.records[1]["memberASessionSha256"] = self.records[0]["memberASessionSha256"] + self.records[1]["claimIdSha256"] = self.records[0]["claimIdSha256"] + self.write_records() + + with self.assertRaisesRegex(ValueError, "four distinct remote sessions"): + verify(self.directory) + + def test_rejects_missing_invocation_or_failed_cleanup(self): + (self.directory / "control-2.json").unlink() + with self.assertRaisesRegex(ValueError, "expected two fresh"): + verify(self.directory) + + self.records[1]["memberACleaned"] = False + self.write_records() + with self.assertRaisesRegex(ValueError, "cleanup did not pass"): + verify(self.directory) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify_shared_app_controls.py b/scripts/verify_shared_app_controls.py new file mode 100644 index 0000000..de7babe --- /dev/null +++ b/scripts/verify_shared_app_controls.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Validate the two sanitized shared-app session-control records.""" + +import json +import re +import sys +from pathlib import Path + +HASH_PATTERN = re.compile(r"^[0-9a-f]{64}$") + + +def verify(directory): + records = [] + for path in sorted(Path(directory).glob("control-*.json")): + record = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(record, dict): + raise ValueError(f"control record must be a JSON object in {path.name}") + required = { + "scenarioId", + "memberASessionSha256", + "memberBSessionSha256", + "claimIdSha256", + "memberACleaned", + "memberBCleaned", + } + if set(record) != required: + raise ValueError(f"unexpected control record fields in {path.name}") + if not isinstance(record["scenarioId"], str) or not record["scenarioId"]: + raise ValueError(f"missing scenario identity in {path.name}") + for field in ("memberASessionSha256", "memberBSessionSha256", "claimIdSha256"): + if not isinstance(record[field], str) or not HASH_PATTERN.fullmatch(record[field]): + raise ValueError(f"invalid sanitized hash in {path.name}: {field}") + if record["memberACleaned"] is not True or record["memberBCleaned"] is not True: + raise ValueError(f"driver cleanup did not pass in {path.name}") + records.append(record) + + if len(records) != 2: + raise ValueError(f"expected two fresh scenario invocations, found {len(records)}") + if len({record["scenarioId"] for record in records}) != 2: + raise ValueError("scenario invocation identities were reused") + + session_hashes = [ + record[field] + for record in records + for field in ("memberASessionSha256", "memberBSessionSha256") + ] + if len(set(session_hashes)) != 4: + raise ValueError("expected four distinct remote sessions across two invocations") + + claim_hashes = [record["claimIdSha256"] for record in records] + if len(set(claim_hashes)) != 2: + raise ValueError("claim identity was reused across fresh invocations") + + return {"invocations": 2, "uniqueSessions": 4, "cleanedSessions": 4, "uniqueClaims": 2} + + +if __name__ == "__main__": + try: + summary = verify(sys.argv[1] if len(sys.argv) > 1 else "target/shared-app-controls") + print( + "shared-app control passed: " + f"{summary['invocations']} invocations, " + f"{summary['uniqueSessions']} unique sessions, " + f"{summary['cleanedSessions']} cleanups, " + f"{summary['uniqueClaims']} unique claims" + ) + except (OSError, json.JSONDecodeError, ValueError) as failure: + print(f"shared-app control failed: {failure}", file=sys.stderr) + sys.exit(1) diff --git a/src/test/java/io/github/selenium/saucedemo/tests/SharedAppCoordination.java b/src/test/java/io/github/selenium/saucedemo/tests/SharedAppCoordination.java new file mode 100644 index 0000000..850d00e --- /dev/null +++ b/src/test/java/io/github/selenium/saucedemo/tests/SharedAppCoordination.java @@ -0,0 +1,56 @@ +package io.github.selenium.saucedemo.tests; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** Per-invocation handoff helpers for the shared-app two-session scenario. */ +final class SharedAppCoordination { + + private SharedAppCoordination() {} + + @FunctionalInterface + interface CheckedSupplier { + T get() throws Exception; + } + + static T produce(CompletableFuture signal, CheckedSupplier producer) throws Exception { + try { + T value = producer.get(); + signal.complete(value); + return value; + } catch (Exception | Error failure) { + signal.completeExceptionally(failure); + throw failure; + } + } + + static T await(Future future, Duration timeout) throws Exception { + try { + return future.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (ExecutionException executionFailure) { + Throwable cause = executionFailure.getCause(); + if (cause instanceof Exception exception) { + throw exception; + } + if (cause instanceof Error error) { + throw error; + } + throw new IllegalStateException("Shared-app worker failed without a cause", cause); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw interrupted; + } catch (TimeoutException timeoutFailure) { + throw timeoutFailure; + } + } + + static void completeExceptionallyIfPending(CompletableFuture future, Throwable failure) { + if (!future.isDone()) { + future.completeExceptionally(failure); + } + } +} diff --git a/src/test/java/io/github/selenium/saucedemo/tests/SharedAppCoordinationTests.java b/src/test/java/io/github/selenium/saucedemo/tests/SharedAppCoordinationTests.java new file mode 100644 index 0000000..82efb11 --- /dev/null +++ b/src/test/java/io/github/selenium/saucedemo/tests/SharedAppCoordinationTests.java @@ -0,0 +1,135 @@ +package io.github.selenium.saucedemo.tests; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Duration; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.testng.annotations.Test; + +public class SharedAppCoordinationTests { + + @Test + public void memberBSignalsReadyAndReceivesTheClaimCreatedByMemberA() throws Exception { + CompletableFuture memberBReady = new CompletableFuture<>(); + CompletableFuture claimIdForMemberB = new CompletableFuture<>(); + + String sessionHash = SharedAppCoordination.produce(memberBReady, () -> "member-b-session"); + claimIdForMemberB.complete("new-claim-id"); + + assertThat(SharedAppCoordination.await(memberBReady, Duration.ofSeconds(1))) + .isEqualTo(sessionHash); + assertThat(SharedAppCoordination.await(claimIdForMemberB, Duration.ofSeconds(1))) + .isEqualTo("new-claim-id"); + } + + @Test + public void producerFailureReachesTheWaitingConsumerWithoutBeingWrapped() { + CompletableFuture memberBReady = new CompletableFuture<>(); + IllegalStateException failure = new IllegalStateException("controlled producer failure"); + + assertThatThrownBy( + () -> + SharedAppCoordination.produce( + memberBReady, + () -> { + throw failure; + })) + .isSameAs(failure); + assertThatThrownBy(() -> SharedAppCoordination.await(memberBReady, Duration.ofSeconds(2))) + .isSameAs(failure); + } + + @Test + public void memberBInitializationFailureFailsBothHandoffsImmediately() { + CompletableFuture memberBReady = new CompletableFuture<>(); + CompletableFuture claimIdForMemberB = new CompletableFuture<>(); + IllegalStateException initializationFailure = + new IllegalStateException("controlled member B initialization failure"); + + try { + SharedAppCoordination.produce( + memberBReady, + () -> { + throw initializationFailure; + }); + } catch (Exception failure) { + SharedAppCoordination.completeExceptionallyIfPending(claimIdForMemberB, failure); + } + + assertThatThrownBy(() -> SharedAppCoordination.await(memberBReady, Duration.ofSeconds(2))) + .isSameAs(initializationFailure); + assertThatThrownBy(() -> SharedAppCoordination.await(claimIdForMemberB, Duration.ofSeconds(2))) + .isSameAs(initializationFailure); + } + + @Test + @SuppressWarnings("PMD.CloseResource") + public void cancellationStopsTheWorkerAndCompletesUnresolvedHandoffs() throws Exception { + CompletableFuture memberBReady = new CompletableFuture<>(); + CompletableFuture claimIdForMemberB = new CompletableFuture<>(); + CancellationException cancellation = new CancellationException("controlled scenario close"); + CountDownLatch workerStarted = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + Future worker = + executor.submit( + () -> { + workerStarted.countDown(); + try { + SharedAppCoordination.await(claimIdForMemberB, Duration.ofSeconds(20)); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } catch (Exception failure) { + throw new IllegalStateException(failure); + } + }); + + try { + assertThat(workerStarted.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(worker.cancel(true)).isTrue(); + } finally { + SharedAppCoordination.completeExceptionallyIfPending(memberBReady, cancellation); + SharedAppCoordination.completeExceptionallyIfPending(claimIdForMemberB, cancellation); + worker.cancel(true); + executor.shutdownNow(); + assertThat(executor.awaitTermination(2, TimeUnit.SECONDS)).isTrue(); + } + assertThat(worker.isCancelled()).isTrue(); + + assertThatThrownBy(() -> SharedAppCoordination.await(memberBReady, Duration.ofSeconds(2))) + .isSameAs(cancellation); + assertThatThrownBy(() -> SharedAppCoordination.await(claimIdForMemberB, Duration.ofSeconds(2))) + .isSameAs(cancellation); + } + + @Test + public void eachScenarioInvocationGetsFreshHandoffs() throws Exception { + CompletableFuture firstReady = new CompletableFuture<>(); + CompletableFuture firstClaim = new CompletableFuture<>(); + CompletableFuture secondReady = new CompletableFuture<>(); + CompletableFuture secondClaim = new CompletableFuture<>(); + + SharedAppCoordination.produce(firstReady, () -> "first-session"); + firstClaim.complete("first-claim"); + + assertThat(secondReady).isNotSameAs(firstReady).isNotDone(); + assertThat(secondClaim).isNotSameAs(firstClaim).isNotDone(); + SharedAppCoordination.produce(secondReady, () -> "second-session"); + secondClaim.complete("second-claim"); + + assertThat(SharedAppCoordination.await(firstReady, Duration.ofSeconds(1))) + .isEqualTo("first-session"); + assertThat(SharedAppCoordination.await(firstClaim, Duration.ofSeconds(1))) + .isEqualTo("first-claim"); + assertThat(SharedAppCoordination.await(secondReady, Duration.ofSeconds(1))) + .isEqualTo("second-session"); + assertThat(SharedAppCoordination.await(secondClaim, Duration.ofSeconds(1))) + .isEqualTo("second-claim"); + } +} diff --git a/src/test/java/io/github/selenium/saucedemo/tests/SharedReferenceApplicationTests.java b/src/test/java/io/github/selenium/saucedemo/tests/SharedReferenceApplicationTests.java index 85e152e..223f672 100644 --- a/src/test/java/io/github/selenium/saucedemo/tests/SharedReferenceApplicationTests.java +++ b/src/test/java/io/github/selenium/saucedemo/tests/SharedReferenceApplicationTests.java @@ -1,100 +1,296 @@ package io.github.selenium.saucedemo.tests; +import io.github.selenium.saucedemo.framework.driver.WebDriverFactory; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.Duration; -import java.util.concurrent.CountDownLatch; +import java.util.HexFormat; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import org.openqa.selenium.By; +import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; +import org.openqa.selenium.remote.RemoteWebDriver; +import org.openqa.selenium.remote.SessionId; import org.openqa.selenium.support.ui.ExpectedConditions; import org.openqa.selenium.support.ui.Select; import org.openqa.selenium.support.ui.WebDriverWait; import org.testng.Assert; +import org.testng.ITestResult; +import org.testng.Reporter; +import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; -/** Small Grid-backed smoke of the shared fictional claims application. */ +/** One complete two-session review journey against the isolated fictional claims application. */ public class SharedReferenceApplicationTests extends BaseTestCase { - private static final CountDownLatch claimSubmitted = new CountDownLatch(1); - private static final AtomicReference memberAClaimId = new AtomicReference<>(); + private static final String EVIDENCE_ATTRIBUTE = "sharedAppScenarioEvidence"; + private static final Duration BROWSER_WAIT = Duration.ofSeconds(20); + private static final Duration HANDOFF_WAIT = Duration.ofSeconds(25); + private static final Duration WORKER_SHUTDOWN_WAIT = Duration.ofSeconds(5); - @Test(description = "member claim can be reviewed while another member remains isolated") - public void memberAClaimCanBeReviewed() { - var wait = new WebDriverWait(getDriver(), Duration.ofSeconds(30)); - getDriver().get(referenceAppUrl()); - startSession(wait, "member-a"); + @Test( + invocationCount = 2, + description = "member claim review stays isolated across two fresh scenario invocations") + @SuppressWarnings("PMD.CloseResource") + public void memberClaimCanBeReviewedWhileAnotherMemberRemainsIsolated() throws Exception { + WebDriver reviewer = getDriver(); + CompletableFuture memberBReady = new CompletableFuture<>(); + CompletableFuture claimIdForMemberB = new CompletableFuture<>(); + ExecutorService memberBExecutor = + Executors.newSingleThreadExecutor( + task -> new Thread(task, "shared-app-member-b-" + UUID.randomUUID())); + Future memberBTask = + memberBExecutor.submit(() -> observeAsMemberB(memberBReady, claimIdForMemberB)); - wait.until(ExpectedConditions.visibilityOfElementLocated(By.id("claim-form-section"))); - new Select(getDriver().findElement(By.id("policy"))).selectByIndex(0); - getDriver().findElement(By.id("amount")).sendKeys("123.45"); - new Select(getDriver().findElement(By.id("category"))).selectByValue("auto"); - getDriver().findElement(By.cssSelector("#claim-form button[type='submit']")).click(); + try { + reviewer.get(referenceAppUrl()); + startSession(reviewer, "member-a"); + String memberASessionHash = sessionIdentityHash(reviewer); + + String memberBSessionHash = SharedAppCoordination.await(memberBReady, HANDOFF_WAIT); + Assert.assertNotEquals( + memberASessionHash, + memberBSessionHash, + "member A and member B must own different remote sessions"); + + WebDriverWait reviewerWait = new WebDriverWait(reviewer, BROWSER_WAIT); + Set existingClaimIds = visibleClaimIds(reviewer, reviewerWait); + String claimId = submitNewClaim(reviewer, reviewerWait, existingClaimIds); + Assert.assertTrue( + claimIdForMemberB.complete(claimId), "the new claim ID should be delivered exactly once"); + + reviewClaimAsReviewer(reviewer, reviewerWait, claimId); + MemberBObservation memberB = SharedAppCoordination.await(memberBTask, HANDOFF_WAIT); + Assert.assertEquals(memberB.sessionIdentityHash(), memberBSessionHash); + Assert.assertEquals(memberB.claimIdentityHash(), sha256(claimId)); + Assert.assertTrue(memberB.cleaned(), "member B must clean up its own driver thread"); + + Reporter.getCurrentTestResult() + .setAttribute( + EVIDENCE_ATTRIBUTE, + new ScenarioEvidence( + UUID.randomUUID().toString(), + memberASessionHash, + memberBSessionHash, + sha256(claimId), + memberB.cleaned())); + } catch (Exception | Error failure) { + SharedAppCoordination.completeExceptionallyIfPending(memberBReady, failure); + SharedAppCoordination.completeExceptionallyIfPending(claimIdForMemberB, failure); + throw failure; + } finally { + CancellationException cancellation = + new CancellationException("shared-app scenario is closing"); + SharedAppCoordination.completeExceptionallyIfPending(memberBReady, cancellation); + SharedAppCoordination.completeExceptionallyIfPending(claimIdForMemberB, cancellation); + if (!memberBTask.isDone()) { + memberBTask.cancel(true); + } + memberBExecutor.shutdownNow(); + if (!memberBExecutor.awaitTermination(WORKER_SHUTDOWN_WAIT.toSeconds(), TimeUnit.SECONDS)) { + throw new IllegalStateException("member B worker did not stop within five seconds"); + } + } + } + + @Override + @AfterMethod(alwaysRun = true) + public void afterMethod(ITestResult result) { + try { + super.afterMethod(result); + } finally { + boolean memberACleaned = WebDriverFactory.getThreadLocalWebDriver() == null; + Object evidence = result.getAttribute(EVIDENCE_ATTRIBUTE); + if (evidence instanceof ScenarioEvidence scenario) { + writeSanitizedEvidence(scenario, memberACleaned); + Assert.assertTrue(memberACleaned, "BaseTestCase must clean up member A's driver"); + } + } + } + + private MemberBObservation observeAsMemberB( + CompletableFuture memberBReady, CompletableFuture claimIdForMemberB) + throws Exception { + MemberBObservation observation; + try { + String memberBSessionHash = + SharedAppCoordination.produce( + memberBReady, + () -> { + WebDriverFactory.initThreadLocalDriver(); + WebDriver memberB = WebDriverFactory.getThreadLocalWebDriver(); + Assert.assertNotNull(memberB, "member B must initialize its own driver"); + memberB.get(referenceAppUrl()); + startSession(memberB, "member-b"); + return sessionIdentityHash(memberB); + }); + + String claimId = SharedAppCoordination.await(claimIdForMemberB, HANDOFF_WAIT); + WebDriver memberB = WebDriverFactory.getThreadLocalWebDriver(); + Assert.assertNotNull(memberB, "member B must keep using its own session"); + memberB.navigate().refresh(); + startSession(memberB, "member-b"); + WebDriverWait memberBWait = new WebDriverWait(memberB, BROWSER_WAIT); + Set visibleClaimsAfterRefresh = visibleClaimIds(memberB, memberBWait); + Assert.assertFalse( + visibleClaimsAfterRefresh.contains(claimId), + "member B must not see the exact claim created by member A after refresh"); + observation = new MemberBObservation(memberBSessionHash, sha256(claimId), false); + } catch (Exception | Error failure) { + SharedAppCoordination.completeExceptionallyIfPending(memberBReady, failure); + SharedAppCoordination.completeExceptionallyIfPending(claimIdForMemberB, failure); + throw failure; + } finally { + WebDriverFactory.cleanUpDriver(); + } + + boolean memberBCleaned = WebDriverFactory.getThreadLocalWebDriver() == null; + if (!memberBCleaned) { + throw new AssertionError("member B thread still owns a WebDriver after cleanup"); + } + return new MemberBObservation( + observation.sessionIdentityHash(), observation.claimIdentityHash(), memberBCleaned); + } + + private static void startSession(WebDriver driver, String identity) { + new Select(driver.findElement(By.id("identity"))).selectByValue(identity); + driver.findElement(By.cssSelector("#session-form button[type='submit']")).click(); + new WebDriverWait(driver, BROWSER_WAIT) + .until( + ExpectedConditions.textToBePresentInElementLocated( + By.id("notice"), "Data is scoped to this application run")); + } + + private static Set visibleClaimIds(WebDriver driver, WebDriverWait wait) { + wait.until(ExpectedConditions.visibilityOfElementLocated(By.id("claims-section"))); + return driver.findElements(By.cssSelector("#claims-body tr")).stream() + .map(SharedReferenceApplicationTests::claimId) + .collect(Collectors.toUnmodifiableSet()); + } + + private static String submitNewClaim( + WebDriver driver, WebDriverWait wait, Set existingClaimIds) { + new Select(driver.findElement(By.id("policy"))).selectByIndex(0); + driver.findElement(By.id("amount")).sendKeys("123.45"); + new Select(driver.findElement(By.id("category"))).selectByValue("auto"); + driver.findElement(By.cssSelector("#claim-form button[type='submit']")).click(); wait.until( ExpectedConditions.textToBePresentInElementLocated(By.id("notice"), "claim was submitted")); - WebElement claimRow = + String claimId = wait.until( - driver -> { - var rows = driver.findElements(By.cssSelector("#claims-body tr")); - return rows.isEmpty() ? null : rows.get(0); - }); - String claimId = claimRow.findElement(By.cssSelector("td:first-child")).getText(); + currentDriver -> + currentDriver.findElements(By.cssSelector("#claims-body tr")).stream() + .map(SharedReferenceApplicationTests::claimId) + .filter(candidate -> !existingClaimIds.contains(candidate)) + .findFirst() + .orElse(null)); Assert.assertTrue( - claimId.matches("[0-9a-f-]{36}"), "the UI should expose the persisted claim UUID"); - memberAClaimId.set(claimId); - claimSubmitted.countDown(); - - getDriver().findElement(By.id("sign-out")).click(); - startSession(wait, "reviewer"); - WebElement reviewerRow = wait.until(driver -> rowWithStatus(claimId, "submitted")); - reviewerRow.findElement(By.xpath(".//button[normalize-space()='Set under review']")).click(); + claimId.matches("[0-9a-f-]{36}"), "the UI should expose a persisted claim UUID"); + return claimId; + } + + private static void reviewClaimAsReviewer(WebDriver driver, WebDriverWait wait, String claimId) { + driver.findElement(By.id("sign-out")).click(); + startSession(driver, "reviewer"); + WebElement claimRow = + wait.until(currentDriver -> rowWithStatus(currentDriver, claimId, "submitted")); + claimRow.findElement(By.xpath(".//button[normalize-space()='Set under review']")).click(); wait.until( ExpectedConditions.textToBePresentInElementLocated( By.id("notice"), "moved to under review")); - reviewerRow = wait.until(driver -> rowWithStatus(claimId, "under review")); - reviewerRow.findElement(By.xpath(".//button[normalize-space()='Set approved']")).click(); + claimRow = wait.until(currentDriver -> rowWithStatus(currentDriver, claimId, "under review")); + claimRow.findElement(By.xpath(".//button[normalize-space()='Set approved']")).click(); wait.until( ExpectedConditions.textToBePresentInElementLocated(By.id("notice"), "moved to approved")); - Assert.assertNotNull(wait.until(driver -> rowWithStatus(claimId, "approved"))); + Assert.assertNotNull( + wait.until(currentDriver -> rowWithStatus(currentDriver, claimId, "approved"))); } - @Test(description = "member B cannot see member A's claim in a separate Grid session") - public void memberBDoesNotSeeMemberAClaim() throws InterruptedException { - Assert.assertTrue( - claimSubmitted.await(120, TimeUnit.SECONDS), "member A must submit a claim first"); - var wait = new WebDriverWait(getDriver(), Duration.ofSeconds(30)); - getDriver().get(referenceAppUrl()); - startSession(wait, "member-b"); - wait.until(ExpectedConditions.visibilityOfElementLocated(By.id("claims-section"))); - Assert.assertEquals(getDriver().findElements(By.cssSelector("#claims-body tr")).size(), 0); - Assert.assertFalse( - getDriver().findElement(By.id("claims-body")).getText().contains(memberAClaimId.get()), - "cross-member data must remain hidden in the UI"); + private static WebElement rowWithStatus(WebDriver driver, String claimId, String status) { + return driver.findElements(By.cssSelector("#claims-body tr")).stream() + .filter(row -> claimId(row).equals(claimId)) + .filter(row -> row.getText().contains(status)) + .findFirst() + .orElse(null); } - private void startSession(WebDriverWait wait, String identity) { - new Select(getDriver().findElement(By.id("identity"))).selectByValue(identity); - getDriver().findElement(By.cssSelector("#session-form button[type='submit']")).click(); - wait.until( - ExpectedConditions.textToBePresentInElementLocated( - By.id("notice"), "Data is scoped to this application run")); + private static String claimId(WebElement row) { + return row.findElement(By.cssSelector("td:first-child")).getText(); } - private WebElement rowForClaim(String claimId) { - return getDriver().findElements(By.cssSelector("#claims-body tr")).stream() - .filter(row -> row.findElement(By.cssSelector("td:first-child")).getText().equals(claimId)) - .findFirst() - .orElse(null); + private static String sessionIdentityHash(WebDriver driver) { + if (!(driver instanceof RemoteWebDriver remoteDriver)) { + throw new AssertionError("shared-app experiment requires an identifiable remote session"); + } + SessionId sessionId = remoteDriver.getSessionId(); + if (sessionId == null) { + throw new AssertionError("shared-app experiment requires an identifiable remote session"); + } + return sha256(sessionId.toString()); } - private WebElement rowWithStatus(String claimId, String status) { - WebElement row = rowForClaim(claimId); - return row != null && row.getText().contains(status) ? row : null; + private static String sha256(String value) { + try { + byte[] digest = + MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(digest); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } } - private String referenceAppUrl() { + private static String referenceAppUrl() { String url = System.getProperty("app.url", ""); Assert.assertFalse(url.isBlank(), "pass -Dapp.url for the shared reference target"); return url.endsWith("/") ? url : url + "/"; } + + private static void writeSanitizedEvidence(ScenarioEvidence evidence, boolean memberACleaned) { + Path outputDirectory = Path.of("target", "shared-app-controls"); + Path output = outputDirectory.resolve("control-" + evidence.scenarioId() + ".json"); + String json = + "{\"scenarioId\":\"" + + evidence.scenarioId() + + "\",\"memberASessionSha256\":\"" + + evidence.memberASessionHash() + + "\",\"memberBSessionSha256\":\"" + + evidence.memberBSessionHash() + + "\",\"claimIdSha256\":\"" + + evidence.claimIdentityHash() + + "\",\"memberACleaned\":" + + memberACleaned + + ",\"memberBCleaned\":" + + evidence.memberBCleaned() + + "}\n"; + try { + Files.createDirectories(outputDirectory); + Files.writeString(output, json, StandardCharsets.UTF_8); + } catch (IOException failure) { + throw new IllegalStateException( + "Unable to write sanitized shared-app control record", failure); + } + } + + private record MemberBObservation( + String sessionIdentityHash, String claimIdentityHash, boolean cleaned) {} + + private record ScenarioEvidence( + String scenarioId, + String memberASessionHash, + String memberBSessionHash, + String claimIdentityHash, + boolean memberBCleaned) {} } diff --git a/testng-framework-unit.xml b/testng-framework-unit.xml index cc52258..8b61abf 100644 --- a/testng-framework-unit.xml +++ b/testng-framework-unit.xml @@ -8,6 +8,7 @@ + diff --git a/testng-shared-app-unit.xml b/testng-shared-app-unit.xml new file mode 100644 index 0000000..49f945c --- /dev/null +++ b/testng-shared-app-unit.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/testng-shared-app.xml b/testng-shared-app.xml index f2f8817..2c2d971 100644 --- a/testng-shared-app.xml +++ b/testng-shared-app.xml @@ -1,5 +1,5 @@ - +