diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e5d2c1460..f97654c2d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -420,11 +420,11 @@ jobs: client/java/opensysml-client/src/main/java/org/openmbee/opensysml/proto # The race-enabled suite. It is the longest single step in the pipeline, so - # it gets the larger class and four executors, one named package shard each. + # it gets the larger class and five executors, one named package shard each. go-race-test: executor: go-executor resource_class: large - parallelism: 4 + parallelism: 5 steps: - checkout - go/load-cache @@ -444,7 +444,7 @@ jobs: name: Run Go race tests command: | set -o pipefail - shards=(runtime model export rest) + shards=(runtime model export rest migrate) if [ "$CIRCLE_NODE_TOTAL" != "${#shards[@]}" ]; then echo "error: parallelism is $CIRCLE_NODE_TOTAL, but scripts/race-shard.sh has ${#shards[@]} shards" >&2 exit 1 @@ -459,7 +459,10 @@ jobs: go test -v -race -pgo=off -timeout 30m ./tests/wasm go test -C tools -v -race -pgo=off -timeout 30m ./... else - go test -v -race -pgo=off -timeout 30m $pkgs + # tests/migrate alone runs 23-32 minutes under -race on this class, so its + # shard gets the Makefile's per-package timeout (see make test-shard). + timeout=30m; [ "$shard" = migrate ] && timeout=55m + go test -v -race -pgo=off -timeout "$timeout" $pkgs fi no_output_timeout: 30m environment: @@ -483,11 +486,11 @@ jobs: # A second, race-free pass, because the profile the scan reads is written # with -coverpkg, which is too slow to instrument under -race. Serially the # instrumented suite overran the 60-minute job limit even on large, so it runs - # as the race test's four package shards, and go-coverage-merge joins them. + # as the race test's five package shards, and go-coverage-merge joins them. go-coverage: executor: go-executor resource_class: large - parallelism: 4 + parallelism: 5 steps: - checkout - go/load-cache @@ -501,7 +504,7 @@ jobs: - run: name: Write this shard's coverage profile command: | - shards=(runtime model export rest) + shards=(runtime model export rest migrate) if [ "$CIRCLE_NODE_TOTAL" != "${#shards[@]}" ]; then echo "error: parallelism is $CIRCLE_NODE_TOTAL, but scripts/race-shard.sh has ${#shards[@]} shards" >&2 exit 1 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 9006ca33ba..4fa12dfeb1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -83,12 +83,12 @@ jobs: name: Go race tests (${{ matrix.shard }}) needs: changes if: needs.changes.outputs.go == 'true' - # The shards split the suite's slowest packages across runners so each gets its own job cap (see scripts/race-shard.sh). - # internal/exec/runtime alone outruns a job under -race, so it is two shards split by test name (see make test-shard). + # The shards split the suite's slowest packages across runners so each gets its own job cap (see scripts/race-shard.sh); + # internal/exec/runtime is two shards split by test name (see make test-shard) and tests/migrate, the next-longest, is one of its own. strategy: fail-fast: false matrix: - shard: [runtime, runtime-corpus, model, export, rest] + shard: [runtime, runtime-corpus, model, export, rest, migrate] runs-on: ubuntu-latest permissions: contents: read @@ -297,7 +297,7 @@ jobs: set -euo pipefail profiles=(shards/coverage-*/coverage.txt) # One per race-tests shard. - if [ "${#profiles[@]}" -ne 5 ]; then echo "error: expected 5 shard profiles, found ${#profiles[@]}" >&2; exit 1; fi + if [ "${#profiles[@]}" -ne 6 ]; then echo "error: expected 6 shard profiles, found ${#profiles[@]}" >&2; exit 1; fi for f in "${profiles[@]}"; do [ "$(head -n 1 "$f")" = "mode: atomic" ] || { echo "error: $f is not an atomic-mode profile" >&2; exit 1; } done diff --git a/AGENTS.md b/AGENTS.md index ba4f8c48be..6d6d88c14c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,7 @@ make build # build bin/sysml and bin/sysml-lsp (with version ldflags) make build-sysml # REPL binary only make build-lsp # LSP binary only make test # full suite: go test -race -coverprofile ... ./... -make test-shard SHARD=runtime # one CI shard of the race suite (runtime|model|export|rest) +make test-shard SHARD=runtime # one CI shard of the race suite (runtime|runtime-corpus|model|export|rest|migrate) make test-short # faster, no race detector make clean # remove build artifacts ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index fcf9e37a6e..319b02354b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -444,6 +444,7 @@ release is described in [docs/project/releasing.md](docs/project/releasing.md). - The non-normative `OpenSysML Libraries/` extension libraries are now maintained upstream at [Open-MBEE/OpenSysML-Extensions-Library](https://github.com/Open-MBEE/OpenSysML-Extensions-Library); the bundled copy is pinned by `scripts/extension-libraries-pin.sh`, refreshed by `scripts/sync-extension-libraries.sh` (`make extension-libraries-sync`), checked for drift on every pull request (`make extension-libraries-check`), and carries `engine-contract.json` recording the qualified names the engine binds. - **Declare the bundled OpenSysML extension libraries as non-standard.** `DiagramLayout`, `IdentityMetadata`, `MigrationMetadata` and `SysMLValidation` were written `standard library package`, which both asserted membership of the normative Kernel/SysML libraries and drew a pilot warning (`User library packages should not be marked as standard`); they are project extensions and now read `library package`. RDF export no longer marks them `sysml:isStandard true`. +- **`tests/migrate` is a race shard of its own.** The SysML v1 migrator's suite ran as part of the `rest` shard, where under `-race` it grew past 24 minutes on the pull-request runners and past the 30-minute package timeout on CircleCI's `develop` builds, failing every merge. `scripts/race-shard.sh migrate` names it, `make test-shard SHARD=migrate` and `make coverage-shard SHARD=migrate` run it, and both CI configurations run six race shards and five coverage shards. ### Deprecated diff --git a/Makefile b/Makefile index 528424f641..c6e1878859 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ TOOLS_DIR := tools COVERAGE_PACKAGES ?= ./... COVERAGE_TOOLS ?= ./... COVERAGE_PROFILE ?= coverage.txt -COVERAGE_SHARDS := runtime model export rest +COVERAGE_SHARDS := runtime model export rest migrate # The commands whose manual pages are generated and shipped, in section 1. COMMANDS := sysml sysml-lsp sysml-grpc sysml-jupyter-kernel @@ -288,7 +288,7 @@ RACE_SHARD_SKIP := ^(TestTrainingExamplesSemanticErrors|TestCorpusGatesCacheStat RACE_RUNTIME_CORPUS := ^Test(ExploreWithIsExploreOverTheConformanceCorpus|CheckAgreesWithExploreOverTheConformanceCorpus|CheckWitnessesReplayOverTheConformanceCorpus|ExecutionConformance|ExecutionConformanceUnderPolicies)$$ RACE_SHARD_TOOLS_SKIP := ^(TestSuiteRead|TestSuiteClassification|TestEmitSuite|TestSuiteClassificationReasons|TestSuiteLibraryCallsAreClassified|TestSuiteReadsControlAndObjectFlow|TestSuiteReadsClassifiers|TestSuiteReadsExceptionHandlers)$$ -test-shard: ## Run one CI shard of the race suite (SHARD=runtime|runtime-corpus|model|export|rest) +test-shard: ## Run one CI shard of the race suite (SHARD=runtime|runtime-corpus|model|export|rest|migrate) @echo "Running Go race tests, shard $(SHARD)..." @# 55m per package; its job's ceiling is 60. Whole, internal/exec/runtime took 31-57 minutes @# under -race on the CI runners, so it runs as two shards of about half that each. @@ -324,14 +324,14 @@ coverage: ## Write the coverage profile the SonarCloud scan reads python3 scripts/dedupe-coverage.py $(COVERAGE_PROFILE) @go tool cover -func=$(COVERAGE_PROFILE) | tail -n 1 -coverage-shard: ## Write one CI shard's coverage profile, coverage-$(SHARD).txt (SHARD=runtime|model|export|rest) +coverage-shard: ## Write one CI shard's coverage profile, coverage-$(SHARD).txt (SHARD=runtime|model|export|rest|migrate) @# The race shards' packages; tests/wasm and the tools, which no race shard runs, go with runtime. pkgs=$$(scripts/race-shard.sh $(SHARD)) && \ $(MAKE) --no-print-directory coverage COVERAGE_PROFILE=coverage-$(SHARD).txt \ COVERAGE_PACKAGES="$$(echo $$pkgs) $(if $(filter runtime,$(SHARD)),./tests/wasm)" \ COVERAGE_TOOLS="$(if $(filter runtime,$(SHARD)),./...)" -coverage-merge: ## Merge the shard profiles coverage-runtime/model/export/rest.txt into coverage.txt +coverage-merge: ## Merge the shard profiles coverage-runtime/model/export/rest/migrate.txt into coverage.txt { echo "mode: atomic"; for shard in $(COVERAGE_SHARDS); do tail -n +2 coverage-$$shard.txt || exit 1; done; } > coverage.txt python3 scripts/dedupe-coverage.py coverage.txt @go tool cover -func=coverage.txt | tail -n 1 diff --git a/scripts/race-shard.sh b/scripts/race-shard.sh index 390424faf8..61c2d3e2a5 100755 --- a/scripts/race-shard.sh +++ b/scripts/race-shard.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # Prints the root-module package paths assigned to one race-test shard. -# Usage: scripts/race-shard.sh runtime|runtime-corpus|model|export|rest +# Usage: scripts/race-shard.sh runtime|runtime-corpus|model|export|rest|migrate # runtime and runtime-corpus are one package, split by test name in make test-shard. set -euo pipefail cd "$(dirname "$0")/.." if [[ $# -ne 1 ]]; then - echo "usage: scripts/race-shard.sh runtime|runtime-corpus|model|export|rest" >&2 + echo "usage: scripts/race-shard.sh runtime|runtime-corpus|model|export|rest|migrate" >&2 exit 2 fi @@ -20,14 +20,17 @@ smt="$module/internal/exec/smt" export="$module/tests/export" model="$module/tests/model" passes="$module/internal/check/passes" +# tests/migrate runs the SysML v1 migrator over whole models and is the next-longest +# package under -race after the runtime, so it is a shard of its own. +migrate="$module/tests/migrate" # tests/wasm runs in the WebAssembly gate job, not a race shard. wasm="$module/tests/wasm" -named_packages=("$runtime" "$workspace_model" "$corpus" "$smt" "$export" "$model" "$passes" "$wasm") +named_packages=("$runtime" "$workspace_model" "$corpus" "$smt" "$export" "$model" "$passes" "$wasm" "$migrate") case "$shard" in - runtime|runtime-corpus|model|export|rest) ;; + runtime|runtime-corpus|model|export|rest|migrate) ;; *) - echo "usage: scripts/race-shard.sh runtime|runtime-corpus|model|export|rest" >&2 + echo "usage: scripts/race-shard.sh runtime|runtime-corpus|model|export|rest|migrate" >&2 exit 2 ;; esac @@ -50,6 +53,9 @@ case "$shard" in export) printf '%s\n' "$export" "$model" "$passes" ;; + migrate) + printf '%s\n' "$migrate" + ;; rest) while IFS= read -r package; do case " ${named_packages[*]} " in