From ba910ed2b05dc1589165f332f656edc44a29c491 Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Fri, 21 Aug 2026 17:11:35 -0700 Subject: [PATCH 1/2] testing 1 2 3 --- .github/workflows/ci.yml | 258 +++++++++++++-- crates/testing/src/sdk.rs | 485 ++++++++++++++++++++++++++++- sdks/rust/tests/test.rs | 31 +- tools/ci/commands/test/src/main.rs | 447 +++++++++++++++++++------- 4 files changed, 1065 insertions(+), 156 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2395eeb346..27aadc900ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -579,36 +579,20 @@ jobs: - name: Skip duplicate merge queue smoketest run: echo "Merge queue commit has the same tree as the PR head; smoketest already ran for the PR." - test: + test_build: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} - name: Test Suite + name: Prepare Test Suite runs-on: spacetimedb-new-runner-2 - + timeout-minutes: 30 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: - - name: Find Git ref - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - PR_NUMBER="${{ github.event.inputs.pr_number || null }}" - if test -n "${PR_NUMBER}"; then - GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" - else - GIT_REF="${{ github.ref }}" - fi - echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - - - name: Checkout sources - uses: actions/checkout@v4 - with: - ref: ${{ env.GIT_REF }} - + - *find-git-ref + - *checkout-sources - uses: dsherret/rust-toolchain-file@v1 - - name: Set default rust toolchain - run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) + - *set-default-rust-toolchain - *set-native-cache-keys - &restore-jemalloc name: Restore jemalloc @@ -630,14 +614,10 @@ jobs: with: &v8-debug-cache path: ${{ env.CARGO_TARGET_DIR }}/debug/gn_out/obj key: ${{ steps.native-cache-keys.outputs.v8-debug-key }} - - &restore-rusty-v8 - name: Restore rusty_v8 - uses: actions/cache/restore@v4 - with: *v8-release-cache - *restore-openssl - *configure-cached-openssl - - uses: actions/setup-dotnet@v3 + - uses: actions/setup-dotnet@v4 with: global-json-file: global.json @@ -650,8 +630,114 @@ jobs: with: run_install: true - # Install cmake and emscripten for C++ module compilation tests. - - name: Install cmake and emscripten + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Prepare core test suite + run: | + cargo ci test archive-core \ + --archive-file test-suite-nextest.tar.zst \ + --unstable-archive-file test-suite-unstable-nextest.tar.zst + + - name: Upload core test build + uses: actions/upload-artifact@v4 + with: + name: test-suite-build + path: | + test-suite-nextest.tar.zst + test-suite-unstable-nextest.tar.zst + if-no-files-found: error + overwrite: true + retention-days: 14 + + - name: Upload timing reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: cargo-timings-test-build + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ + if-no-files-found: warn + retention-days: 14 + + test_partitions: + needs: [test_build] + name: Test Suite (${{ matrix.partition }}/2) + strategy: + fail-fast: false + matrix: + partition: [1, 2] + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 30 + env: + RUST_BACKTRACE: full + PARTITION_COUNT: 2 + steps: + - *find-git-ref + - *checkout-sources + - uses: dsherret/rust-toolchain-file@v1 + - *set-default-rust-toolchain + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Download core test build + uses: actions/download-artifact@v4 + with: + name: test-suite-build + + - name: Run core test partition + run: | + cargo nextest run \ + --archive-file test-suite-nextest.tar.zst \ + --workspace-remap "${GITHUB_WORKSPACE}" \ + --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} \ + --no-fail-fast --no-tests pass -j 2 \ + -E 'not test(/unreal/)' + cargo nextest run \ + --archive-file test-suite-unstable-nextest.tar.zst \ + --workspace-remap "${GITHUB_WORKSPACE}" \ + --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} \ + --no-fail-fast --no-tests pass -j 2 + bash tools/check-diff.sh + + sdk_test_build: + needs: [merge_queue_noop, lints] + if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} + name: Prepare SDK Tests + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 30 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + steps: + - *find-git-ref + - *checkout-sources + - uses: dsherret/rust-toolchain-file@v1 + - *set-default-rust-toolchain + - *set-native-cache-keys + - *restore-jemalloc + - *configure-cached-jemalloc + - *restore-zstd + - *configure-cached-zstd + - *restore-rusty-v8-debug + - *restore-openssl + - *configure-cached-openssl + + - uses: actions/setup-dotnet@v4 + with: + global-json-file: global.json + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - uses: ./.github/actions/setup-pnpm + with: + run_install: true + + # SDK preparation compiles TypeScript, C#, and C++ test modules once. + - name: Install module toolchains run: | sudo apt-get update sudo apt-get install -y cmake @@ -684,20 +770,121 @@ jobs: wasm-bindgen --version - # Source emsdk environment to make emcc (Emscripten compiler) available in PATH. - - name: Run tests + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Prepare SDK tests run: | source ~/emsdk/emsdk_env.sh - cargo ci test + cargo ci test archive-sdk \ + --native-archive-file sdk-tests-native-nextest.tar.zst \ + --browser-archive-file sdk-tests-browser-nextest.tar.zst \ + --support-archive-file sdk-test-support.tar.gz + + - name: Upload SDK test build + uses: actions/upload-artifact@v4 + with: + name: sdk-test-build + path: | + sdk-tests-native-nextest.tar.zst + sdk-tests-browser-nextest.tar.zst + sdk-test-support.tar.gz + if-no-files-found: error + overwrite: true + retention-days: 14 - name: Upload timing reports if: always() uses: actions/upload-artifact@v4 with: - name: cargo-timings-test + name: cargo-timings-sdk-test-build path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ + if-no-files-found: warn retention-days: 14 + sdk_test_partitions: + needs: [sdk_test_build, upload-build-artifacts-linux] + name: SDK Tests (${{ matrix.partition }}/2) + strategy: + fail-fast: false + matrix: + partition: [1, 2] + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 30 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + CARGO_BUILD_PROFILE: release + RUST_BACKTRACE: full + PARTITION_COUNT: 2 + SPACETIME_SDK_TEST_MODE: prebuilt + SPACETIME_SDK_TEST_ARTIFACT_DIR: ${{ github.workspace }}/sdk-test-support + steps: + - *find-git-ref + - *checkout-sources + - uses: dsherret/rust-toolchain-file@v1 + - *set-default-rust-toolchain + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: build-artifacts-linux + path: build-artifacts + + - name: Extract build artifacts + run: | + tar -xzf build-artifacts/build-support.tar.gz + test -f target/release/spacetimedb-cli + test -f target/release/spacetimedb-standalone + + - name: Download SDK test build + uses: actions/download-artifact@v4 + with: + name: sdk-test-build + + - name: Extract SDK support files + run: tar -xzf sdk-test-support.tar.gz + + - name: Run SDK test partition + run: | + cargo nextest run \ + --archive-file sdk-tests-native-nextest.tar.zst \ + --workspace-remap "${GITHUB_WORKSPACE}" \ + --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} \ + --no-fail-fast --no-tests pass -j 2 \ + -E 'not test(/unreal/)' + cargo nextest run \ + --archive-file sdk-tests-browser-nextest.tar.zst \ + --workspace-remap "${GITHUB_WORKSPACE}" \ + --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} \ + --no-fail-fast --no-tests pass -j 2 \ + -E 'not test(/unreal/)' + + tests_noop: + needs: [merge_queue_noop] + if: ${{ needs.merge_queue_noop.outputs.skip == 'true' }} + name: ${{ matrix.name }} + strategy: + matrix: + name: + - Prepare Test Suite + - Test Suite (1/2) + - Test Suite (2/2) + - Prepare SDK Tests + - SDK Tests (1/2) + - SDK Tests (2/2) + runs-on: ubuntu-latest + steps: + - name: Skip duplicate merge queue tests + run: echo "Merge queue commit has the same tree as the PR head; tests already ran for the PR." + index_scan_bench: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} @@ -737,7 +924,10 @@ jobs: - *configure-cached-jemalloc - *restore-zstd - *configure-cached-zstd - - *restore-rusty-v8 + - &restore-rusty-v8 + name: Restore rusty_v8 + uses: actions/cache/restore@v4 + with: *v8-release-cache - *restore-openssl - *configure-cached-openssl diff --git a/crates/testing/src/sdk.rs b/crates/testing/src/sdk.rs index eb45353aa2b..beb2a142548 100644 --- a/crates/testing/src/sdk.rs +++ b/crates/testing/src/sdk.rs @@ -1,16 +1,88 @@ use duct::cmd; use rand::seq::IteratorRandom; +use serde::{Deserialize, Serialize}; use spacetimedb::messages::control_db::HostType; use spacetimedb_data_structures::map::HashMap; use spacetimedb_guard::SpacetimeDbGuard; use spacetimedb_paths::{RootDir, SpacetimePaths}; +use std::collections::BTreeMap; +use std::env; +use std::ffi::OsStr; +use std::fs; use std::fs::create_dir_all; +use std::path::{Path, PathBuf}; use std::sync::Mutex; use crate::invoke_cli; use crate::modules::{CompilationMode, CompiledModule}; use tempfile::TempDir; +pub const SDK_TEST_MODE_ENV: &str = "SPACETIME_SDK_TEST_MODE"; +pub const SDK_TEST_ARTIFACT_DIR_ENV: &str = "SPACETIME_SDK_TEST_ARTIFACT_DIR"; + +#[derive(Clone)] +pub enum PrebuiltClient { + NativeRust { + binary_name: String, + args: Vec, + }, + BrowserRust { + package_name: String, + run_selector: String, + }, + TypeScript { + package_name: String, + entrypoint: PathBuf, + args: Vec, + }, +} + +impl PrebuiltClient { + fn key(&self) -> String { + match self { + Self::NativeRust { binary_name, .. } => format!("native-{binary_name}"), + Self::BrowserRust { package_name, .. } => format!("browser-{package_name}"), + Self::TypeScript { package_name, .. } => format!("typescript-{package_name}"), + } + } +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum SdkTestMode { + Local, + Prepare, + Prebuilt, +} + +impl SdkTestMode { + fn from_env() -> Self { + match env::var(SDK_TEST_MODE_ENV).as_deref() { + Err(env::VarError::NotPresent) => Self::Local, + Ok("prepare") => Self::Prepare, + Ok("prebuilt") => Self::Prebuilt, + Ok(value) => panic!("invalid {SDK_TEST_MODE_ENV} value {value:?}; expected `prepare` or `prebuilt`"), + Err(err) => panic!("failed to read {SDK_TEST_MODE_ENV}: {err}"), + } + } +} + +#[derive(Default, Deserialize, Serialize)] +struct ArtifactManifest { + modules: BTreeMap, + clients: BTreeMap, +} + +#[derive(Clone, Deserialize, Serialize)] +struct ModuleArtifact { + path: PathBuf, + host_type: String, +} + +#[derive(Clone, Deserialize, Serialize)] +struct ClientArtifact { + path: PathBuf, +} + struct SdkTestPaths { paths: SpacetimePaths, _root: TempDir, @@ -69,6 +141,10 @@ pub struct Test { /// - `SPACETIME_SDK_TEST_DB_NAME` bound to the database identity or name. /// - `SPACETIME_SDK_TEST_SERVER_URL` bound to the server URL for this test. run_command: String, + + /// Describes the client output that CI prepares once and shares with nextest shards. + /// Local runs continue to use `compile_command` and `run_command`. + prebuilt_client: Option, } pub const TEST_MODULE_PROJECT_ENV_VAR: &str = "SPACETIME_SDK_TEST_MODULE_PROJECT"; @@ -88,6 +164,14 @@ impl Test { let sdk_paths = SdkTestPaths::new(); let paths = &sdk_paths.paths; + match SdkTestMode::from_env() { + SdkTestMode::Local => self.run_local(paths), + SdkTestMode::Prepare => self.prepare(paths), + SdkTestMode::Prebuilt => self.run_prebuilt(paths), + } + } + + fn run_local(&self, paths: &SpacetimePaths) { let (file, host_type) = compile_module(&self.module_name); generate_bindings( @@ -101,23 +185,297 @@ impl Test { ); compile_client(&self.compile_command, &self.client_project); + self.run_with_artifacts(paths, &file, host_type, &self.run_command); + } + fn prepare(&self, paths: &SpacetimePaths) { + let artifact_dir = sdk_artifact_dir(); + let (file, host_type) = prepare_module_artifact(&artifact_dir, &self.module_name); + + generate_bindings( + paths, + &self.generate_language, + &file, + host_type, + &self.client_project, + &self.generate_subdir, + self.generate_include_private, + ); + + let prebuilt_client = self.prebuilt_client.as_ref().unwrap_or_else(|| { + panic!( + "SDK test client {} has no prebuilt artifact description", + self.client_project + ) + }); + prepare_client_artifact( + &artifact_dir, + prebuilt_client, + &self.compile_command, + &self.client_project, + ); + } + + fn run_prebuilt(&self, paths: &SpacetimePaths) { + let artifact_dir = sdk_artifact_dir(); + let manifest = read_manifest(&artifact_dir); + let module = manifest + .modules + .get(&self.module_name) + .unwrap_or_else(|| panic!("SDK support manifest has no module entry for {}", self.module_name)); + let file = resolve_artifact(&artifact_dir, &module.path, "module", &self.module_name); + let host_type = parse_host_type(&module.host_type); + + let prebuilt_client = self.prebuilt_client.as_ref().unwrap_or_else(|| { + panic!( + "SDK test client {} has no prebuilt artifact description", + self.client_project + ) + }); + let client_key = prebuilt_client.key(); + let client = manifest + .clients + .get(&client_key) + .unwrap_or_else(|| panic!("SDK support manifest has no client entry for {client_key}")); + let client_path = resolve_artifact(&artifact_dir, &client.path, "client", &client_key); + + let guard = SpacetimeDbGuard::spawn_in_temp_data_dir(); + let server_url = guard.host_url.as_str(); + let db_name = publish_module( + paths, + server_url, + file.to_str().expect("module artifact path is not UTF-8"), + host_type, + ); + run_prebuilt_client( + prebuilt_client, + &client_path, + &self.client_project, + server_url, + &db_name, + ); + } + + fn run_with_artifacts(&self, paths: &SpacetimePaths, file: &str, host_type: HostType, run_command: &str) { let guard = SpacetimeDbGuard::spawn_in_temp_data_dir(); let server_url = guard.host_url.as_str(); - let db_name = publish_module(paths, server_url, &file, host_type); + let db_name = publish_module(paths, server_url, file, host_type); + + run_client(run_command, &self.client_project, server_url, &db_name); + } +} + +fn sdk_artifact_dir() -> PathBuf { + env::var_os(SDK_TEST_ARTIFACT_DIR_ENV) + .map(PathBuf::from) + .unwrap_or_else(|| panic!("{SDK_TEST_ARTIFACT_DIR_ENV} must be set in SDK test prepare/prebuilt mode")) +} + +fn client_target_dir(client_project: &str) -> PathBuf { + match env::var_os("CARGO_TARGET_DIR").map(PathBuf::from) { + Some(path) if path.is_absolute() => path, + Some(path) => Path::new(client_project).join(path), + None => Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target"), + } +} + +fn manifest_path(artifact_dir: &Path) -> PathBuf { + artifact_dir.join("manifest.json") +} + +fn read_manifest(artifact_dir: &Path) -> ArtifactManifest { + let path = manifest_path(artifact_dir); + let bytes = + fs::read(&path).unwrap_or_else(|err| panic!("failed to read SDK support manifest {}: {err}", path.display())); + serde_json::from_slice(&bytes) + .unwrap_or_else(|err| panic!("failed to parse SDK support manifest {}: {err}", path.display())) +} - run_client(&self.run_command, &self.client_project, server_url, &db_name); +fn read_manifest_or_default(artifact_dir: &Path) -> ArtifactManifest { + let path = manifest_path(artifact_dir); + if path.exists() { + read_manifest(artifact_dir) + } else { + ArtifactManifest::default() } } +fn write_manifest(artifact_dir: &Path, manifest: &ArtifactManifest) { + fs::create_dir_all(artifact_dir).expect("failed to create SDK support directory"); + let path = manifest_path(artifact_dir); + let bytes = serde_json::to_vec_pretty(manifest).expect("failed to serialize SDK support manifest"); + fs::write(&path, bytes).expect("failed to write SDK support manifest"); +} + +fn host_type_name(host_type: HostType) -> &'static str { + match host_type { + HostType::Wasm => "wasm", + HostType::Js => "js", + } +} + +fn parse_host_type(host_type: &str) -> HostType { + match host_type { + "wasm" => HostType::Wasm, + "js" => HostType::Js, + value => panic!("invalid SDK support module host type {value:?}"), + } +} + +fn resolve_artifact(artifact_dir: &Path, relative_path: &Path, kind: &str, key: &str) -> PathBuf { + assert!( + relative_path.is_relative(), + "SDK support manifest {kind} path for {key} must be relative" + ); + let path = artifact_dir.join(relative_path); + assert!( + path.exists(), + "SDK support {kind} artifact for {key} is missing at {}", + path.display() + ); + path +} + +fn safe_artifact_name(name: &str) -> String { + name.chars() + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.') { + character + } else { + '_' + } + }) + .collect() +} + +fn prepare_module_artifact(artifact_dir: &Path, module_name: &str) -> (String, HostType) { + let mut manifest = read_manifest_or_default(artifact_dir); + if let Some(module) = manifest.modules.get(module_name) { + let path = resolve_artifact(artifact_dir, &module.path, "module", module_name); + return ( + path.to_str().expect("module artifact path is not UTF-8").to_owned(), + parse_host_type(&module.host_type), + ); + } + + let (source, host_type) = compile_module(module_name); + let extension = Path::new(&source) + .extension() + .and_then(OsStr::to_str) + .unwrap_or_else(|| host_type_name(host_type)); + let relative_path = PathBuf::from("modules").join(format!("{}.{}", safe_artifact_name(module_name), extension)); + let destination = artifact_dir.join(&relative_path); + fs::create_dir_all(destination.parent().unwrap()).expect("failed to create SDK module artifact directory"); + fs::copy(&source, &destination).unwrap_or_else(|err| { + panic!( + "failed to copy SDK module artifact from {} to {}: {err}", + source, + destination.display() + ) + }); + manifest.modules.insert( + module_name.to_owned(), + ModuleArtifact { + path: relative_path, + host_type: host_type_name(host_type).to_owned(), + }, + ); + write_manifest(artifact_dir, &manifest); + ( + destination + .to_str() + .expect("module artifact path is not UTF-8") + .to_owned(), + host_type, + ) +} + +fn copy_dir_recursive(source: &Path, destination: &Path) { + if destination.exists() { + fs::remove_dir_all(destination).expect("failed to replace SDK client artifact directory"); + } + fs::create_dir_all(destination).expect("failed to create SDK client artifact directory"); + for entry in fs::read_dir(source).unwrap_or_else(|err| panic!("failed to read {}: {err}", source.display())) { + let entry = entry.expect("failed to read SDK client artifact entry"); + let source_path = entry.path(); + let destination_path = destination.join(entry.file_name()); + if entry + .file_type() + .expect("failed to inspect SDK client artifact") + .is_dir() + { + copy_dir_recursive(&source_path, &destination_path); + } else { + fs::copy(&source_path, &destination_path).unwrap_or_else(|err| { + panic!( + "failed to copy SDK client artifact from {} to {}: {err}", + source_path.display(), + destination_path.display() + ) + }); + } + } +} + +fn prepare_client_artifact(artifact_dir: &Path, client: &PrebuiltClient, compile_command: &str, client_project: &str) { + let key = client.key(); + let mut manifest = read_manifest_or_default(artifact_dir); + if let Some(client) = manifest.clients.get(&key) { + resolve_artifact(artifact_dir, &client.path, "client", &key); + return; + } + + compile_client(compile_command, client_project); + + let relative_path = PathBuf::from("clients").join(&key); + let destination = artifact_dir.join(&relative_path); + match client { + PrebuiltClient::NativeRust { binary_name, .. } => { + fs::create_dir_all(destination.parent().unwrap()).expect("failed to create SDK client artifact directory"); + let executable_name = if cfg!(windows) { + format!("{binary_name}.exe") + } else { + binary_name.clone() + }; + let source = client_target_dir(client_project).join("debug").join(executable_name); + fs::copy(&source, &destination).unwrap_or_else(|err| { + panic!( + "failed to copy native SDK client from {} to {}: {err}", + source.display(), + destination.display() + ) + }); + } + PrebuiltClient::BrowserRust { package_name, .. } => { + let source = Path::new(client_project) + .join("target/sdk-test-web-bindgen") + .join(package_name); + copy_dir_recursive(&source, &destination); + } + PrebuiltClient::TypeScript { entrypoint, .. } => { + let source = Path::new(client_project).join(entrypoint.parent().unwrap_or_else(|| { + panic!( + "TypeScript SDK client entrypoint {} has no parent", + entrypoint.display() + ) + })); + copy_dir_recursive(&source, &destination); + } + } + + manifest.clients.insert(key, ClientArtifact { path: relative_path }); + write_manifest(artifact_dir, &manifest); +} + fn status_ok_or_panic(output: std::process::Output, command: &str, test_name: &str) { if !output.status.success() { panic!( - "{}: Error running {:?}: exited with non-zero exit status {}. Output:\n{}", + "{}: Error running {:?}: exited with non-zero exit status {}.\nStdout:\n{}\nStderr:\n{}", test_name, command, output.status, String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), ); } } @@ -369,6 +727,80 @@ fn run_client(run_command: &str, client_project: &str, server_url: &str, db_name status_ok_or_panic(output, run_command, "(running)"); } +fn run_prebuilt_client( + client: &PrebuiltClient, + artifact_path: &Path, + client_project: &str, + server_url: &str, + db_name: &str, +) { + let (program, args): (PathBuf, Vec) = match client { + PrebuiltClient::NativeRust { args, .. } => (artifact_path.to_owned(), args.clone()), + PrebuiltClient::BrowserRust { + package_name, + run_selector, + } => { + let artifact_name = package_name.replace('-', "_"); + let js_module = artifact_path.join(format!("{artifact_name}.cjs")); + assert!( + js_module.exists(), + "browser SDK client is missing {}", + js_module.display() + ); + let js_module = serde_json::to_string(&js_module).expect("failed to quote browser SDK client path"); + let run_selector = serde_json::to_string(run_selector).expect("failed to quote browser SDK selector"); + let script = format!( + "(async () => {{ \ + const m = require({js_module}); \ + if (m.default) {{ await m.default(); }} \ + const run = m.run || m.main || m.start; \ + if (!run) throw new Error('No exported run/main/start function from wasm module'); \ + await run({run_selector}, process.env.{TEST_DB_NAME_ENV_VAR}, process.env.{TEST_SERVER_URL_ENV_VAR}); \ + process.exit(0); \ + }})().catch((e) => {{ console.error(e); process.exit(1); }});" + ); + ( + PathBuf::from("node"), + vec!["--experimental-websocket".into(), "-e".into(), script], + ) + } + PrebuiltClient::TypeScript { entrypoint, args, .. } => { + let entrypoint = artifact_path.join(entrypoint.file_name().unwrap_or_else(|| { + panic!( + "TypeScript SDK client entrypoint {} has no filename", + entrypoint.display() + ) + })); + assert!( + entrypoint.exists(), + "TypeScript SDK client is missing {}", + entrypoint.display() + ); + let mut command_args = vec![entrypoint.to_string_lossy().into_owned()]; + command_args.extend(args.iter().cloned()); + (PathBuf::from("node"), command_args) + } + }; + + let command = format!("{} {}", program.display(), args.join(" ")); + let output = std::process::Command::new(&program) + .args(&args) + .current_dir(client_project) + .env(TEST_CLIENT_PROJECT_ENV_VAR, client_project) + .env(TEST_SERVER_URL_ENV_VAR, server_url) + .env(TEST_DB_NAME_ENV_VAR, db_name) + .env( + "RUST_LOG", + "spacetimedb=debug,spacetimedb_client_api=debug,spacetimedb_lib=debug,spacetimedb_standalone=debug", + ) + .stderr(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .output() + .expect("Error running prebuilt SDK client"); + + status_ok_or_panic(output, &command, "(running prebuilt client)"); +} + #[derive(Clone, Default)] pub struct TestBuilder { name: Option, @@ -379,6 +811,7 @@ pub struct TestBuilder { generate_subdir: Option, compile_command: Option, run_command: Option, + prebuilt_client: Option, } impl TestBuilder { @@ -439,6 +872,13 @@ impl TestBuilder { } } + pub fn with_prebuilt_client(self, prebuilt_client: PrebuiltClient) -> Self { + TestBuilder { + prebuilt_client: Some(prebuilt_client), + ..self + } + } + pub fn with_generate_private_items(self, include_private: bool) -> Self { TestBuilder { generate_include_private: include_private, @@ -477,6 +917,45 @@ impl TestBuilder { run_command: self .run_command .expect("Supply a run command using TestBuilder::with_run_command"), + prebuilt_client: self.prebuilt_client, } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn support_manifest_uses_relative_paths() { + let artifact_dir = tempfile::tempdir().unwrap(); + let module_path = PathBuf::from("modules/sdk-test.wasm"); + fs::create_dir_all(artifact_dir.path().join("modules")).unwrap(); + fs::write(artifact_dir.path().join(&module_path), []).unwrap(); + + let mut manifest = ArtifactManifest::default(); + manifest.modules.insert( + "sdk-test".into(), + ModuleArtifact { + path: module_path.clone(), + host_type: "wasm".into(), + }, + ); + write_manifest(artifact_dir.path(), &manifest); + + let manifest = read_manifest(artifact_dir.path()); + let module = manifest.modules.get("sdk-test").unwrap(); + assert_eq!(module.path, module_path); + assert_eq!( + resolve_artifact(artifact_dir.path(), &module.path, "module", "sdk-test"), + artifact_dir.path().join("modules/sdk-test.wasm"), + ); + } + + #[test] + #[should_panic(expected = "is missing")] + fn missing_support_artifact_fails_instead_of_falling_back() { + let artifact_dir = tempfile::tempdir().unwrap(); + resolve_artifact(artifact_dir.path(), Path::new("clients/missing"), "client", "missing"); + } +} diff --git a/sdks/rust/tests/test.rs b/sdks/rust/tests/test.rs index af7c535824b..e3ab36a5bb9 100644 --- a/sdks/rust/tests/test.rs +++ b/sdks/rust/tests/test.rs @@ -1,7 +1,7 @@ #[cfg(feature = "browser")] use std::path::Path; -use spacetimedb_testing::sdk::{Test, TestBuilder}; +use spacetimedb_testing::sdk::{PrebuiltClient, Test, TestBuilder}; fn platform_test_builder(client_project: &str, run_selector: Option<&str>) -> TestBuilder { let builder = Test::builder(); @@ -77,6 +77,10 @@ fn platform_test_builder(client_project: &str, run_selector: Option<&str>) -> Te builder .with_compile_command(compile_command) .with_run_command(run_command) + .with_prebuilt_client(PrebuiltClient::BrowserRust { + package_name, + run_selector: run_selector.to_owned(), + }) } #[cfg(not(feature = "browser"))] @@ -86,9 +90,17 @@ fn platform_test_builder(client_project: &str, run_selector: Option<&str>) -> Te None => "cargo run".to_owned(), }; + let binary_name = std::path::Path::new(client_project) + .file_name() + .and_then(|name| name.to_str()) + .expect("client project path should end in a UTF-8 directory name") + .to_owned(); + let args = run_selector.into_iter().map(str::to_owned).collect(); + builder .with_compile_command("cargo build") .with_run_command(run_command) + .with_prebuilt_client(PrebuiltClient::NativeRust { binary_name, args }) } } @@ -627,14 +639,11 @@ mod case_conversion_ts { const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/case-conversion-client"); fn make_test(subcommand: &str) -> Test { - Test::builder() + super::platform_test_builder(CLIENT, Some(subcommand)) .with_name(subcommand) .with_module(MODULE) - .with_client(CLIENT) .with_language("rust") .with_bindings_dir("src/module_bindings") - .with_compile_command("cargo build") - .with_run_command(format!("cargo run -- {}", subcommand)) .build() } @@ -676,14 +685,11 @@ mod case_conversion_rust { const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/case-conversion-client"); fn make_test(subcommand: &str) -> Test { - Test::builder() + super::platform_test_builder(CLIENT, Some(subcommand)) .with_name(subcommand) .with_module(MODULE) - .with_client(CLIENT) .with_language("rust") .with_bindings_dir("src/module_bindings") - .with_compile_command("cargo build") - .with_run_command(format!("cargo run -- {}", subcommand)) .build() } @@ -724,7 +730,7 @@ mod case_conversion_rust { /// - Table accessors, field names with digit boundaries, nested structs, enum variants /// - Reducers with explicit names, query builder filters and joins mod case_conversion_rust_ts_client { - use spacetimedb_testing::sdk::Test; + use spacetimedb_testing::sdk::{PrebuiltClient, Test}; const MODULE: &str = "sdk-test-case-conversion"; const CLIENT: &str = concat!( @@ -743,6 +749,11 @@ mod case_conversion_rust_ts_client { "sh -c 'pnpm install && pnpm --dir .. run build && pnpm exec prettier --write src/module_bindings && pnpm run build'", ) .with_run_command(format!("node dist/index.js {}", subcommand)) + .with_prebuilt_client(PrebuiltClient::TypeScript { + package_name: "case-conversion-test-client".to_owned(), + entrypoint: "dist/index.js".into(), + args: vec![subcommand.to_owned()], + }) .build() } diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs index 373a30ef49a..6cb66a9eed7 100644 --- a/tools/ci/commands/test/src/main.rs +++ b/tools/ci/commands/test/src/main.rs @@ -1,128 +1,357 @@ #![allow(clippy::disallowed_macros)] -use anyhow::Result; +use anyhow::{ensure, Context, Result}; use ci_common::pnpm; -use clap::Parser; -use duct::cmd; +use clap::{Parser, Subcommand}; +use duct::{cmd, Expression}; +use std::fs; +use std::path::{Path, PathBuf}; -/// Runs tests +const SDK_TEST_MODE_ENV: &str = "SPACETIME_SDK_TEST_MODE"; +const SDK_TEST_ARTIFACT_DIR_ENV: &str = "SPACETIME_SDK_TEST_ARTIFACT_DIR"; + +/// Runs the public test suite locally or prepares its CI nextest archives. /// -/// Runs rust tests, codegens csharp sdk and runs csharp tests. -/// This does not include Unreal tests. -/// This expects to run in a clean git state. +/// With no subcommand, this preserves the all-in-one local test workflow. #[derive(Parser)] -struct Cli {} +struct Cli { + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum Command { + /// CI build job: run unsharded checks and archive the core Rust test binaries. + ArchiveCore { + #[arg(long)] + archive_file: PathBuf, + #[arg(long)] + unstable_archive_file: PathBuf, + }, + /// CI build job: prepare SDK modules/clients and archive native and browser tests. + ArchiveSdk { + #[arg(long)] + native_archive_file: PathBuf, + #[arg(long)] + browser_archive_file: PathBuf, + #[arg(long)] + support_archive_file: PathBuf, + }, +} fn main() -> Result<()> { - Cli::parse(); + match Cli::parse().command { + None => run_all_locally(), + Some(Command::ArchiveCore { + archive_file, + unstable_archive_file, + }) => archive_core(&archive_file, &unstable_archive_file), + Some(Command::ArchiveSdk { + native_archive_file, + browser_archive_file, + support_archive_file, + }) => archive_sdk(&native_archive_file, &browser_archive_file, &support_archive_file), + } +} - pnpm(["build"]).dir("crates/bindings-typescript").run()?; +fn run(expression: Expression, failure: &str) -> Result<()> { + let output = expression.unchecked().run()?; + ensure!(output.status.success(), "{failure}"); + Ok(()) +} - // TODO: This doesn't work on at least user Linux machines, because something here apparently uses `sudo`? +fn run_typescript_bindings_build() -> Result<()> { + pnpm(["build"]).dir("crates/bindings-typescript").run()?; + Ok(()) +} - // Exclude smoketests from `cargo test --all` since they require pre-built binaries. - // Smoketests have their own dedicated command: `cargo ci smoketests` - cmd!( - "cargo", - "test", - "--all", - "--exclude", - "spacetimedb-smoketests", - "--exclude", - "spacetimedb-sdk", - "--exclude", - "spacetimedb", - "--", - "--test-threads=2", - "--skip", - "unreal" - ) - .run()?; - // Bindings snapshot tests rely on the unstable feature, - // as they compile and test APIs which are gated behind that feature, - // e.g. procedures, HTTP handlers. - cmd!( - "cargo", - "test", - "-p", - "spacetimedb", - "--features", - "unstable", - "--", - "--test-threads=2", - ) - .run()?; - // The SDK test harness uses the same child-process server guard as smoketests, - // which expects release CLI/standalone binaries to already exist. - cmd!( - "cargo", - "build", - "--release", - "-p", - "spacetimedb-cli", - "-p", - "spacetimedb-standalone", - "--features", - "spacetimedb-standalone/allow_loopback_http_for_tests", +fn run_core_cargo_tests() -> Result<()> { + run( + cmd!( + "cargo", + "test", + "--workspace", + "--exclude", + "spacetimedb-smoketests", + "--exclude", + "spacetimedb-sdk", + "--exclude", + "spacetimedb", + "--", + "--test-threads=2", + "--skip", + "unreal", + ), + "workspace tests failed", + )?; + run( + cmd!( + "cargo", + "test", + "-p", + "spacetimedb", + "--features", + "unstable", + "--", + "--test-threads=2", + ), + "unstable spacetimedb tests failed", ) - .run()?; - // SDK procedure tests intentionally make localhost HTTP requests. - cmd!( - "cargo", - "test", - "-p", - "spacetimedb-sdk", - "--features", - "allow_loopback_http_for_tests", +} + +fn run_sdk_tests_locally() -> Result<()> { + run( + cmd!( + "cargo", + "build", + "--release", + "-p", + "spacetimedb-cli", + "-p", + "spacetimedb-standalone", + "--features", + "spacetimedb-standalone/allow_loopback_http_for_tests", + ), + "SDK support binary build failed", + )?; + run_sdk_test_command(false, None, false)?; + run_sdk_test_command(true, None, false) +} + +fn run_sdk_test_command(browser: bool, artifact_dir: Option<&Path>, prepare: bool) -> Result<()> { + let features = if browser { + "allow_loopback_http_for_tests,browser" + } else { + "allow_loopback_http_for_tests" + }; + let mut args = vec!["test", "-p", "spacetimedb-sdk", "--features", features]; + if prepare { + args.extend(["--test", "test"]); + } + args.extend([ "--", - "--test-threads=2", + if prepare { + "--test-threads=1" + } else { + "--test-threads=2" + }, "--skip", - "unreal" + "unreal", + ]); + if prepare { + // These failure-path tests share their module and client artifacts with ordinary + // tests. Do not execute them during preparation: libtest would otherwise reject + // their intentionally successful prepare-only path because they are `should_panic`. + args.extend(["--skip", "should_fail", "--skip", "subscribe_all_select_star"]); + } + let mut command = cmd("cargo", args); + if let Some(artifact_dir) = artifact_dir { + command = command + .env(SDK_TEST_MODE_ENV, "prepare") + .env(SDK_TEST_ARTIFACT_DIR_ENV, artifact_dir); + } + run(command, "SDK tests failed") +} + +fn run_durability_fallocate_tests() -> Result<()> { + run( + cmd!( + "cargo", + "test", + "-p", + "spacetimedb-durability", + "--features", + "fallocate", + "--", + "--test-threads=1", + ), + "durability fallocate tests failed", ) - .run()?; - // Run the same SDK suite against wasm/browser test clients. - cmd!( - "cargo", - "test", - "-p", - "spacetimedb-sdk", - "--features", - "allow_loopback_http_for_tests,browser", - "--", - "--test-threads=2", - "--skip", - "unreal" +} + +fn run_csharp_bindings_checks() -> Result<()> { + run( + cmd!("bash", "tools/check-diff.sh"), + "repository has changes before C# regeneration", + )?; + run( + cmd!( + "cargo", + "run", + "-p", + "spacetimedb-codegen", + "--example", + "regen-csharp-moduledef", + ), + "C# module definition regeneration failed", + )?; + run( + cmd!("bash", "tools/check-diff.sh", "crates/bindings-csharp"), + "generated C# bindings are out of date", + )?; + run( + cmd!("dotnet", "test", "-warnaserror").dir("crates/bindings-csharp"), + "C# bindings tests failed", ) - .run()?; - // TODO: This should check for a diff at the start. If there is one, we should alert the user - // that we're disabling diff checks because they have a dirty git repo, and to re-run in a clean one - // if they want those checks. - - // The fallocate tests have been flakely when running in parallel - cmd!( - "cargo", - "test", - "-p", - "spacetimedb-durability", - "--features", - "fallocate", - "--", - "--test-threads=1", +} + +fn run_all_locally() -> Result<()> { + run_typescript_bindings_build()?; + // TODO: This doesn't work on at least user Linux machines, because something here apparently uses `sudo`? + run_core_cargo_tests()?; + run_sdk_tests_locally()?; + run_durability_fallocate_tests()?; + run_csharp_bindings_checks() +} + +fn archive_core(archive_file: &Path, unstable_archive_file: &Path) -> Result<()> { + run_typescript_bindings_build()?; + run( + cmd!( + "cargo", + "nextest", + "archive", + "--timings", + "--workspace", + "--exclude", + "spacetimedb-smoketests", + "--exclude", + "spacetimedb-sdk", + "--exclude", + "spacetimedb", + "--archive-file", + archive_file, + ), + "failed to archive workspace tests", + )?; + run( + cmd!( + "cargo", + "nextest", + "archive", + "--timings", + "-p", + "spacetimedb", + "--features", + "unstable", + "--archive-file", + unstable_archive_file, + ), + "failed to archive unstable spacetimedb tests", + )?; + + // Nextest does not support doctests, so preserve the current doctest coverage here. + run( + cmd!( + "cargo", + "test", + "--workspace", + "--exclude", + "spacetimedb-smoketests", + "--exclude", + "spacetimedb-sdk", + "--exclude", + "spacetimedb", + "--doc", + "--", + "--test-threads=2", + "--skip", + "unreal", + ), + "workspace doctests failed", + )?; + run( + cmd!( + "cargo", + "test", + "-p", + "spacetimedb", + "--features", + "unstable", + "--doc", + "--", + "--test-threads=2", + ), + "unstable spacetimedb doctests failed", + )?; + run_durability_fallocate_tests()?; + run_csharp_bindings_checks() +} + +fn archive_sdk_tests(browser: bool, archive_file: &Path) -> Result<()> { + let features = if browser { + "allow_loopback_http_for_tests,browser" + } else { + "allow_loopback_http_for_tests" + }; + run( + cmd!( + "cargo", + "nextest", + "archive", + "--timings", + "-p", + "spacetimedb-sdk", + "--features", + features, + "--archive-file", + archive_file, + ), + "failed to archive SDK tests", ) - .run()?; - cmd!("bash", "tools/check-diff.sh").run()?; - cmd!( - "cargo", - "run", - "-p", - "spacetimedb-codegen", - "--example", - "regen-csharp-moduledef", +} + +fn run_sdk_doctests(browser: bool) -> Result<()> { + let features = if browser { + "allow_loopback_http_for_tests,browser" + } else { + "allow_loopback_http_for_tests" + }; + run( + cmd!( + "cargo", + "test", + "-p", + "spacetimedb-sdk", + "--features", + features, + "--doc", + ), + "SDK doctests failed", ) - .run()?; - cmd!("bash", "tools/check-diff.sh", "crates/bindings-csharp").run()?; - cmd!("dotnet", "test", "-warnaserror") - .dir("crates/bindings-csharp") - .run()?; +} - Ok(()) +fn archive_sdk(native_archive_file: &Path, browser_archive_file: &Path, support_archive_file: &Path) -> Result<()> { + let artifact_dir = PathBuf::from("target/sdk-test-support"); + if artifact_dir.exists() { + fs::remove_dir_all(&artifact_dir).context("failed to clear old SDK support artifacts")?; + } + + // Libtest runs preparation serially so the harness's in-process memoization + // builds each unique module and client once per feature mode. + run_sdk_test_command(false, Some(&artifact_dir), true)?; + run_sdk_test_command(true, Some(&artifact_dir), true)?; + archive_sdk_tests(false, native_archive_file)?; + archive_sdk_tests(true, browser_archive_file)?; + run_sdk_doctests(false)?; + run_sdk_doctests(true)?; + run( + cmd!("bash", "tools/check-diff.sh"), + "SDK binding generation changed tracked files", + )?; + + let artifact_parent = artifact_dir.parent().context("SDK support directory has no parent")?; + let artifact_name = artifact_dir + .file_name() + .context("SDK support directory has no filename")?; + run( + cmd!( + "tar", + "-czf", + support_archive_file, + "-C", + artifact_parent, + artifact_name + ), + "failed to package SDK support artifacts", + ) } From 22e580966d2a26559547e484291fa94ef47bf868 Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Mon, 24 Aug 2026 12:30:51 -0700 Subject: [PATCH 2/2] fixes --- .github/workflows/ci.yml | 29 +++++-- Cargo.lock | 1 + crates/testing/src/modules.rs | 128 ++++++++++++++++++++++++++++- sdks/rust/tests/test.rs | 22 ++++- tools/ci/commands/test/Cargo.toml | 1 + tools/ci/commands/test/src/main.rs | 49 ++++++++++- 6 files changed, 214 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27aadc900ea..41a6dc0412b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -630,14 +630,27 @@ jobs: with: run_install: true + # Core tests exercise modules written in TypeScript, C#, and C++ at runtime. + - &install-test-module-toolchains + name: Install test module toolchains + run: | + sudo apt-get update + sudo apt-get install -y cmake + git clone https://github.com/emscripten-core/emsdk.git ~/emsdk + cd ~/emsdk + ./emsdk install 4.0.21 + ./emsdk activate 4.0.21 + - name: Install cargo-nextest uses: taiki-e/install-action@nextest - name: Prepare core test suite run: | + source ~/emsdk/emsdk_env.sh cargo ci test archive-core \ --archive-file test-suite-nextest.tar.zst \ - --unstable-archive-file test-suite-unstable-nextest.tar.zst + --unstable-archive-file test-suite-unstable-nextest.tar.zst \ + --support-archive-file test-module-support.tar.gz - name: Upload core test build uses: actions/upload-artifact@v4 @@ -646,6 +659,7 @@ jobs: path: | test-suite-nextest.tar.zst test-suite-unstable-nextest.tar.zst + test-module-support.tar.gz if-no-files-found: error overwrite: true retention-days: 14 @@ -671,6 +685,7 @@ jobs: env: RUST_BACKTRACE: full PARTITION_COUNT: 2 + SPACETIME_TEST_MODULE_ARTIFACT_DIR: ${{ github.workspace }}/test-module-support steps: - *find-git-ref - *checkout-sources @@ -685,6 +700,9 @@ jobs: with: name: test-suite-build + - name: Extract test module artifacts + run: tar -xzf test-module-support.tar.gz + - name: Run core test partition run: | cargo nextest run \ @@ -737,14 +755,7 @@ jobs: run_install: true # SDK preparation compiles TypeScript, C#, and C++ test modules once. - - name: Install module toolchains - run: | - sudo apt-get update - sudo apt-get install -y cmake - git clone https://github.com/emscripten-core/emsdk.git ~/emsdk - cd ~/emsdk - ./emsdk install 4.0.21 - ./emsdk activate 4.0.21 + - *install-test-module-toolchains - name: Install wasm-bindgen CLI run: | diff --git a/Cargo.lock b/Cargo.lock index 22cd022fb09..e3d2ae3d370 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -994,6 +994,7 @@ dependencies = [ "ci-common", "clap 4.5.50", "duct", + "spacetimedb-testing", ] [[package]] diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 465e22c3d6e..55f7e652f97 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -1,4 +1,6 @@ +use std::collections::BTreeMap; use std::env; +use std::fs; use std::future::Future; use std::panic::AssertUnwindSafe; use std::path::{Path, PathBuf}; @@ -228,14 +230,62 @@ pub struct CompiledModule { program_bytes: OnceLock, } -#[derive(Debug, PartialEq, Eq)] +pub const TEST_MODULE_ARTIFACT_DIR_ENV: &str = "SPACETIME_TEST_MODULE_ARTIFACT_DIR"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CompilationMode { Debug, Release, } +impl CompilationMode { + fn name(self) -> &'static str { + match self { + Self::Debug => "debug", + Self::Release => "release", + } + } +} + +#[derive(Default, serde::Deserialize, serde::Serialize)] +struct ModuleArtifactManifest { + modules: BTreeMap, +} + +#[derive(serde::Deserialize, serde::Serialize)] +struct ModuleArtifact { + path: PathBuf, + host_type: String, +} + +fn module_artifact_key(name: &str, mode: CompilationMode) -> String { + format!("{}:{name}", mode.name()) +} + +fn host_type_name(host_type: HostType) -> &'static str { + match host_type { + HostType::Wasm => "wasm", + HostType::Js => "js", + } +} + +fn parse_host_type(host_type: &str) -> HostType { + match host_type { + "wasm" => HostType::Wasm, + "js" => HostType::Js, + value => panic!("invalid test module host type {value:?}"), + } +} + impl CompiledModule { pub fn compile(name: &str, mode: CompilationMode) -> Self { + if let Some(artifact_dir) = env::var_os(TEST_MODULE_ARTIFACT_DIR_ENV) { + return Self::from_artifact(Path::new(&artifact_dir), name, mode); + } + Self::compile_locally(name, mode) + } + + fn compile_locally(name: &str, mode: CompilationMode) -> Self { let (path, host_type) = spacetimedb_cli::build( &module_path(name), Some(PathBuf::from("src")).as_deref(), @@ -253,6 +303,41 @@ impl CompiledModule { } } + fn from_artifact(artifact_dir: &Path, name: &str, mode: CompilationMode) -> Self { + let manifest_path = artifact_dir.join("manifest.json"); + let manifest: ModuleArtifactManifest = + serde_json::from_slice(&fs::read(&manifest_path).unwrap_or_else(|err| { + panic!("failed to read test module manifest {}: {err}", manifest_path.display()) + })) + .unwrap_or_else(|err| { + panic!( + "failed to parse test module manifest {}: {err}", + manifest_path.display() + ) + }); + let key = module_artifact_key(name, mode); + let artifact = manifest + .modules + .get(&key) + .unwrap_or_else(|| panic!("test module manifest has no {mode:?} artifact for {name}")); + assert!( + artifact.path.is_relative(), + "test module manifest path for {key} must be relative" + ); + let path = artifact_dir.join(&artifact.path); + assert!( + path.is_file(), + "test module artifact for {key} is missing at {}", + path.display() + ); + Self { + name: name.to_owned(), + path, + host_type: parse_host_type(&artifact.host_type), + program_bytes: OnceLock::new(), + } + } + pub fn path(&self) -> &Path { &self.path } @@ -388,6 +473,47 @@ impl CompiledModule { } } +/// Compiles the modules needed by the sharded core test suite and writes a +/// relocatable support directory. Local tests continue to compile on demand; +/// CI shards opt into these artifacts with [`TEST_MODULE_ARTIFACT_DIR_ENV`]. +pub fn prepare_module_artifacts( + artifact_dir: &Path, + module_names: &[&str], + mode: CompilationMode, +) -> anyhow::Result<()> { + if artifact_dir.exists() { + fs::remove_dir_all(artifact_dir)?; + } + let modules_dir = artifact_dir.join("modules").join(mode.name()); + fs::create_dir_all(&modules_dir)?; + + let mut manifest = ModuleArtifactManifest::default(); + for &name in module_names { + let module = CompiledModule::compile_locally(name, mode); + let filename = module + .path + .file_name() + .ok_or_else(|| anyhow::anyhow!("compiled module {name} has no filename"))?; + let relative_path = PathBuf::from("modules").join(mode.name()).join(name).join(filename); + let destination = artifact_dir.join(&relative_path); + fs::create_dir_all(destination.parent().unwrap())?; + fs::copy(&module.path, &destination)?; + manifest.modules.insert( + module_artifact_key(name, mode), + ModuleArtifact { + path: relative_path, + host_type: host_type_name(module.host_type).to_owned(), + }, + ); + } + + fs::write( + artifact_dir.join("manifest.json"), + serde_json::to_vec_pretty(&manifest)?, + )?; + Ok(()) +} + /// These standalone module tests run a `StandaloneEnv` in-process. /// That means [`CompiledModule::with_module_async`] and [`CompiledModule::with_module`] /// own the Tokio runtime, host controller, module host, scheduler, `RelationalDB`, diff --git a/sdks/rust/tests/test.rs b/sdks/rust/tests/test.rs index e3ab36a5bb9..01a345e98da 100644 --- a/sdks/rust/tests/test.rs +++ b/sdks/rust/tests/test.rs @@ -633,17 +633,24 @@ view_tests!(rust_view, ""); //view_tests!(cpp_view, "-cpp"); mod case_conversion_ts { - use spacetimedb_testing::sdk::Test; + use spacetimedb_testing::sdk::{PrebuiltClient, Test}; const MODULE: &str = "sdk-test-case-conversion-ts"; const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/case-conversion-client"); fn make_test(subcommand: &str) -> Test { - super::platform_test_builder(CLIENT, Some(subcommand)) + Test::builder() + .with_client(CLIENT) .with_name(subcommand) .with_module(MODULE) .with_language("rust") .with_bindings_dir("src/module_bindings") + .with_compile_command("cargo build") + .with_run_command(format!("cargo run -- {subcommand}")) + .with_prebuilt_client(PrebuiltClient::NativeRust { + binary_name: "case-conversion-client".into(), + args: vec![subcommand.into()], + }) .build() } @@ -679,17 +686,24 @@ mod case_conversion_ts { } mod case_conversion_rust { - use spacetimedb_testing::sdk::Test; + use spacetimedb_testing::sdk::{PrebuiltClient, Test}; const MODULE: &str = "sdk-test-case-conversion"; const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/case-conversion-client"); fn make_test(subcommand: &str) -> Test { - super::platform_test_builder(CLIENT, Some(subcommand)) + Test::builder() + .with_client(CLIENT) .with_name(subcommand) .with_module(MODULE) .with_language("rust") .with_bindings_dir("src/module_bindings") + .with_compile_command("cargo build") + .with_run_command(format!("cargo run -- {subcommand}")) + .with_prebuilt_client(PrebuiltClient::NativeRust { + binary_name: "case-conversion-client".into(), + args: vec![subcommand.into()], + }) .build() } diff --git a/tools/ci/commands/test/Cargo.toml b/tools/ci/commands/test/Cargo.toml index 91d98a7f490..8825acd8671 100644 --- a/tools/ci/commands/test/Cargo.toml +++ b/tools/ci/commands/test/Cargo.toml @@ -8,3 +8,4 @@ anyhow.workspace = true clap.workspace = true duct.workspace = true ci-common = { path = "../../common" } +spacetimedb-testing = { path = "../../../../crates/testing" } diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs index 6cb66a9eed7..c12e0abc3db 100644 --- a/tools/ci/commands/test/src/main.rs +++ b/tools/ci/commands/test/src/main.rs @@ -3,6 +3,7 @@ use anyhow::{ensure, Context, Result}; use ci_common::pnpm; use clap::{Parser, Subcommand}; use duct::{cmd, Expression}; +use spacetimedb_testing::modules::{prepare_module_artifacts, CompilationMode}; use std::fs; use std::path::{Path, PathBuf}; @@ -26,6 +27,8 @@ enum Command { archive_file: PathBuf, #[arg(long)] unstable_archive_file: PathBuf, + #[arg(long)] + support_archive_file: PathBuf, }, /// CI build job: prepare SDK modules/clients and archive native and browser tests. ArchiveSdk { @@ -44,7 +47,8 @@ fn main() -> Result<()> { Some(Command::ArchiveCore { archive_file, unstable_archive_file, - }) => archive_core(&archive_file, &unstable_archive_file), + support_archive_file, + }) => archive_core(&archive_file, &unstable_archive_file, &support_archive_file), Some(Command::ArchiveSdk { native_archive_file, browser_archive_file, @@ -203,8 +207,49 @@ fn run_all_locally() -> Result<()> { run_csharp_bindings_checks() } -fn archive_core(archive_file: &Path, unstable_archive_file: &Path) -> Result<()> { +fn archive_core(archive_file: &Path, unstable_archive_file: &Path, support_archive_file: &Path) -> Result<()> { + // Keep this inventory in sync with CompiledModule uses in the archived core suites. + // A missing entry fails in a shard rather than silently compiling there. + const CORE_TEST_MODULES: &[&str] = &[ + "benchmarks", + "benchmarks-cpp", + "benchmarks-cs", + "benchmarks-ts", + "module-test", + "module-test-cpp", + "module-test-cs", + "module-test-ts", + "sdk-test", + "sdk-test-cs", + "sdk-test-ts", + "sdk-test-case-conversion", + "sdk-test-case-conversion-ts", + "sdk-test-connect-disconnect", + "sdk-test-connect-disconnect-cs", + "sdk-test-connect-disconnect-ts", + ]; + run_typescript_bindings_build()?; + let artifact_dir = PathBuf::from("target/test-module-support"); + prepare_module_artifacts(&artifact_dir, CORE_TEST_MODULES, CompilationMode::Debug) + .context("failed to prepare core test modules")?; + let artifact_parent = artifact_dir + .parent() + .context("test module support directory has no parent")?; + let artifact_name = artifact_dir + .file_name() + .context("test module support directory has no filename")?; + run( + cmd!( + "tar", + "-czf", + support_archive_file, + "-C", + artifact_parent, + artifact_name + ), + "failed to package core test module artifacts", + )?; run( cmd!( "cargo",