diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f5ff896..cde170a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -310,12 +310,21 @@ jobs: run: pnpm test:native # --------------------------------------------------------------------------- - # Optional / manual jobs + # Required end-to-end correctness # --------------------------------------------------------------------------- - playwright: - name: Playwright (Manual) - if: github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest + e2e: + name: E2E ${{ matrix.name }} + strategy: + fail-fast: false + matrix: + include: + - name: Visual + project: visual + runner: macos-latest + - name: GPU + project: gpu + runner: macos-latest + runs-on: ${{ matrix.runner }} permissions: contents: read steps: @@ -332,12 +341,7 @@ jobs: - name: Install toolchain run: rustup show - - uses: Swatinem/rust-cache@v2 - with: - shared-key: shift-rust - - name: Cache node_modules - id: modules-cache uses: actions/cache@v6 with: path: | @@ -350,17 +354,6 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile --prefer-offline - - name: Cache Playwright browsers - id: playwright-cache - uses: actions/cache@v6 - with: - path: ~/.cache/ms-playwright - key: playwright-${{ runner.os }}-${{ hashFiles('apps/desktop/node_modules/@playwright/test/package.json') }} - - - name: Install Playwright browsers - if: steps.playwright-cache.outputs.cache-hit != 'true' - run: pnpm --filter @shift/desktop exec playwright install - - name: Cache native module binary id: native-cache uses: actions/cache@v6 @@ -368,20 +361,23 @@ jobs: path: crates/shift-bridge/*.node key: native-module-${{ runner.os }}-${{ hashFiles('crates/**', 'Cargo.lock') }} + - uses: Swatinem/rust-cache@v2 + if: steps.native-cache.outputs.cache-hit != 'true' + with: + shared-key: shift-rust-${{ runner.os }} + - name: Build native module if: steps.native-cache.outputs.cache-hit != 'true' run: pnpm build:native - - name: Run Playwright tests - run: xvfb-run pnpm test:playwright - env: - LIBGL_ALWAYS_SOFTWARE: "true" + - name: Run E2E tests + run: pnpm test:e2e:${{ matrix.project }} - - name: Upload snapshots on failure + - name: Upload Playwright failures if: failure() uses: actions/upload-artifact@v7 with: - name: playwright-results + name: playwright-${{ matrix.project }}-results path: apps/desktop/e2e/test-results/ retention-days: 7 @@ -400,6 +396,7 @@ jobs: - rust-fmt - rust-check - integration + - e2e runs-on: ubuntu-latest permissions: {} steps: diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 0c3a6fff..29074411 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -119,17 +119,9 @@ jobs: - name: Run engine benchmarks run: pnpm test:perf - playwright: - name: Playwright ${{ matrix.name }} + playwright-perf: + name: Playwright Perf if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - strategy: - fail-fast: false - matrix: - include: - - name: GPU - project: gpu - - name: Perf - project: perf runs-on: macos-latest permissions: contents: read @@ -212,22 +204,19 @@ jobs: path: crates/shift-bridge/*.node key: ${{ steps.native-cache.outputs.cache-primary-key }} - - name: Build E2E app - run: pnpm --filter @shift/desktop build:e2e - - - name: Run Playwright ${{ matrix.name }} tests - run: pnpm --filter @shift/desktop exec playwright test --project=${{ matrix.project }} + - name: Run Playwright perf tests + run: pnpm test:e2e:perf - name: Upload Playwright failures if: failure() uses: actions/upload-artifact@v7 with: - name: playwright-${{ matrix.project }}-results + name: playwright-perf-results path: apps/desktop/e2e/test-results/ retention-days: 7 - name: Upload perf results - if: ${{ always() && matrix.project == 'perf' }} + if: always() uses: actions/upload-artifact@v7 with: name: perf-results diff --git a/Cargo.lock b/Cargo.lock index 2af545c9..a8f5a156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,6 +228,20 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + [[package]] name = "block2" version = "0.6.2" @@ -317,6 +331,8 @@ version = "1.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ad45f4f74e4e20eaa392913b7b33a7091c87e59628f4dd27888205ad888843c" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -423,6 +439,12 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "convert_case" version = "0.11.0" @@ -438,6 +460,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -982,10 +1013,21 @@ checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasi", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + [[package]] name = "gimli" version = "0.32.3" @@ -1260,6 +1302,16 @@ dependencies = [ "syn", ] +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.99" @@ -1898,6 +1950,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.8.5" @@ -2026,6 +2084,25 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + [[package]] name = "rusqlite" version = "0.37.0" @@ -2187,6 +2264,7 @@ dependencies = [ "ordered-float", "plist", "quick-xml 0.37.5", + "rayon", "serde", "shift-font", "shift-source", @@ -2237,7 +2315,7 @@ name = "shift-font" version = "0.1.0" dependencies = [ "fontdrasil", - "getrandom", + "getrandom 0.3.3", "indexmap", "kurbo 0.13.0", "linesweeper", @@ -2246,10 +2324,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "shift-glyph-codec" -version = "0.1.0" - [[package]] name = "shift-slug" version = "0.1.0" @@ -2258,7 +2332,6 @@ dependencies = [ "pollster", "shift-backends", "shift-font", - "shift-glyph-codec", "skrifa", "tiny-skia", "wgpu", @@ -2268,7 +2341,7 @@ dependencies = [ name = "shift-source" version = "0.1.0" dependencies = [ - "getrandom", + "getrandom 0.3.3", "serde", "serde_json", "shift-font", @@ -2281,11 +2354,16 @@ dependencies = [ name = "shift-store" version = "0.1.0" dependencies = [ + "blake3", + "rayon", + "rmp-serde", "rusqlite", + "serde", "serde_json", "shift-font", "tempfile", "thiserror", + "zstd", ] [[package]] @@ -2302,6 +2380,7 @@ dependencies = [ name = "shift-workspace" version = "0.1.0" dependencies = [ + "rusqlite", "shift-backends", "shift-font", "shift-source", @@ -2441,7 +2520,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand", - "getrandom", + "getrandom 0.3.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2656,7 +2735,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ - "getrandom", + "getrandom 0.3.3", "js-sys", "wasm-bindgen", ] @@ -3327,3 +3406,31 @@ dependencies = [ "memchr", "thiserror", ] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 85965ef8..35d3cab0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,6 @@ shift-wire = { path = "crates/shift-wire" } shift-backends = { path = "crates/shift-backends" } shift-bridge = { path = "crates/shift-bridge" } shift-font = { path = "crates/shift-font" } -shift-glyph-codec = { path = "crates/shift-glyph-codec" } shift-slug = { path = "crates/shift-slug" } shift-store = { path = "crates/shift-store" } shift-source = { path = "crates/shift-source" } diff --git a/apps/desktop/e2e/README.md b/apps/desktop/e2e/README.md new file mode 100644 index 00000000..98fe5650 --- /dev/null +++ b/apps/desktop/e2e/README.md @@ -0,0 +1,70 @@ +# Desktop E2E tests + +Playwright launches the built Electron application against an isolated user-data directory and a copied workspace. The suites cover different execution environments and should not share rendering assumptions. + +## Commands + +Run commands from the repository root: + +| Command | Purpose | +| ----------------------------- | ---------------------------------------------- | +| `pnpm test:e2e` | Run required visual and GPU correctness suites | +| `pnpm test:e2e:visual` | Run deterministic visual and interaction tests | +| `pnpm test:e2e:visual:update` | Regenerate visual snapshots | +| `pnpm test:e2e:gpu` | Run hardware-GPU correctness tests | +| `pnpm test:e2e:perf` | Run Playwright performance measurements | + +Append a Playwright file filter for a focused run: + +```sh +pnpm test:e2e:visual e2e/home.spec.ts +pnpm test:e2e:gpu e2e/gpu.spec.ts +``` + +Build the native bridge first with `pnpm build:native` when its binary is absent or stale. Each E2E command builds the Electron main, workspace, preload, and renderer bundles through `e2e/build.ts`. + +## Projects and fixtures + +| Project | Fixture | Rendering | CI policy | +| -------- | ------------------------- | ------------------------------------------------- | ------------------------------------ | +| `visual` | `fixtures/electronApp.ts` | Software rendering, DPR 1, `1200×600` page window | Required on pull requests and `main` | +| `gpu` | `fixtures/perfApp.ts` | Hardware GPU and host device scale | Required on pull requests and `main` | +| `perf` | `fixtures/perfApp.ts` | Hardware GPU and host device scale | Nightly and manual only | + +Visual tests default to MutatorSans. The GPU and performance fixture also defaults to MutatorSans, but accepts a real font or designspace through `SHIFT_E2E_FONT_PATH`: + +```sh +SHIFT_E2E_FONT_PATH=/path/to/font.ttf pnpm test:e2e:gpu e2e/gpu.spec.ts +``` + +Fixtures copy source files into a temporary workspace. Tests must not depend on a developer's existing Shift workspace or user-data directory. + +## Visual snapshots + +The visual fixture forces a fixed device scale and sizes the `BrowserWindow` that owns the Playwright page. This prevents snapshots from inheriting the host display's scale or available work area. + +After an intentional visual change: + +1. Run `pnpm test:e2e:visual:update`. +2. Review every changed image under `e2e/__screenshots__/`. +3. Run `pnpm test:e2e:visual` without update mode. + +A snapshot match alone does not prove GPU content exists. Rendering tests that can pass with a blank canvas must also compare frames with and without the relevant canvas or assert equivalent semantic output. + +## Interaction rules + +- Use locator-relative positions for canvas clicks. +- For raw mouse drags, derive page coordinates from the target canvas's `boundingBox()`. +- Keep interactions inside measured bounds; do not assume the host desktop is wider than the fixture window. +- Wait for a route, visible surface, animation frame, or domain state instead of assuming startup completed after a fixed delay. +- Do not force software rendering or a fixed DPR in GPU and performance tests. + +## Failures and artifacts + +Local failures are written to `apps/desktop/e2e/test-results/`. CI uploads the same directory as a Playwright artifact, including screenshots, diffs, traces, and error context. Open a trace with: + +```sh +pnpm --filter @shift/desktop exec playwright show-trace apps/desktop/e2e/test-results//trace.zip +``` + +The project definitions and retry policy live in `apps/desktop/playwright.config.ts`. diff --git a/apps/desktop/e2e/__screenshots__/editor.spec.ts/bg-canvas-A.png b/apps/desktop/e2e/__screenshots__/editor.spec.ts/bg-canvas-A.png index fc00ecbf..a2b19e7e 100644 Binary files a/apps/desktop/e2e/__screenshots__/editor.spec.ts/bg-canvas-A.png and b/apps/desktop/e2e/__screenshots__/editor.spec.ts/bg-canvas-A.png differ diff --git a/apps/desktop/e2e/__screenshots__/editor.spec.ts/editor-canvas-A.png b/apps/desktop/e2e/__screenshots__/editor.spec.ts/editor-canvas-A.png index fc00ecbf..a2b19e7e 100644 Binary files a/apps/desktop/e2e/__screenshots__/editor.spec.ts/editor-canvas-A.png and b/apps/desktop/e2e/__screenshots__/editor.spec.ts/editor-canvas-A.png differ diff --git a/apps/desktop/e2e/__screenshots__/editor.spec.ts/editor-glyph-A.png b/apps/desktop/e2e/__screenshots__/editor.spec.ts/editor-glyph-A.png index 87e55e7a..13a5b218 100644 Binary files a/apps/desktop/e2e/__screenshots__/editor.spec.ts/editor-glyph-A.png and b/apps/desktop/e2e/__screenshots__/editor.spec.ts/editor-glyph-A.png differ diff --git a/apps/desktop/e2e/__screenshots__/editor.spec.ts/handles-canvas-A.png b/apps/desktop/e2e/__screenshots__/editor.spec.ts/handles-canvas-A.png index fc00ecbf..a2b19e7e 100644 Binary files a/apps/desktop/e2e/__screenshots__/editor.spec.ts/handles-canvas-A.png and b/apps/desktop/e2e/__screenshots__/editor.spec.ts/handles-canvas-A.png differ diff --git a/apps/desktop/e2e/__screenshots__/editor.spec.ts/scene-canvas-A.png b/apps/desktop/e2e/__screenshots__/editor.spec.ts/scene-canvas-A.png index fc00ecbf..a2b19e7e 100644 Binary files a/apps/desktop/e2e/__screenshots__/editor.spec.ts/scene-canvas-A.png and b/apps/desktop/e2e/__screenshots__/editor.spec.ts/scene-canvas-A.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/bg-S-guides.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/bg-S-guides.png index 6c306467..51685012 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/bg-S-guides.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/bg-S-guides.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-B-composited.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-B-composited.png index 1adf32f4..1ef63e0a 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-B-composited.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-B-composited.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-I-composited.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-I-composited.png index c390ad81..466808dd 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-I-composited.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-I-composited.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-Q-composited.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-Q-composited.png index add36285..5a3d8d99 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-Q-composited.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-Q-composited.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-S-all-selected.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-S-all-selected.png index 5896c79a..91ea23f9 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-S-all-selected.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-S-all-selected.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-S-composited.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-S-composited.png index 6c306467..51685012 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-S-composited.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/canvas-S-composited.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/editor-S-full.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/editor-S-full.png index 8079c38e..883f42ea 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/editor-S-full.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/editor-S-full.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/handles-S-all-selected.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/handles-S-all-selected.png index 5896c79a..91ea23f9 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/handles-S-all-selected.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/handles-S-all-selected.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/handles-S-idle.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/handles-S-idle.png index 6c306467..51685012 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/handles-S-idle.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/handles-S-idle.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-cubic-curve.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-cubic-curve.png index 0316c6de..d909ecf5 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-cubic-curve.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-cubic-curve.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-mixed-segments.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-mixed-segments.png index 24b227e5..c668a0f1 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-mixed-segments.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-mixed-segments.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-s-curve-handles.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-s-curve-handles.png index c1896ee5..8bf13e3d 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-s-curve-handles.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-s-curve-handles.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-single-point.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-single-point.png index d3f6cc9a..70963d7a 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-single-point.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-single-point.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-straight-segment.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-straight-segment.png index d1c9902b..12a724c0 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-straight-segment.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/pen-straight-segment.png differ diff --git a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/scene-S-filled.png b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/scene-S-filled.png index 6c306467..51685012 100644 Binary files a/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/scene-S-filled.png and b/apps/desktop/e2e/__screenshots__/glyph-rendering.spec.ts/scene-S-filled.png differ diff --git a/apps/desktop/e2e/__screenshots__/home.spec.ts/home-glyph-grid.png b/apps/desktop/e2e/__screenshots__/home.spec.ts/home-glyph-grid.png index bfbb0cdf..a8204af1 100644 Binary files a/apps/desktop/e2e/__screenshots__/home.spec.ts/home-glyph-grid.png and b/apps/desktop/e2e/__screenshots__/home.spec.ts/home-glyph-grid.png differ diff --git a/apps/desktop/e2e/__screenshots__/landing.spec.ts/landing-default.png b/apps/desktop/e2e/__screenshots__/landing.spec.ts/landing-default.png index 5bdb878a..2bdaddbd 100644 Binary files a/apps/desktop/e2e/__screenshots__/landing.spec.ts/landing-default.png and b/apps/desktop/e2e/__screenshots__/landing.spec.ts/landing-default.png differ diff --git a/apps/desktop/e2e/__screenshots__/theme.spec.ts/theme-light-home.png b/apps/desktop/e2e/__screenshots__/theme.spec.ts/theme-light-home.png index bfbb0cdf..a8204af1 100644 Binary files a/apps/desktop/e2e/__screenshots__/theme.spec.ts/theme-light-home.png and b/apps/desktop/e2e/__screenshots__/theme.spec.ts/theme-light-home.png differ diff --git a/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-hand.png b/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-hand.png index 70fa2d17..1ca434d1 100644 Binary files a/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-hand.png and b/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-hand.png differ diff --git a/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-pen.png b/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-pen.png index 33835316..8ae8dc88 100644 Binary files a/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-pen.png and b/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-pen.png differ diff --git a/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-select.png b/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-select.png index 99bc4a8c..41edacf5 100644 Binary files a/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-select.png and b/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-select.png differ diff --git a/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-shape.png b/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-shape.png index 762db9be..0f376dbc 100644 Binary files a/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-shape.png and b/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-shape.png differ diff --git a/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-text.png b/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-text.png index 2852be83..6e0ad1a4 100644 Binary files a/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-text.png and b/apps/desktop/e2e/__screenshots__/tools.spec.ts/tool-text.png differ diff --git a/apps/desktop/e2e/fixtures/electronApp.ts b/apps/desktop/e2e/fixtures/electronApp.ts index 436d1772..e765838f 100644 --- a/apps/desktop/e2e/fixtures/electronApp.ts +++ b/apps/desktop/e2e/fixtures/electronApp.ts @@ -15,8 +15,8 @@ const MAIN_JS = path.join(APP_ROOT, ".vite/build/main.js"); const FONT_PATH = path.resolve(APP_ROOT, "../../fixtures/fonts/mutatorsans/MutatorSans.ttf"); /** Fixed window size for deterministic snapshots. */ -const WINDOW_WIDTH = 1280; -const WINDOW_HEIGHT = 800; +const WINDOW_WIDTH = 1200; +const WINDOW_HEIGHT = 600; // --------------------------------------------------------------------------- // Fixtures @@ -68,29 +68,32 @@ export const test = base.extend({ try { app = await electron.launch({ - args: [MAIN_JS, `--user-data-dir=${userDataDir}`], + args: [MAIN_JS, `--user-data-dir=${userDataDir}`, "--force-device-scale-factor=1"], env: environment, }); - // Wait for the startup window, then set a fixed size for deterministic tests. - await app.firstWindow(); + // Size the window that owns the test page instead of whichever app window was created first. + const page = await app.firstWindow(); const activeUserDataDir = await app.evaluate(({ app: electronApp }) => electronApp.getPath("userData"), ); if (fs.realpathSync(activeUserDataDir) !== fs.realpathSync(userDataDir)) { throw new Error(`Electron ignored isolated user data directory: ${activeUserDataDir}`); } - await app.evaluate( - async ({ BrowserWindow }, { w, h }) => { - const win = BrowserWindow.getAllWindows()[0]; - if (win) { - win.unmaximize(); - win.setSize(w, h); - win.center(); - } + const browserWindow = await app.browserWindow(page); + await browserWindow.evaluate( + (win, { w, h }) => { + win.unmaximize(); + win.setSize(w, h); + win.center(); }, { w: WINDOW_WIDTH, h: WINDOW_HEIGHT }, ); + await browserWindow.dispose(); + await page.waitForFunction( + ({ w, h }) => window.innerWidth === w && window.innerHeight === h, + { w: WINDOW_WIDTH, h: WINDOW_HEIGHT }, + ); await use(app); } finally { diff --git a/apps/desktop/e2e/glyph-rendering.spec.ts b/apps/desktop/e2e/glyph-rendering.spec.ts index 6ec24764..4bd33377 100644 --- a/apps/desktop/e2e/glyph-rendering.spec.ts +++ b/apps/desktop/e2e/glyph-rendering.spec.ts @@ -2,15 +2,15 @@ * Visual snapshot tests for glyph rendering — covers handles, curves, filled * outlines, and distinct visual styles for on-curve vs off-curve points. * - * Uses MutatorSans "S" (U+0053) which has 44 points with cubic beziers, - * smooth/corner nodes, and off-curve handles — exercising every visual style. + * Uses MutatorSans "S" (U+0053), whose TrueType quadratic contours include + * smooth/corner nodes and off-curve handles — exercising every visual style. */ import { workspaceTest as test, expect, navigateToEditor } from "./fixtures/electronApp"; import { CanvasUtil } from "./fixtures/CanvasUtil"; // MutatorSans glyph codepoints (hex). -const GLYPH_S = "53"; // Complex cubic curves, 44 points +const GLYPH_S = "53"; // Complex quadratic curves const GLYPH_B = "42"; // Mix of curves + straights const GLYPH_I = "49"; // Simple straight segments const GLYPH_Q = "51"; // Counter with curves @@ -19,7 +19,7 @@ const GLYPH_Q = "51"; // Counter with curves // S glyph — full layer snapshots // --------------------------------------------------------------------------- -test.describe("Glyph rendering — S (cubic curves)", () => { +test.describe("Glyph rendering — S (quadratic curves)", () => { test.beforeEach(async ({ page }) => { await navigateToEditor(page, GLYPH_S); }); @@ -153,8 +153,13 @@ test.describe("Pen tool drawing — segment snapshots", () => { await page.mouse.move(750, 250, { steps: 5 }); await page.mouse.up(); - // Segment 3: another straight. - await canvas.click({ position: { x: 850, y: 500 } }); + // Segment 3: another straight, kept inside the measured canvas bounds. + const canvasBounds = await canvas.boundingBox(); + if (!canvasBounds) throw new Error("Expected interactive canvas bounds"); + + await canvas.click({ + position: { x: Math.round(canvasBounds.width * 0.9), y: 500 }, + }); await page.waitForTimeout(300); const canvasUtil = new CanvasUtil(page); diff --git a/apps/desktop/e2e/gpu.spec.ts b/apps/desktop/e2e/gpu.spec.ts index e5c9b892..9c921b95 100644 --- a/apps/desktop/e2e/gpu.spec.ts +++ b/apps/desktop/e2e/gpu.spec.ts @@ -28,6 +28,7 @@ test.describe("Resident catalog GPU", () => { expect(initialSize.width).toBeGreaterThan(1); expect(initialSize.height).toBeGreaterThan(1); await trackSlugFrameSubmits(page); + await trackSlugAtlasLoads(page); const viewportBox = await scrollViewport.boundingBox(); expect(viewportBox).not.toBeNull(); if (!viewportBox) throw new Error("Expected catalog viewport bounds"); @@ -85,10 +86,19 @@ test.describe("Resident catalog GPU", () => { glyphCanvas.evaluate((canvas) => ({ width: canvas.width, height: canvas.height })), ) .toEqual(initialSize); + expect( + await page.evaluate(() => document.documentElement.dataset.slugCompleteAtlasPrepares), + ).toBe("0"); + expect(await page.evaluate(() => document.documentElement.dataset.slugPatchRootCounts)).toBe( + "[]", + ); expect(errors).toEqual([]); }); - test("prioritizes the final viewport while scrubbing the catalog", async ({ page }) => { + test("keeps the complete atlas painted while scrolling downward and upward", async ({ + electronApp, + page, + }) => { const errors: string[] = []; page.on("console", (message) => { if (message.type() === "error" && RESIDENT_GPU_ERROR.test(message.text())) { @@ -97,192 +107,52 @@ test.describe("Resident catalog GPU", () => { }); await expect.poll(() => page.evaluate(() => Boolean(navigator.gpu))).toBe(true); - await page.waitForFunction(() => (window.shift?.font.glyphRecords().length ?? 0) > 0); - await trackSlugPageLoads(page); const scrollViewport = page.getByLabel("Glyph catalog"); - await expect(scrollViewport).toBeVisible(); - const scrollable = await scrollViewport.evaluate( - (element) => element.scrollHeight > element.clientHeight, - ); - test.skip(!scrollable, "Catalog needs more than one viewport"); - - const scrubStarted = performance.now(); - await scrollViewport.evaluate(async (element) => { - const maximum = element.scrollHeight - element.clientHeight; - for (let step = 1; step <= 12; step += 1) { - element.scrollTop = (maximum * step) / 12; - await new Promise((resolve) => requestAnimationFrame(() => resolve())); - } - for (let step = 11; step >= 0; step -= 1) { - element.scrollTop = (maximum * step) / 12; - await new Promise((resolve) => requestAnimationFrame(() => resolve())); - } - }); - - const glyphCanvas = scrollViewport.locator("..").locator("canvas").first(); - await expect(glyphCanvas).toBeVisible({ timeout: 30_000 }); - const scrubDuration = performance.now() - scrubStarted; - const pageLoads = await page.evaluate( - () => JSON.parse(document.documentElement.dataset.slugPageRootCounts ?? "[]") as number[], - ); - console.log( - `Resident catalog scrub recovered in ${scrubDuration.toFixed(0)}ms after ${pageLoads.length} page loads`, - ); - expect(pageLoads.length).toBeLessThanOrEqual(1); - - const finalFrame = await scrollViewport.locator("..").screenshot(); - const visibility = await glyphCanvas.evaluate((canvas) => { - const previous = canvas.style.visibility; - canvas.style.visibility = "hidden"; - return previous; - }); - const frameWithoutGlyphs = await scrollViewport.locator("..").screenshot(); - await glyphCanvas.evaluate((canvas, previous) => { - canvas.style.visibility = previous; - }, visibility); - expect(finalFrame.equals(frameWithoutGlyphs)).toBe(false); - expect(errors).toEqual([]); - }); - - test("keeps the fully resident catalog painted while thumb scrubbing", async ({ - electronApp, - page, - }) => { - test.setTimeout(180_000); - await expect.poll(() => page.evaluate(() => Boolean(navigator.gpu))).toBe(true); - await page.waitForFunction(() => Boolean(window.shift?.font.editCoordinator)); - const glyphCount = await page.evaluate(() => window.shift?.font.glyphRecords().length ?? 0); - test.skip(glyphCount < 300, "Catalog needs a background atlas page"); - - await page.evaluate(() => { - const coordinator = window.shift?.font.editCoordinator; - if (!coordinator) throw new Error("Expected workspace edit coordinator"); - - const originalPrepare = coordinator.prepareSlugAtlasPage.bind(coordinator); - const originalStream = coordinator.streamSlugAtlasPage.bind(coordinator); - let blocked = false; - document.documentElement.dataset.slugPageRootCounts = "[]"; - document.documentElement.dataset.slugPageStreams = "0"; - document.documentElement.dataset.slugBackgroundBlocked = "false"; - coordinator.prepareSlugAtlasPage = async (glyphIds, alignment) => { - const counts = JSON.parse( - document.documentElement.dataset.slugPageRootCounts ?? "[]", - ) as number[]; - counts.push(glyphIds.length); - document.documentElement.dataset.slugPageRootCounts = JSON.stringify(counts); - const descriptor = await originalPrepare(glyphIds, alignment); - if (!blocked && glyphIds.length >= 300) { - blocked = true; - document.documentElement.dataset.slugBackgroundBlocked = "true"; - await new Promise((resolve) => { - window.addEventListener("shift:e2e-release-slug-background", () => resolve(), { - once: true, - }); - }); - } - return descriptor; - }; - coordinator.streamSlugAtlasPage = async (generation, maximumLength, write) => { - document.documentElement.dataset.slugPageStreams = String( - Number(document.documentElement.dataset.slugPageStreams) + 1, - ); - return originalStream(generation, maximumLength, write); - }; - }); - - const scrollViewport = page.getByLabel("Glyph catalog"); - const glyphCanvas = scrollViewport.locator("..").locator("canvas").first(); - await expect(glyphCanvas).toBeVisible({ timeout: 30_000 }); - await expect - .poll(() => page.evaluate(() => document.documentElement.dataset.slugBackgroundBlocked)) - .toBe("true"); - const catalogSurface = scrollViewport.locator(".."); - const initialViewport = await catalogSurface.screenshot(); - const initialVisibility = await glyphCanvas.evaluate((canvas) => { - const previous = canvas.style.visibility; - canvas.style.visibility = "hidden"; - return previous; - }); - const initialViewportWithoutGlyphs = await catalogSurface.screenshot(); - await glyphCanvas.evaluate((canvas, previous) => { - canvas.style.visibility = previous; - }, initialVisibility); - expect(initialViewport.equals(initialViewportWithoutGlyphs)).toBe(false); + const glyphCanvas = catalogSurface.locator("canvas").first(); + await expect(glyphCanvas).toBeVisible({ timeout: 30_000 }); + await expect(glyphCanvas).toHaveAttribute("data-fully-resident", "true"); - await scrollViewport.evaluate((element) => { - element.scrollTop = element.scrollHeight - element.clientHeight; - }); - await expect(glyphCanvas).toBeHidden(); - await page.evaluate(async () => { - const font = window.shift?.font; - const source = font?.sources[0]; - if (!font || !source) throw new Error("Expected a source for version invalidation"); - await font.updateSource({ ...source, name: `${source.name} E2E` }); - window.dispatchEvent(new Event("shift:e2e-release-slug-background")); + await electronApp.evaluate(async ({ BrowserWindow }) => { + BrowserWindow.getAllWindows()[0]?.setSize(760, 500); }); - await expect(glyphCanvas).toBeVisible({ timeout: 2_000 }); + await afterNextPaint(page); await expect - .poll(() => glyphCanvas.getAttribute("data-fully-resident"), { timeout: 120_000 }) - .toBe("true"); - if (glyphCount >= 5_000) { - const residencyPageLoads = await page.evaluate( - () => JSON.parse(document.documentElement.dataset.slugPageRootCounts ?? "[]") as number[], - ); - expect(residencyPageLoads.length).toBeGreaterThan(2); - expect( - await page.evaluate(() => Number(document.documentElement.dataset.slugPageStreams)), - ).toBeGreaterThan(2); - } - + .poll(() => scrollViewport.evaluate((element) => element.scrollHeight > element.clientHeight)) + .toBe(true); + await trackSlugAtlasLoads(page); await page.evaluate(() => { const canvas = document.querySelector( '[aria-label="Glyph catalog"] + canvas', ); if (!canvas) throw new Error("Expected resident glyph canvas"); + document.documentElement.dataset.slugHiddenTransitions = "0"; - document.documentElement.dataset.slugPageRootCounts = "[]"; - document.documentElement.dataset.slugPageStreams = "0"; + document.documentElement.dataset.slugIgnoreHiddenTransitions = "false"; new MutationObserver(() => { - if (canvas.style.visibility !== "hidden") return; + if ( + canvas.style.visibility !== "hidden" || + document.documentElement.dataset.slugIgnoreHiddenTransitions === "true" + ) { + return; + } + document.documentElement.dataset.slugHiddenTransitions = String( Number(document.documentElement.dataset.slugHiddenTransitions) + 1, ); }).observe(canvas, { attributeFilter: ["style"], attributes: true }); }); - await scrollViewport.evaluate(async (element) => { - const maximum = element.scrollHeight - element.clientHeight; - for (let step = 11; step >= 0; step -= 1) { - element.scrollTop = (maximum * step) / 11; - await new Promise((resolve) => requestAnimationFrame(() => resolve())); - } - for (let step = 1; step <= 11; step += 1) { - element.scrollTop = (maximum * step) / 11; - await new Promise((resolve) => requestAnimationFrame(() => resolve())); - } - }); - await afterNextPaint(page); - - await expect(glyphCanvas).toBeVisible(); - await expect - .poll(() => page.evaluate(() => document.documentElement.dataset.slugHiddenTransitions)) - .toBe("0"); - await expect - .poll(() => page.evaluate(() => document.documentElement.dataset.slugPageRootCounts)) - .toBe("[]"); - await expect - .poll(() => page.evaluate(() => document.documentElement.dataset.slugPageStreams)) - .toBe("0"); - - for (const fraction of [0, 0.5, 1]) { + for (const fraction of [0, 0.5, 1, 0.5, 0]) { await scrollViewport.evaluate((element, nextFraction) => { element.scrollTop = (element.scrollHeight - element.clientHeight) * nextFraction; }, fraction); await afterNextPaint(page); + const paintedFrame = await catalogSurface.screenshot(); const visibility = await glyphCanvas.evaluate((canvas) => { + document.documentElement.dataset.slugIgnoreHiddenTransitions = "true"; const previous = canvas.style.visibility; canvas.style.visibility = "hidden"; return previous; @@ -290,38 +160,22 @@ test.describe("Resident catalog GPU", () => { const frameWithoutGlyphs = await catalogSurface.screenshot(); await glyphCanvas.evaluate((canvas, previous) => { canvas.style.visibility = previous; + document.documentElement.dataset.slugIgnoreHiddenTransitions = "false"; }, visibility); expect(paintedFrame.equals(frameWithoutGlyphs)).toBe(false); } - const search = page.getByPlaceholder("Search glyphs..."); - await search.fill("A"); - await expect(glyphCanvas).toHaveAttribute("data-fully-resident", "true"); - await expect(glyphCanvas).toBeVisible(); - await afterNextPaint(page); - await search.fill(""); - await expect(glyphCanvas).toHaveAttribute("data-fully-resident", "true"); - await electronApp.evaluate(async ({ BrowserWindow }) => { - BrowserWindow.getAllWindows()[0]?.setSize(1100, 700); - }); - await afterNextPaint(page); await expect(glyphCanvas).toBeVisible(); - await expect(glyphCanvas).toHaveAttribute("data-fully-resident", "true"); - const resizedFrame = await catalogSurface.screenshot(); - const resizedVisibility = await glyphCanvas.evaluate((canvas) => { - const previous = canvas.style.visibility; - canvas.style.visibility = "hidden"; - return previous; - }); - const resizedFrameWithoutGlyphs = await catalogSurface.screenshot(); - await glyphCanvas.evaluate((canvas, previous) => { - canvas.style.visibility = previous; - }, resizedVisibility); - expect(resizedFrame.equals(resizedFrameWithoutGlyphs)).toBe(false); - expect(await page.evaluate(() => document.documentElement.dataset.slugPageRootCounts)).toBe( - "[]", - ); - expect(await page.evaluate(() => document.documentElement.dataset.slugPageStreams)).toBe("0"); + await expect + .poll(() => page.evaluate(() => document.documentElement.dataset.slugHiddenTransitions)) + .toBe("0"); + await expect + .poll(() => page.evaluate(() => document.documentElement.dataset.slugCompleteAtlasPrepares)) + .toBe("0"); + await expect + .poll(() => page.evaluate(() => document.documentElement.dataset.slugPatchRootCounts)) + .toBe("[]"); + expect(errors).toEqual([]); }); test("restores the current viewport after a topology edit", async ({ page }) => { @@ -334,7 +188,7 @@ test.describe("Resident catalog GPU", () => { await navigateToEditor(page, "53"); await trackSlugFrameSubmits(page); - await trackSlugPageLoads(page); + await trackSlugAtlasLoads(page); const editStarted = performance.now(); await page.evaluate(async () => { @@ -371,29 +225,35 @@ test.describe("Resident catalog GPU", () => { const recoveryDuration = performance.now() - returnStarted; const refreshDuration = performance.now() - editStarted; - const pageLoad = await page.evaluate(() => ({ - counts: JSON.parse(document.documentElement.dataset.slugPageRootCounts ?? "[]") as number[], + const atlasLoads = await page.evaluate(() => ({ + complete: Number(document.documentElement.dataset.slugCompleteAtlasPrepares), + patches: JSON.parse(document.documentElement.dataset.slugPatchRootCounts ?? "[]") as number[], glyphCount: window.shift?.font.glyphRecords().length ?? 0, })); console.log( `Resident catalog topology recovery took ${recoveryDuration.toFixed(0)}ms (${refreshDuration.toFixed(0)}ms from edit)`, ); expect(recoveryDuration).toBeLessThan(1_000); - expect(pageLoad.counts[0]).toBeGreaterThan(0); - expect(pageLoad.counts[0]).toBeLessThan(pageLoad.glyphCount); + expect(atlasLoads.complete).toBe(0); + expect(atlasLoads.patches).toHaveLength(1); + expect(atlasLoads.patches[0]).toBeGreaterThan(0); + expect(atlasLoads.patches[0]).toBeLessThan(atlasLoads.glyphCount); }); - test("prioritizes a distant viewport after a topology edit", async ({ page }) => { + test("keeps distant glyphs resident after a topology patch", async ({ electronApp, page }) => { await expect.poll(() => page.evaluate(() => Boolean(navigator.gpu))).toBe(true); const scrollViewport = page.getByLabel("Glyph catalog"); await scrollViewport.waitFor({ state: "visible" }); const glyphCanvas = scrollViewport.locator("..").locator("canvas").first(); await expect(glyphCanvas).toBeVisible({ timeout: 30_000 }); - const scrollable = await scrollViewport.evaluate( - (element) => element.scrollHeight > element.clientHeight, - ); - test.skip(!scrollable, "Catalog needs more than one viewport"); + await electronApp.evaluate(async ({ BrowserWindow }) => { + BrowserWindow.getAllWindows()[0]?.setSize(760, 500); + }); + await afterNextPaint(page); + await expect + .poll(() => scrollViewport.evaluate((element) => element.scrollHeight > element.clientHeight)) + .toBe(true); await navigateToEditor(page, "53"); await page.evaluate(async () => { @@ -437,7 +297,7 @@ test.describe("Resident catalog GPU", () => { expect(scrollDuration).toBeLessThan(1_000); }); - test("restores the current viewport after deleting a source and axis", async ({ page }) => { + test("rebuilds the complete atlas after a source change", async ({ page }) => { await expect.poll(() => page.evaluate(() => Boolean(navigator.gpu))).toBe(true); const scrollViewport = page.getByLabel("Glyph catalog"); @@ -447,21 +307,15 @@ test.describe("Resident catalog GPU", () => { await navigateToEditor(page, "53"); await trackSlugFrameSubmits(page); - const changed = await page.evaluate(async () => { + await trackSlugAtlasLoads(page); + await page.evaluate(async () => { const font = window.shift?.font; - if (!font || font.getAxes().length === 0 || font.sources.length < 2) return false; - - const source = font.sources.find((candidate) => candidate.id !== font.defaultSource.id); - const axis = font.getAxes().at(-1); - if (!source || !axis) return false; + const source = font?.sources[0]; + if (!font || !source) throw new Error("Expected a source for global invalidation"); - font.deleteSource(source.id); - await font.editCoordinator.settled(); - font.deleteAxis(axis.id); + await font.updateSource({ ...source, name: `${source.name} E2E` }); await font.editCoordinator.settled(); - return true; }); - test.skip(!changed, "Font needs at least one axis and two sources"); const returnStarted = performance.now(); await page.getByRole("button", { name: "Display all glyphs" }).click(); @@ -474,8 +328,13 @@ test.describe("Resident catalog GPU", () => { await expect(glyphCanvas).toBeVisible({ timeout: 30_000 }); const recoveryDuration = performance.now() - returnStarted; - console.log(`Resident catalog source/axis recovery took ${recoveryDuration.toFixed(0)}ms`); + const atlasLoads = await page.evaluate(() => ({ + complete: Number(document.documentElement.dataset.slugCompleteAtlasPrepares), + patches: document.documentElement.dataset.slugPatchRootCounts, + })); + console.log(`Resident catalog source recovery took ${recoveryDuration.toFixed(0)}ms`); expect(recoveryDuration).toBeLessThan(1_000); + expect(atlasLoads).toEqual({ complete: 1, patches: "[]" }); }); }); @@ -497,20 +356,28 @@ async function trackSlugFrameSubmits(page: Page): Promise { }); } -async function trackSlugPageLoads(page: Page): Promise { +async function trackSlugAtlasLoads(page: Page): Promise { await page.evaluate(() => { const coordinator = window.shift?.font.editCoordinator; if (!coordinator) throw new Error("Expected workspace edit coordinator"); - const originalPrepare = coordinator.prepareSlugAtlasPage.bind(coordinator); - document.documentElement.dataset.slugPageRootCounts = "[]"; + const originalCompletePrepare = coordinator.prepareSlugAtlas.bind(coordinator); + const originalPatchPrepare = coordinator.prepareSlugAtlasPage.bind(coordinator); + document.documentElement.dataset.slugCompleteAtlasPrepares = "0"; + document.documentElement.dataset.slugPatchRootCounts = "[]"; + coordinator.prepareSlugAtlas = async (alignment) => { + document.documentElement.dataset.slugCompleteAtlasPrepares = String( + Number(document.documentElement.dataset.slugCompleteAtlasPrepares) + 1, + ); + return originalCompletePrepare(alignment); + }; coordinator.prepareSlugAtlasPage = async (glyphIds, alignment) => { const counts = JSON.parse( - document.documentElement.dataset.slugPageRootCounts ?? "[]", + document.documentElement.dataset.slugPatchRootCounts ?? "[]", ) as number[]; counts.push(glyphIds.length); - document.documentElement.dataset.slugPageRootCounts = JSON.stringify(counts); - return originalPrepare(glyphIds, alignment); + document.documentElement.dataset.slugPatchRootCounts = JSON.stringify(counts); + return originalPatchPrepare(glyphIds, alignment); }; }); } diff --git a/apps/desktop/e2e/home.spec.ts b/apps/desktop/e2e/home.spec.ts index f643e47e..0a20aa9a 100644 --- a/apps/desktop/e2e/home.spec.ts +++ b/apps/desktop/e2e/home.spec.ts @@ -6,6 +6,26 @@ test.describe("Home view", () => { await expect(page).toHaveScreenshot("home-glyph-grid.png"); }); + test("glyph canvas contributes rendered outlines", async ({ page }) => { + const scrollViewport = page.getByLabel("Glyph catalog"); + const catalogSurface = scrollViewport.locator(".."); + const glyphCanvas = catalogSurface.locator("canvas").first(); + await expect(glyphCanvas).toBeVisible({ timeout: 30_000 }); + + const renderedFrame = await catalogSurface.screenshot(); + const visibility = await glyphCanvas.evaluate((canvas) => { + const previous = canvas.style.visibility; + canvas.style.visibility = "hidden"; + return previous; + }); + const frameWithoutGlyphs = await catalogSurface.screenshot(); + await glyphCanvas.evaluate((canvas, previous) => { + canvas.style.visibility = previous; + }, visibility); + + expect(renderedFrame.equals(frameWithoutGlyphs)).toBe(false); + }); + test("keeps the resident grid when returning from the editor", async ({ page }) => { const scrollViewport = page.getByLabel("Glyph catalog"); const glyphCanvas = scrollViewport.locator("..").locator("canvas").first(); diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 27539291..5547bd67 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -15,8 +15,12 @@ "lint:check": "oxlint --deny-warnings src/", "test": "vitest run", "test:watch": "vitest", - "build:e2e": "tsx e2e/build.ts", - "test:e2e": "pnpm build:e2e && pnpm exec playwright test" + "build:e2e": "pnpm --workspace-root generate:glyph-info && tsx e2e/build.ts", + "test:e2e": "pnpm build:e2e && pnpm exec playwright test --project=visual --project=gpu", + "test:e2e:visual": "pnpm build:e2e && pnpm exec playwright test --project=visual", + "test:e2e:visual:update": "pnpm build:e2e && pnpm exec playwright test --project=visual --update-snapshots", + "test:e2e:gpu": "pnpm build:e2e && pnpm exec playwright test --project=gpu", + "test:e2e:perf": "pnpm build:e2e && pnpm exec playwright test --project=perf" }, "author": { "name": "Kostya Farber", diff --git a/apps/desktop/src/renderer/src/components/home/GlyphCatalogController.ts b/apps/desktop/src/renderer/src/components/home/GlyphCatalogController.ts index 1f0a21cc..2751e7d7 100644 --- a/apps/desktop/src/renderer/src/components/home/GlyphCatalogController.ts +++ b/apps/desktop/src/renderer/src/components/home/GlyphCatalogController.ts @@ -17,8 +17,6 @@ import type { } from "@/types/glyphCatalog"; import type { GlyphPreviewInstance } from "@/types/glyphPreview"; -const BACKGROUND_PAGE_SIZE = 2048; - /** Owns catalog DOM events, frame scheduling, layout, and resident rendering. */ export class GlyphCatalogController { readonly #container: HTMLDivElement; @@ -37,7 +35,7 @@ export class GlyphCatalogController { #frame: GlyphCatalogControllerFrame | null = null; #layer: ResidentGlyphLayer | null = null; - /** Single native page operation; aborted work retains this slot until it actually settles. */ + /** Single native atlas operation; aborted work retains this slot until it actually settles. */ #refresh: AbortController | null = null; #pointer: Point2D | null = null; #hoveredCatalogIndex: number | null = null; @@ -77,10 +75,11 @@ export class GlyphCatalogController { void this.#redrawWhenFontsReady(); this.#fontEffect = effect( - () => { - const invalidGlyphIds = font.invalidGlyphIdsCell.value; - this.#invalidate(invalidGlyphIds ?? font.glyphRecords().map((glyph) => glyph.id)); - }, + () => + this.#invalidate( + font.invalidGlyphIdsCell.value, + font.glyphRecords().map((glyph) => glyph.id), + ), { name: "glyphCatalog.residentFont" }, ); this.#startLayer(); @@ -145,11 +144,29 @@ export class GlyphCatalogController { document.fonts.removeEventListener("loadingdone", this.#handleFontsLoaded); } - #invalidate(glyphIds: readonly GlyphId[]): void { - for (const glyphId of glyphIds) this.#invalidGlyphIds.add(glyphId); + #invalidate(glyphIds: readonly GlyphId[] | null, fontGlyphIds: readonly GlyphId[]): void { + if (glyphIds === null) { + this.#invalidGlyphIds.clear(); + this.#refresh?.abort(new Error("resident font changed")); + this.#layer?.destroy(); + this.#layer = null; + this.#glyphCanvas.dataset.fullyResident = "false"; + this.#firstFrameStarted = false; + this.#needsRedraw = true; + this.#onReadyChange(false); + + if (this.#frame?.active) this.#startLayer(); + return; + } + if (glyphIds.length === 0) return; + this.#layer?.invalidate(glyphIds); + const fontGlyphIdSet = new Set(fontGlyphIds); + for (const glyphId of glyphIds) { + if (fontGlyphIdSet.has(glyphId)) this.#invalidGlyphIds.add(glyphId); + } this.#updateFullyResident(); - this.#refresh?.abort(new Error("resident font changed")); + this.#refresh?.abort(new Error("resident glyphs changed")); this.#firstFrameStarted = false; this.#needsRedraw = true; this.#onReadyChange(false); @@ -179,9 +196,12 @@ export class GlyphCatalogController { } this.#layer = layer; + this.#invalidGlyphIds.clear(); this.#updateFullyResident(); this.#refresh = null; - if (this.#frame?.active) await this.#refreshVisible(); + this.#needsRedraw = true; + this.#firstFrameStarted = false; + this.redraw(); } catch (error) { if (this.#disposed || this.#refresh !== refresh) return; this.#refresh = null; @@ -202,25 +222,25 @@ export class GlyphCatalogController { return; } - const pageGlyphIds = this.#viewportPageGlyphIds(); - const missingGlyphIds = pageGlyphIds.filter( - (glyphId) => this.#invalidGlyphIds.has(glyphId) || !layer.hasGlyphs([glyphId]), - ); - if (missingGlyphIds.length === 0) { - if (pageGlyphIds.length === 0) this.#onReadyChange(true); + const missingGlyphIds = new Set(this.#invalidGlyphIds); + for (const cell of this.#currentFrame().cells) { + if (!layer.hasGlyphs([cell.glyph.id])) missingGlyphIds.add(cell.glyph.id); + } + if (missingGlyphIds.size === 0) { + this.#updateFullyResident(); this.redraw(); - if (!this.#refresh) this.#startBackgroundRefresh(pageGlyphIds); return; } + const glyphIds = [...missingGlyphIds]; const refresh = new AbortController(); this.#refresh = refresh; this.#onReadyChange(false); try { - await layer.loadPage(missingGlyphIds, refresh.signal); + await layer.loadPatch(glyphIds, refresh.signal); if (this.#disposed || this.#refresh !== refresh) return; - for (const glyphId of missingGlyphIds) this.#invalidGlyphIds.delete(glyphId); + for (const glyphId of glyphIds) this.#invalidGlyphIds.delete(glyphId); this.#updateFullyResident(); this.#refresh = null; this.#needsRedraw = true; @@ -237,70 +257,6 @@ export class GlyphCatalogController { } } - #startBackgroundRefresh(pageGlyphIds: readonly GlyphId[]): void { - const refresh = new AbortController(); - this.#refresh = refresh; - void this.#refreshRemaining(refresh, pageGlyphIds).catch((error: unknown) => { - if (this.#disposed || this.#refresh !== refresh) return; - this.#refresh = null; - if (refresh.signal.aborted) { - if (this.#frame?.active) void this.#refreshVisible(); - return; - } - if (this.#layer) this.#failFrame(this.#layer, error); - }); - } - - async #refreshRemaining( - refresh: AbortController, - pageGlyphIds: readonly GlyphId[], - ): Promise { - const layer = this.#layer; - const input = this.#frame; - if (!layer || !input) return; - - const pageSet = new Set(pageGlyphIds); - const remaining = input.glyphs - .map((glyph) => glyph.id) - .filter( - (glyphId) => - !pageSet.has(glyphId) && - (this.#invalidGlyphIds.has(glyphId) || !layer.hasGlyphs([glyphId])), - ); - const ordered = outwardGlyphIds(input.glyphs, remaining, pageGlyphIds); - - let visibleNeedsRefresh = false; - for (let offset = 0; offset < ordered.length; offset += BACKGROUND_PAGE_SIZE) { - refresh.signal.throwIfAborted(); - const visibleGlyphIds = this.#viewportPageGlyphIds(); - visibleNeedsRefresh = visibleGlyphIds.some( - (glyphId) => this.#invalidGlyphIds.has(glyphId) || !layer.hasGlyphs([glyphId]), - ); - if (visibleNeedsRefresh) break; - - const glyphIds = ordered.slice(offset, offset + BACKGROUND_PAGE_SIZE); - await layer.loadPage(glyphIds, refresh.signal); - if (this.#disposed || this.#refresh !== refresh) return; - for (const glyphId of glyphIds) this.#invalidGlyphIds.delete(glyphId); - this.#updateFullyResident(); - this.#needsRedraw = true; - this.#firstFrameStarted = false; - this.redraw(); - - visibleNeedsRefresh = this.#viewportPageGlyphIds().some( - (glyphId) => this.#invalidGlyphIds.has(glyphId) || !layer.hasGlyphs([glyphId]), - ); - if (visibleNeedsRefresh) break; - } - - if (this.#refresh !== refresh) return; - this.#refresh = null; - if (visibleNeedsRefresh) { - void this.#refreshVisible(); - this.redraw(); - } - } - #handleDeviceLoss(reason: string): void { if (this.#disposed) return; console.error("resident glyph device lost", reason); @@ -326,10 +282,6 @@ export class GlyphCatalogController { return layout.frame(this.#frame?.glyphs ?? [], this.#container.scrollTop); } - #viewportPageGlyphIds(): GlyphId[] { - return viewportPageGlyphIds(this.#frame?.glyphs ?? [], this.#currentFrame()); - } - #draw(): void { if (this.#disposed) return; const input = this.#frame; @@ -424,7 +376,6 @@ export class GlyphCatalogController { return; } this.#onReadyChange(true); - if (!this.#refresh) this.#startBackgroundRefresh(this.#viewportPageGlyphIds()); } catch (error) { if (this.#disposed || this.#layer !== layer) return; this.#failFrame(layer, error); @@ -519,48 +470,3 @@ export class GlyphCatalogController { } } } - -function viewportPageGlyphIds( - glyphs: readonly GlyphCatalogItem[], - frame: GlyphCatalogFrame, -): GlyphId[] { - if (frame.cells.length === 0) return []; - - const firstIndex = frame.cells[0]!.catalogIndex; - const lastIndex = frame.cells.at(-1)!.catalogIndex; - const viewportCount = lastIndex - firstIndex + 1; - const start = Math.max(0, firstIndex - viewportCount); - const end = Math.min(glyphs.length, lastIndex + viewportCount + 1); - return glyphs.slice(start, end).map((glyph) => glyph.id); -} - -function outwardGlyphIds( - glyphs: readonly GlyphCatalogItem[], - remainingGlyphIds: readonly GlyphId[], - pageGlyphIds: readonly GlyphId[], -): GlyphId[] { - const remaining = new Set(remainingGlyphIds); - const firstPageIndex = - pageGlyphIds.length > 0 ? glyphs.findIndex((glyph) => glyph.id === pageGlyphIds[0]) : 0; - const lastPageGlyphId = pageGlyphIds.at(-1); - const lastPageIndex = lastPageGlyphId - ? glyphs.findIndex((glyph) => glyph.id === lastPageGlyphId) - : firstPageIndex; - const ordered: GlyphId[] = []; - let before = firstPageIndex - 1; - let after = lastPageIndex + 1; - - while (before >= 0 || after < glyphs.length) { - for (let count = 0; count < BACKGROUND_PAGE_SIZE && after < glyphs.length; count += 1) { - const glyphId = glyphs[after++]!.id; - if (remaining.delete(glyphId)) ordered.push(glyphId); - } - for (let count = 0; count < BACKGROUND_PAGE_SIZE && before >= 0; count += 1) { - const glyphId = glyphs[before--]!.id; - if (remaining.delete(glyphId)) ordered.push(glyphId); - } - } - - ordered.push(...remaining); - return ordered; -} diff --git a/apps/desktop/src/renderer/src/lib/graphics/backends/ResidentGlyphLayer.ts b/apps/desktop/src/renderer/src/lib/graphics/backends/ResidentGlyphLayer.ts index 62ff4d33..0ae9d3b2 100644 --- a/apps/desktop/src/renderer/src/lib/graphics/backends/ResidentGlyphLayer.ts +++ b/apps/desktop/src/renderer/src/lib/graphics/backends/ResidentGlyphLayer.ts @@ -6,10 +6,11 @@ import { SlugRenderer } from "@/lib/slug/SlugRenderer"; const COPY_ALIGNMENT = 256; const UPLOAD_CHUNK_BYTES = 4 * 1024 * 1024; -const PREFERRED_PAGE_BYTES = 64 * 1024 * 1024; +const PREFERRED_ATLAS_BYTES = 256 * 1024 * 1024; +const PREFERRED_PATCH_BYTES = 64 * 1024 * 1024; const REQUIRED_STORAGE_BUFFERS = 8; -/** Generic paged glyph-preview surface backed by the current Slug implementation. */ +/** Complete resident glyph-preview surface with incremental edit patches. */ export class ResidentGlyphLayer { readonly #renderer: SlugRenderer; readonly #device: GPUDevice; @@ -39,10 +40,20 @@ export class ResidentGlyphLayer { ): Promise { if (!navigator.gpu) throw new Error("WebGPU is unavailable"); + let preparedGeneration: number | null = null; let device: GPUDevice | null = null; let context: GPUCanvasContext | null = null; + let atlas: SlugAtlas | null = null; let renderer: SlugRenderer | null = null; + async function discardPrepared(): Promise { + if (preparedGeneration === null) return; + + const generation = preparedGeneration; + preparedGeneration = null; + await edits.discardSlugAtlas(generation); + } + try { const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" }); throwIfAborted(signal); @@ -53,8 +64,34 @@ export class ResidentGlyphLayer { ); } + const alignment = Math.max(COPY_ALIGNMENT, adapter.limits.minStorageBufferOffsetAlignment); + const descriptor = await edits.prepareSlugAtlas(alignment); + preparedGeneration = descriptor.generation; + throwIfAborted(signal); + + const adapterMaximumBindingSize = Math.min( + adapter.limits.maxBufferSize, + adapter.limits.maxStorageBufferBindingSize, + ); + const [, firstLength, secondLength] = SlugAtlas.bindingLengths( + descriptor.layout.totalLength, + adapterMaximumBindingSize, + ); + const largestResidentBuffer = Math.max(firstLength, secondLength); + if (descriptor.layout.totalLength > PREFERRED_ATLAS_BYTES) { + console.warn("resident glyph atlas exceeds preferred size", { + bytes: descriptor.layout.totalLength, + preferredBytes: PREFERRED_ATLAS_BYTES, + adapterMaximumBytes: adapterMaximumBindingSize * 2, + }); + } + device = await adapter.requestDevice({ - requiredLimits: { maxStorageBuffersPerShaderStage: REQUIRED_STORAGE_BUFFERS }, + requiredLimits: { + maxBufferSize: largestResidentBuffer, + maxStorageBufferBindingSize: largestResidentBuffer, + maxStorageBuffersPerShaderStage: REQUIRED_STORAGE_BUFFERS, + }, }); throwIfAborted(signal); @@ -63,11 +100,33 @@ export class ResidentGlyphLayer { const format = navigator.gpu.getPreferredCanvasFormat(); context.configure({ device, format, alphaMode: "premultiplied" }); renderer = new SlugRenderer(device, context, format, onDeviceLost); - const alignment = Math.max(COPY_ALIGNMENT, device.limits.minStorageBufferOffsetAlignment); const maximumBindingSize = Math.min( device.limits.maxBufferSize, device.limits.maxStorageBufferBindingSize, ); + atlas = SlugAtlas.create(descriptor, device, maximumBindingSize); + const activeAtlas = atlas; + const uploadDevice = device; + const totalLength = await edits.streamSlugAtlas( + descriptor.generation, + UPLOAD_CHUNK_BYTES, + (offset, bytes) => { + throwIfAborted(signal); + activeAtlas.write(uploadDevice.queue, offset, bytes); + }, + ); + preparedGeneration = null; + throwIfAborted(signal); + if (totalLength !== descriptor.layout.totalLength) { + throw new Error( + `resident glyph stream wrote ${totalLength} bytes; expected ${descriptor.layout.totalLength}`, + ); + } + + const loadedAtlas = atlas; + atlas = null; + renderer.loadPage(loadedAtlas); + throwIfAborted(signal); const layer = new ResidentGlyphLayer(renderer, device, edits, alignment, maximumBindingSize); renderer = null; @@ -76,34 +135,37 @@ export class ResidentGlyphLayer { return layer; } catch (error) { renderer?.destroy(); + if (renderer) { + context = null; + device = null; + } + atlas?.destroy(); context?.unconfigure(); device?.destroy(); + try { + await discardPrepared(); + } catch (discardError) { + console.error("failed to release rejected resident glyph atlas", discardError); + } throw error; } } - async loadPage(glyphIds: readonly GlyphId[], signal: AbortSignal): Promise { + async loadPatch(glyphIds: readonly GlyphId[], signal: AbortSignal): Promise { if (glyphIds.length === 0) return; let preparedGeneration: number | null = null; let atlas: SlugAtlas | null = null; - async function discardPrepared( - edits: WorkspaceEditCoordinator, - generation: number, - ): Promise { - await edits.discardSlugAtlasPage(generation); - } - try { const descriptor = await this.#edits.prepareSlugAtlasPage(glyphIds, this.#alignment); preparedGeneration = descriptor.generation; throwIfAborted(signal); - if (descriptor.layout.totalLength > PREFERRED_PAGE_BYTES) { - console.warn("resident glyph atlas page exceeds preferred size", { + if (descriptor.layout.totalLength > PREFERRED_PATCH_BYTES) { + console.warn("resident glyph atlas patch exceeds preferred size", { bytes: descriptor.layout.totalLength, - preferredBytes: PREFERRED_PAGE_BYTES, + preferredBytes: PREFERRED_PATCH_BYTES, }); } @@ -121,7 +183,7 @@ export class ResidentGlyphLayer { throwIfAborted(signal); if (totalLength !== descriptor.layout.totalLength) { throw new Error( - `resident glyph page stream wrote ${totalLength} bytes; expected ${descriptor.layout.totalLength}`, + `resident glyph patch stream wrote ${totalLength} bytes; expected ${descriptor.layout.totalLength}`, ); } @@ -132,9 +194,9 @@ export class ResidentGlyphLayer { atlas?.destroy(); if (preparedGeneration !== null) { try { - await discardPrepared(this.#edits, preparedGeneration); + await this.#edits.discardSlugAtlasPage(preparedGeneration); } catch (discardError) { - console.error("failed to release rejected resident glyph atlas page", discardError); + console.error("failed to release rejected resident glyph atlas patch", discardError); } } throw error; diff --git a/apps/desktop/src/renderer/src/lib/graphics/docs/DOCS.md b/apps/desktop/src/renderer/src/lib/graphics/docs/DOCS.md index 4ca6b551..f1dbb4ea 100644 --- a/apps/desktop/src/renderer/src/lib/graphics/docs/DOCS.md +++ b/apps/desktop/src/renderer/src/lib/graphics/docs/DOCS.md @@ -8,9 +8,9 @@ Renderer vector-path values and the accelerated marker-layer backend for editor - **Architecture Invariant:** `Renderer` owns the `MarkerLayer` lifecycle. `CanvasContextProvider` only reports DOM canvas mount, resize, and unmount events. -- **Architecture Invariant:** `ResidentGlyphLayer` is the generic catalog-preview boundary. It owns one WebGPU adapter/device/context and independently replaceable atlas pages; Slug names and packed-layout knowledge remain behind that backend rather than entering React or catalog frame types. +- **Architecture Invariant:** `ResidentGlyphLayer` is the generic catalog-preview boundary. It owns one WebGPU adapter/device/context, one complete base atlas, and independently replaceable edit patches; Slug names and packed-layout knowledge remain behind that backend rather than entering React or catalog frame types. -- **Architecture Invariant:** Catalog route activity and GPU readiness are independent. Leaving `/home` makes the catalog inert and keeps it painted behind the opaque editor without destroying the resident layer or resizing its canvas. Returning submits one cheap redraw from current resident pages because Chromium may discard the WebGPU canvas presentation; it does not compile or upload those pages. Readiness means the current viewport plus overscan is current and its submitted frame completed. +- **Architecture Invariant:** Catalog route activity and GPU readiness are independent. Leaving `/home` makes the catalog inert and keeps it painted behind the opaque editor without destroying the resident layer or resizing its canvas. Returning submits one cheap redraw because Chromium may discard the WebGPU canvas presentation. Readiness means the complete atlas, including any required local edit patches, is current and its submitted frame completed. - **Architecture Invariant:** **CRITICAL**: The instance buffer layout (attribute offsets in the draw command) must exactly match the packing order in `MarkerHandleRenderer.#writeInstance`. If either side changes stride/offset, handles render garbage with no error. @@ -28,7 +28,7 @@ graphics/ canvasText.ts — width-constrained Canvas2D label fitting backends/ MarkerLayer.ts — WebGL context: REGL init, instance buffer management, draw command - ResidentGlyphLayer.ts — WebGPU catalog device, page upload, draw, invalidation, and teardown boundary + ResidentGlyphLayer.ts — WebGPU catalog device, complete upload, edit patches, draw, and teardown ``` Supporting files live in the editor rendering module: @@ -51,7 +51,7 @@ editor/rendering/markers/ - `MarkerLayer` -- WebGL context wrapper. Manages REGL instance, instance buffer, and draw command. Provides `resizeCanvas`, `draw`, `clear`, `destroy`, and `isAvailable`. -- `ResidentGlyphLayer` -- algorithm-neutral surface used by the catalog controller. It keeps one device/context resident, prepares and streams native Slug pages, invalidates roots independently, and delegates multi-page rendering to the internal Slug backend. +- `ResidentGlyphLayer` -- algorithm-neutral surface used by the catalog controller. It prepares and streams one complete native Slug atlas, retains one device/context, and overlays small replacement patches for locally invalidated roots. - `MarkerInstance` -- logical representation of one marker shape. The current marker path packs directly into a `Float32Array` for zero steady-state allocation. @@ -73,7 +73,7 @@ editor/rendering/markers/ ### Resident catalog lifecycle -`GlyphCatalogController` retains `ResidentGlyphLayer` across routes and tracks `Font.invalidGlyphIdsCell`. Local edits invalidate only touched roots and component dependents; axis/source changes invalidate every root. The remembered viewport plus one viewport of overscan loads as the first page, then 2,048-root pages populate outward in both directions. At most one native page compilation is in flight: scrolling redraws already-resident roots immediately, and when that compilation settles the scheduler rechecks the latest viewport before starting anything else. Every published background page schedules a current-viewport redraw, even when that publication itself satisfies a viewport that became visible while the page was in flight. Intermediate scrub positions therefore cannot queue native compilation work. Once all pages are resident, `GlyphCatalogController` marks the glyph canvas `data-fully-resident="true"`; thumb scrubbing then neither hides the canvas nor prepares or uploads atlas data. Invalidation clears full residency until every current catalog root is uploaded again. Invalid or missing roots are never submitted, so stale previews are not displayed. Route-dependent navigation is accessed through a stable callback ref so it cannot recreate the controller or device. `#needsRedraw` keeps overlay-only pointer updates from submitting glyph frames. +`GlyphCatalogController` retains `ResidentGlyphLayer` across routes and tracks `Font.invalidGlyphIdsCell`. Initial residency acquires every authored layer, streams one complete atlas, and marks the glyph canvas `data-fully-resident="true"`. Scrolling in either direction therefore only submits frames and never prepares or uploads geometry. Local edits invalidate touched roots and component dependents, then stream one replacement patch while untouched roots remain in the complete base atlas. Axis/source changes discard the complete atlas and rebuild it because every root is invalid. Invalid or missing roots are never submitted, so stale previews are not displayed. Route-dependent navigation is accessed through a stable callback ref so it cannot recreate the controller or device. `#needsRedraw` keeps overlay-only pointer updates from submitting glyph frames. ### Per-frame draw pipeline @@ -120,7 +120,7 @@ Set a breakpoint or add logging in `MarkerLayer.draw`. Check `isAvailable()` ret ## Verification -- Run `pnpm --filter @shift/desktop test:e2e -- --project=gpu e2e/gpu.spec.ts` after changing catalog residency or scheduling. The suite blocks background publication to verify visible-first painting, verifies that publication restores a viewport reached while the page was in flight, and samples top/middle/bottom screenshots while asserting that a fully resident atlas never hides, prepares, or uploads during thumb scrubbing. +- See [Desktop E2E tests](../../../../../../e2e/README.md) for visual and hardware-GPU verification commands. - Run `pnpm test:desktop apps/desktop/src/renderer/src/lib/graphics/ContourPath.test.ts` after changing contour command, transform, SVG, Canvas, or bounds behavior. - No dedicated tests exist for `MarkerLayer`. Verify GPU markers visually: open a glyph with mixed point types (corner, smooth, off-curve), hover and select points, confirm correct shapes and state colors. - Check CPU fallback: in `MarkerLayer.#initialize`, temporarily throw before `this.#available = true`. Handles should still render via Canvas 2D. diff --git a/apps/desktop/src/renderer/src/lib/model/Font.ts b/apps/desktop/src/renderer/src/lib/model/Font.ts index 685dd016..f9375869 100644 --- a/apps/desktop/src/renderer/src/lib/model/Font.ts +++ b/apps/desktop/src/renderer/src/lib/model/Font.ts @@ -473,7 +473,7 @@ export class Font { return this.#committedFontCell; } - /** Glyph roots whose resident atlas pages no longer match the committed font. */ + /** Glyph roots whose resident atlas entries no longer match the committed font. */ get invalidGlyphIdsCell(): Signal { return this.#store.invalidGlyphIdsCell; } diff --git a/apps/desktop/src/renderer/src/lib/model/FontStore.ts b/apps/desktop/src/renderer/src/lib/model/FontStore.ts index 2208ff4f..dd11152e 100644 --- a/apps/desktop/src/renderer/src/lib/model/FontStore.ts +++ b/apps/desktop/src/renderer/src/lib/model/FontStore.ts @@ -97,7 +97,7 @@ export class FontStore { return this.#committedFont; } - /** Glyph roots whose resident atlas pages no longer match the committed font. */ + /** Glyph roots whose resident atlas entries no longer match the committed font. */ get invalidGlyphIdsCell(): Signal { return this.#invalidGlyphIds; } diff --git a/apps/desktop/src/renderer/src/lib/slug/SlugRenderer.ts b/apps/desktop/src/renderer/src/lib/slug/SlugRenderer.ts index 29158cc0..93df34fe 100644 --- a/apps/desktop/src/renderer/src/lib/slug/SlugRenderer.ts +++ b/apps/desktop/src/renderer/src/lib/slug/SlugRenderer.ts @@ -4,7 +4,7 @@ import { SlugAtlas } from "./SlugAtlas"; import { SlugAtlasPage } from "./SlugAtlasPage"; import { SlugRendererPipelines } from "./SlugRendererResources"; -/** Concrete paged Slug implementation behind the resident glyph preview layer. */ +/** Complete Slug atlas renderer with independently replaceable glyph patches. */ export class SlugRenderer { readonly #device: GPUDevice; readonly #context: GPUCanvasContext; diff --git a/apps/desktop/src/renderer/src/perf/workspaceApply.bench.ts b/apps/desktop/src/renderer/src/perf/workspaceApply.bench.ts index ef03861b..3986478e 100644 --- a/apps/desktop/src/renderer/src/perf/workspaceApply.bench.ts +++ b/apps/desktop/src/renderer/src/perf/workspaceApply.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe } from "vitest"; +import { afterAll, bench, describe } from "vitest"; import type { GlyphName, PointType, Unicode } from "@shift/types"; import { mintContourId, mintGlyphId, mintLayerId, mintPointId } from "@shift/types"; import { createWorkspaceStack } from "@/testing/workspaceStack"; @@ -60,6 +60,74 @@ function squareIntents(width: number) { // Seed one contour so undo/redo and pulls always have real geometry. await stack.editCoordinator.apply([...squareIntents(100)]); +const LARGE_POINT_COUNT = 50_000; +const largeStack = createWorkspaceStack(); +await largeStack.createWorkspace(); +const largeGlyphId = mintGlyphId(); +const largeLayerId = mintLayerId(); +const largeContourId = mintContourId(); +const largePointIds = Array.from({ length: LARGE_POINT_COUNT }, () => mintPointId()); +const largePoints = largePointIds.map((id, index) => ({ + id, + x: index % 500, + y: Math.floor(index / 500), + pointType: "onCurve" as PointType, + smooth: false, +})); +const largeCreated = await largeStack.editCoordinator.apply([ + { + kind: "createGlyph", + createGlyph: { glyphId: largeGlyphId, name: "large" as GlyphName, unicodes: [] }, + }, + { + kind: "createGlyphLayer", + createGlyphLayer: { + layerId: largeLayerId, + glyphId: largeGlyphId, + sourceId: largeStack.font.defaultSource.id, + }, + }, + { + kind: "addContour", + addContour: { layerId: largeLayerId, contourId: largeContourId, closed: true }, + }, + { + kind: "addPoints", + addPoints: { + layerId: largeLayerId, + contourId: largeContourId, + points: largePoints, + }, + }, +]); +if (largeCreated.layers[0]?.structure?.contours[0]?.points.length !== LARGE_POINT_COUNT) { + throw new Error("50K benchmark setup did not create the complete contour"); +} +await largeStack.editCoordinator.apply([ + { + kind: "movePoints", + movePoints: { + layerId: largeLayerId, + pointIds: largePointIds, + coords: largePoints.flatMap((point) => [point.x + 1, point.y - 1]), + }, + }, +]); + +const largeUndoSamples: number[] = []; +const largeRedoSamples: number[] = []; + +function percentile(samples: readonly number[], fraction: number): number { + const sorted = [...samples].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length * fraction)] ?? 0; +} + +afterAll(() => { + console.log( + `50K position replay: undo p50=${percentile(largeUndoSamples, 0.5).toFixed(2)}ms p95=${percentile(largeUndoSamples, 0.95).toFixed(2)}ms; redo p50=${percentile(largeRedoSamples, 0.5).toFixed(2)}ms p95=${percentile(largeRedoSamples, 0.95).toFixed(2)}ms`, + ); +}); + describe("workspace apply round trip (channel + NAPI + SQLite)", () => { let width = 0; @@ -83,4 +151,30 @@ describe("workspace apply round trip (channel + NAPI + SQLite)", () => { bench("replace-grade glyph state pull", async () => { await stack.editCoordinator.readGlyphSnapshots([{ glyphId }]); }); + + bench( + "50K point position-only undo + redo", + async () => { + let started = performance.now(); + const undone = await largeStack.editCoordinator.undo(); + largeUndoSamples.push(performance.now() - started); + if (!undone || undone.layers.length !== 1) { + throw new Error("50K position edit did not undo exactly one layer"); + } + if (undone.layers[0].structure) { + throw new Error("50K position undo crossed the bridge with structure"); + } + + started = performance.now(); + const redone = await largeStack.editCoordinator.redo(); + largeRedoSamples.push(performance.now() - started); + if (!redone || redone.layers.length !== 1) { + throw new Error("50K position edit did not redo exactly one layer"); + } + if (redone.layers[0].structure) { + throw new Error("50K position redo crossed the bridge with structure"); + } + }, + { iterations: 10, time: 0, warmupIterations: 0, warmupTime: 0 }, + ); }); diff --git a/crates/shift-backends/Cargo.toml b/crates/shift-backends/Cargo.toml index 6fa0dfc0..81357be0 100644 --- a/crates/shift-backends/Cargo.toml +++ b/crates/shift-backends/Cargo.toml @@ -22,6 +22,7 @@ kurbo = "0.12" ordered-float = "5" plist = "1" quick-xml = { version = "0.37.5", features = ["serialize"] } +rayon = "1.10" serde = "1" tempfile = "3" thiserror = "2.0.18" diff --git a/crates/shift-backends/docs/DOCS.md b/crates/shift-backends/docs/DOCS.md index f01836ce..3d5ff54a 100644 --- a/crates/shift-backends/docs/DOCS.md +++ b/crates/shift-backends/docs/DOCS.md @@ -4,11 +4,11 @@ Font format backends that convert between on-disk font files and the `Font` IR u ## Architecture Invariants -**Architecture Invariant:** All backends convert to/from `Font` (the shift-font representation), never exposing format-specific types (norad, glyphs-reader) to callers. WHY: The rest of the editor operates on a single IR; leaking format types would couple the editor to specific file formats. +**Architecture Invariant:** All backends convert to/from `shift-font` values, never exposing format-specific types (norad, glyphs-reader) to callers. Eager readers return `Font`; bounded foreign imports return a glyph-free `Font` header plus batches of owned `Glyph` values. WHY: The rest of the editor operates on one authored model while large imports avoid constructing the complete model at once. **Architecture Invariant:** `FontReader` and `FontWriter` require `Send + Sync`. WHY: Backends are stored in `FontLoader` which lives inside the editor's shared state; they must be safe to use from multiple threads. -**Architecture Invariant:** Backends are stateless unit structs (no fields). WHY: They are pure converters with no caching or mutable state, making them trivially thread-safe and cheap to construct. +**Architecture Invariant:** Eager reader/writer backends are stateless unit structs. A `FontImport` owns only the foreign bytes or GLIF directory records and its current cursor. WHY: ordinary conversion stays pure, while bounded imports retain only the state needed to produce the next batch. **Architecture Invariant:** `UfoWriter` stages a complete UFO beside the destination and swaps it into place only after the staged tree is durable. WHY: a failed save must preserve the previous source rather than leave a partial directory. @@ -20,23 +20,35 @@ Font format backends that convert between on-disk font files and the `Font` IR u **Architecture Invariant:** TrueType export compiles an owned snapshot of the Shift `Font` IR directly through fontir/fontc. It must not serialize a temporary UFO or fall back to another authoring format. WHY: `.shift` is the canonical authoring source, and an intermediate format would discard or reinterpret Shift concepts before compilation. -**Architecture Invariant:** Designspace source locations are imported as complete design-space locations. An omitted source dimension resolves to that axis's user-space default mapped into design space; default-source selection compares against that same completed mapped location and never silently substitutes the first source. A `layer` attribute only selects where a source's outlines live and does not make it ineligible to be the default. WHY: these are the reference designspaceLib semantics, and mixing user defaults with design coordinates corrupts interpolation bases. +**Architecture Invariant:** TTF/OTF, UFO, and Designspace streaming imports parse glyphs in bounded Rayon batches and preserve input order when publishing each batch. Eager readers drain those same canonical streams rather than maintaining a second parser. UFO and Designspace share `GlifGlyphStream`; only source discovery differs. SQLite remains outside this crate and is written by one workspace-owned sink. WHY: one conversion path prevents eager/streaming semantic drift, while concurrent SQLite authors would add contention and weaken transaction ownership. + +**Architecture Invariant:** Compiled-font streaming enumerates `maxp` glyph IDs, not only `cmap` mappings. Unencoded glyphs receive their `post`/CFF name or a synthesized `gidN` name, and all Unicode mappings for a glyph share one authored glyph identity. WHY: `cmap` is character lookup, not the complete glyph directory. + +**Architecture Invariant:** Compiled-font contour and point identities are deterministic positions within one glyph's emitted outline: `contour_b{glyph-id-hex}_{contour-index-hex}` and `point_b{glyph-id-hex}_{point-index-hex}`. Counters are monotonic from zero; dropping an explicit closing endpoint does not reuse its consumed point index. WHY: compiled fonts have stable glyph/outline order but no Shift identities, and random IDs make equivalent re-imports differ while injecting incompressible entropy into canonical payloads. + +**Architecture Invariant:** TrueType quadratic segments remain one `OffCurve` control plus one `QCurve` endpoint in the authored layer. Closing qcurve endpoints transfer their type to the wrapped start point. CFF cubic segments remain cubic. The bridge may project a qcurve endpoint as on-curve because clients infer the quadratic from its single control; canonical storage and source export retain the distinction. WHY: lifting every TrueType quadratic to cubic adds a point and derived coordinates to every segment, inflating canonical documents without adding information. + +**Architecture Invariant:** Designspace source locations are imported as complete design-space locations. An omitted source dimension resolves to that axis's user-space default mapped into design space; default-source selection compares against that same completed mapped location and never silently substitutes the first source. A `layer` attribute only selects where a source's outlines live and does not make it ineligible to be the default. Each source's standard metrics are translated from that UFO's metric identities into the Designspace header definitions. WHY: mixing user defaults with design coordinates corrupts interpolation bases, while looking up one UFO's metrics with another UFO's random IDs silently drops non-default master metrics. ## Codemap ``` src/ - lib.rs -- re-exports FontReader, FontWriter, FontBackend, and sub-modules + lib.rs -- re-exports FontReader, FontWriter, FontBackend, FontImport, and sub-modules + import.rs -- glyph-free foreign header, bounded cursor, and shared GLIF stream traits.rs -- FontReader, FontWriter, FontBackend trait definitions ufo/ mod.rs -- UfoBackend convenience struct combining reader+writer; round-trip tests - reader.rs -- UfoReader: norad::Font -> shift_font::Font + import.rs -- UFO source discovery configured into the shared GLIF stream + reader.rs -- UfoReader eagerly drains the canonical UFO stream writer.rs -- UfoWriter: shift_font::Font -> atomically written norad::Font glyphs/ mod.rs -- GlyphsReader re-export; fixture-based integration tests reader.rs -- GlyphsReader: glyphs_reader::Font -> shift_font::Font (read-only) + designspace/ + import.rs -- Designspace source discovery configured into the shared GLIF stream binary/ - reader.rs -- TTF/OTF outlines, metrics, names, and fvar metadata -> shift_font::Font + reader.rs -- maxp-complete TTF/OTF stream plus eager stream draining shift2fontir/ source.rs -- owned Shift FontView snapshot and fontir Source implementation axes.rs -- Shift axis/mapping conversion and source normalization @@ -49,6 +61,9 @@ src/ ## Key Types +- `FontImport` -- top-level authored header, an immediately publishable stable-ID/name directory, and layer-aware `next_batch(limit)`, with no glyphs stored in the header +- `GlyphDirectoryEntry` -- cheap foreign glyph ID and name used before geometry batches are parsed +- `ImportBatchLimit` -- simultaneous glyph and authored-layer limits; multi-source projects cannot turn a glyph-count bound into an unbounded layer batch - `FontReader` -- trait with `load(&self, path) -> Result` plus default methods for extracting glyphs, kerning, features from a loaded `Font` - `FontWriter` -- trait with `save(&self, font, path) -> Result<(), String>` - `FontBackend` -- auto-implemented marker trait for types implementing both `FontReader` + `FontWriter` @@ -61,13 +76,13 @@ src/ ## How it works -**Loading a font:** `FontLoader` (in shift-core) dispatches by file extension to the appropriate backend. The backend reads the file using a format-specific library (`norad` for UFO, `glyphs-reader` for Glyphs), then walks the parsed data to build a `Font`. This involves converting point types, contours, components, anchors, guidelines, kerning groups/pairs, OpenType features, and lib data into their IR equivalents. +**Loading a font:** `FontLoader::read_font` retains the eager API but the TTF/OTF, UFO, and Designspace readers implement it by draining their bounded streams. `FontLoader::stream_font` dispatches those sources to the same importers. It first returns complete top-level metadata and a cheap glyph/source directory, then materializes at most the requested batch of `Glyph` values. UFO and Designspace both feed shared GLIF work records into `GlifGlyphStream`; Designspace only adds stable multi-source discovery. Rayon converts geometry records in parallel; indexed collection preserves glyph order. The workspace writes and releases each batch before requesting another. **Point type mapping (read):** norad uses separate `Move`, `Line`, `Curve`, `OffCurve`, `QCurve` types. The IR collapses `Move`/`Line`/`Curve` into `OnCurve` and keeps `OffCurve` and `QCurve` distinct. On write, context (position in contour, open/closed, preceding point type) is used to reconstruct the correct norad variant. -**Multi-layer support:** `UfoReader` iterates all norad layers. The `public.default` layer maps to the IR's default layer; other layers are added via `Font::add_layer`. Glyphs in non-default layers are merged into existing `Glyph` entries when the glyph already exists from another layer. +**Multi-layer support:** `UfoReader` publishes `public.default` first, then preserves the relative authored order of every other entry in `layercontents.plist`. The default layer maps to the IR's default layer; other layers are represented by layer sources. Glyphs in non-default layers are merged into existing `Glyph` entries when the glyph already exists from another layer. -**Binary variation metadata:** The TTF/OTF reader imports `fvar` axis definitions, hidden flags, and named instances into the Shift IR. Binary glyph geometry is still materialized only at the default variation location; recovering editable `gvar` sources is separate work. +**Binary variation metadata:** The TTF/OTF reader imports `fvar` axis definitions, hidden flags, and named instances into the Shift IR. The bounded path enumerates every `maxp` glyph ID, groups all `cmap` values by glyph, deterministically derives contour/point identities from emitted positions, and preserves TrueType quadratics instead of expanding them to cubic control pairs. Binary glyph geometry is still materialized only at the default variation location; recovering editable `gvar` sources is separate work. **Glyphs-format specifics:** `GlyphsReader` also extracts axes, sources, and per-master locations -- data that UFO does not natively represent. Kerning group membership is derived from per-glyph `right_kern`/`left_kern` fields and normalized to `public.kern1.*`/`public.kern2.*` conventions. diff --git a/crates/shift-backends/src/binary/mod.rs b/crates/shift-backends/src/binary/mod.rs index bcb8530c..b45fd929 100644 --- a/crates/shift-backends/src/binary/mod.rs +++ b/crates/shift-backends/src/binary/mod.rs @@ -2,6 +2,7 @@ mod reader; use crate::errors::{FormatBackendError, FormatBackendResult}; use crate::font_loader::FontAdaptor; +use crate::import::GlyphStream; use shift_font::Font; pub struct BytesFontAdaptor; @@ -14,4 +15,9 @@ impl FontAdaptor for BytesFontAdaptor { fn write_font(&self, _font: &Font, _path: &str) -> FormatBackendResult<()> { Err(FormatBackendError::WriteUnsupported) } + + fn stream(&self, path: &str) -> FormatBackendResult)>> { + let (header, stream) = reader::stream_font_file(path)?; + Ok(Some((header, Box::new(stream)))) + } } diff --git a/crates/shift-backends/src/binary/reader.rs b/crates/shift-backends/src/binary/reader.rs index 2dfa9e2b..51e28947 100644 --- a/crates/shift-backends/src/binary/reader.rs +++ b/crates/shift-backends/src/binary/reader.rs @@ -1,11 +1,18 @@ -use crate::errors::{FormatBackendError, FormatBackendResult}; +use std::sync::Arc; + +use crate::{ + errors::{FormatBackendError, FormatBackendResult}, + import::{collect_streamed_font, GlyphDirectoryEntry, GlyphStream, ImportBatchLimit}, +}; +use rayon::prelude::*; use shift_font::{ - Axis, Contour, Font, Glyph, GlyphLayer, LayerId, Location, MetricKind, NamedInstance, PointType, + Axis, Contour, ContourId, Font, Glyph, GlyphId as ShiftGlyphId, GlyphLayer, GlyphName, LayerId, + Location, MetricKind, NamedInstance, PointId, PointType, SourceId, }; use skrifa::{ - outline::{DrawSettings, OutlinePen}, + outline::{DrawSettings, OutlineGlyphCollection, OutlinePen}, prelude::{LocationRef, Size}, - raw::{ReadError, TableProvider}, + raw::{tables::hmtx::Hmtx, types::GlyphId, ReadError, TableProvider}, string::StringId, FontRef, MetadataProvider, }; @@ -13,19 +20,39 @@ use skrifa::{ use crate::metrics::set_metric_position; pub fn read_font_file(path: &str) -> FormatBackendResult { - let bytes = std::fs::read(path) - .map_err(|e| FormatBackendError::Binary(format!("failed to read '{path}': {e}")))?; - let font = FontRef::new(&bytes) - .map_err(|e| FormatBackendError::Binary(format!("failed to parse '{path}': {e}")))?; - font_from_skrifa(&font) + let (header, mut stream) = stream_font_file(path)?; + collect_streamed_font(header, &mut stream) } -#[derive(Default)] struct ShiftPen { + glyph_id: u32, + next_contour_index: usize, + next_point_index: usize, contours: Vec, } impl ShiftPen { + fn new(glyph_id: GlyphId) -> Self { + Self { + glyph_id: glyph_id.to_u32(), + next_contour_index: 0, + next_point_index: 0, + contours: Vec::new(), + } + } + + fn next_contour_id(&mut self) -> ContourId { + let index = self.next_contour_index; + self.next_contour_index += 1; + ContourId::from_raw(format!("b{:x}_{index:x}", self.glyph_id)) + } + + fn next_point_id(&mut self) -> PointId { + let index = self.next_point_index; + self.next_point_index += 1; + PointId::from_raw(format!("b{:x}_{index:x}", self.glyph_id)) + } + /// Returns the contour currently being built. /// Panics if called before `move_to` — skrifa guarantees `move_to` is called first. fn current_contour(&mut self) -> &mut Contour { @@ -41,44 +68,67 @@ impl ShiftPen { impl OutlinePen for ShiftPen { fn move_to(&mut self, x: f32, y: f32) { - self.contours.push(Contour::new()); - self.current_contour() - .add_point(x as f64, y as f64, PointType::OnCurve, false); + let contour_id = self.next_contour_id(); + self.contours.push(Contour::with_id(contour_id)); + let point_id = self.next_point_id(); + self.current_contour().add_point_with_id( + point_id, + x as f64, + y as f64, + PointType::OnCurve, + false, + ); } fn line_to(&mut self, x: f32, y: f32) { - self.current_contour() - .add_point(x as f64, y as f64, PointType::OnCurve, false); + let point_id = self.next_point_id(); + self.current_contour().add_point_with_id( + point_id, + x as f64, + y as f64, + PointType::OnCurve, + false, + ); } - /// Binary imports produce cubic-only IR: the wire and renderer layers do - /// not support quadratic segments, so every TrueType quadratic is lifted - /// to its exact cubic equivalent (mathematically lossless — every - /// quadratic is a cubic with c1 = q0 + 2/3(q1 - q0), c2 = q2 + 2/3(q1 - q2)). + /// Preserves the source quadratic as one control and one qcurve endpoint. + /// The canonical layer codec retains this distinction; the wire projects + /// the endpoint as on-curve, which still lets clients infer one-control + /// quadratic segments without cubic expansion. fn quad_to(&mut self, cx0: f32, cy0: f32, x: f32, y: f32) { + let control_id = self.next_point_id(); + let endpoint_id = self.next_point_id(); let contour = self.current_contour(); - let q0 = contour - .last_point() - .expect("quad_to requires a current point"); - let (q0x, q0y) = (q0.x(), q0.y()); - let (q1x, q1y) = (cx0 as f64, cy0 as f64); - let (q2x, q2y) = (x as f64, y as f64); - let c1x = q0x + 2.0 / 3.0 * (q1x - q0x); - let c1y = q0y + 2.0 / 3.0 * (q1y - q0y); - let c2x = q2x + 2.0 / 3.0 * (q1x - q2x); - let c2y = q2y + 2.0 / 3.0 * (q1y - q2y); - contour.add_point(c1x, c1y, PointType::OffCurve, false); - contour.add_point(c2x, c2y, PointType::OffCurve, false); - contour.add_point(q2x, q2y, PointType::OnCurve, false); + contour.add_point_with_id( + control_id, + cx0 as f64, + cy0 as f64, + PointType::OffCurve, + false, + ); + contour.add_point_with_id(endpoint_id, x as f64, y as f64, PointType::QCurve, false); } fn curve_to(&mut self, cx0: f32, cy0: f32, cx1: f32, cy1: f32, x: f32, y: f32) { - self.current_contour() - .add_point(cx0 as f64, cy0 as f64, PointType::OffCurve, false); - self.current_contour() - .add_point(cx1 as f64, cy1 as f64, PointType::OffCurve, false); - self.current_contour() - .add_point(x as f64, y as f64, PointType::OnCurve, false); + let control_0_id = self.next_point_id(); + let control_1_id = self.next_point_id(); + let endpoint_id = self.next_point_id(); + let contour = self.current_contour(); + contour.add_point_with_id( + control_0_id, + cx0 as f64, + cy0 as f64, + PointType::OffCurve, + false, + ); + contour.add_point_with_id( + control_1_id, + cx1 as f64, + cy1 as f64, + PointType::OffCurve, + false, + ); + contour.add_point_with_id(endpoint_id, x as f64, y as f64, PointType::OnCurve, false); } /// Closes the current contour. skrifa ends contours whose final segment @@ -91,7 +141,11 @@ impl OutlinePen for ShiftPen { let first = contour.first_point().expect("contour is non-empty"); let last = contour.last_point().expect("contour is non-empty"); if last.is_on_curve() && last.x() == first.x() && last.y() == first.y() { + let closes_quadratic = last.point_type() == PointType::QCurve; contour.points_mut().pop(); + if closes_quadratic { + contour.points_mut()[0].set_point_type(PointType::QCurve); + } } } contour.close(); @@ -148,13 +202,7 @@ fn detect_smooth_points(contours: &mut [Contour]) { } } -fn font_from_skrifa(font: &FontRef<'_>) -> FormatBackendResult { - let outlines = font.outline_glyphs(); - let char_map = font.charmap(); - let hmtx = font - .hmtx() - .map_err(|e| FormatBackendError::Binary(format!("failed to read hmtx table: {e}")))?; - +fn font_header_from_skrifa(font: &FontRef<'_>) -> FormatBackendResult { let metrics = font.metrics(Size::unscaled(), LocationRef::default()); let mut ir_font = Font::new(); let default_source_id = ir_font @@ -164,7 +212,7 @@ fn font_from_skrifa(font: &FontRef<'_>) -> FormatBackendResult { ir_font.metrics_mut().units_per_em = metrics.units_per_em as f64; let metric_definitions = ir_font.metric_definitions().to_vec(); let default_source = ir_font - .source_mut(default_source_id.clone()) + .source_mut(default_source_id) .expect("new font should contain its default source"); set_metric_position( &metric_definitions, @@ -199,47 +247,157 @@ fn font_from_skrifa(font: &FontRef<'_>) -> FormatBackendResult { } load_variation_metadata(font, &mut ir_font)?; + Ok(ir_font) +} - for (unicode, glyph_id) in char_map.mappings() { - let outline = outlines.get(glyph_id).ok_or_else(|| { - FormatBackendError::Binary(format!( - "missing outline for glyph {glyph_id} (U+{unicode:04X})" - )) +#[derive(Clone)] +struct BinaryGlyphRecord { + raw_id: GlyphId, + shift_id: ShiftGlyphId, + name: GlyphName, + unicodes: Vec, +} + +pub(crate) struct BinaryGlyphStream { + bytes: Arc<[u8]>, + source_id: SourceId, + glyphs: Vec, + next_glyph: usize, +} + +pub(crate) fn stream_font_file(path: &str) -> FormatBackendResult<(Font, BinaryGlyphStream)> { + let bytes: Arc<[u8]> = std::fs::read(path) + .map_err(|error| FormatBackendError::Binary(format!("failed to read '{path}': {error}")))? + .into(); + let font = FontRef::new(bytes.as_ref()).map_err(|error| { + FormatBackendError::Binary(format!("failed to parse '{path}': {error}")) + })?; + font.hmtx().map_err(|error| { + FormatBackendError::Binary(format!("failed to read hmtx table: {error}")) + })?; + let header = font_header_from_skrifa(&font)?; + let source_id = header.default_source_id().ok_or_else(|| { + FormatBackendError::Binary("binary font header is missing its default source".into()) + })?; + let glyph_count = font + .maxp() + .map_err(|error| FormatBackendError::Binary(format!("failed to read maxp table: {error}")))? + .num_glyphs() as usize; + let mut unicodes = vec![Vec::new(); glyph_count]; + for (unicode, glyph_id) in font.charmap().mappings() { + if let Some(values) = unicodes.get_mut(glyph_id.to_u32() as usize) { + values.push(unicode); + } + } + let names = font.glyph_names(); + let glyphs = unicodes + .into_iter() + .enumerate() + .map(|(index, unicodes)| { + let raw_id = GlyphId::new(index as u32); + let name = names + .get(raw_id) + .map(|name| name.to_string()) + .unwrap_or_else(|| format!("gid{index}")); + BinaryGlyphRecord { + raw_id, + // Compiled fonts carry stable glyph order but no Shift IDs, so + // deterministic GID-derived identities make re-import repeatable. + shift_id: ShiftGlyphId::from_raw(format!("binary{index}")), + name: GlyphName::from(name), + unicodes, + } + }) + .collect(); + + Ok(( + header, + BinaryGlyphStream { + bytes, + source_id, + glyphs, + next_glyph: 0, + }, + )) +} + +impl GlyphStream for BinaryGlyphStream { + fn directory(&self) -> Vec { + self.glyphs + .iter() + .map(|glyph| GlyphDirectoryEntry { + glyph_id: glyph.shift_id.clone(), + name: glyph.name.clone(), + }) + .collect() + } + + fn glyph_count(&self) -> usize { + self.glyphs.len() + } + + fn next_batch(&mut self, limit: ImportBatchLimit) -> FormatBackendResult> { + if self.next_glyph == self.glyphs.len() { + return Ok(Vec::new()); + } + + let batch_glyphs = limit.max_glyphs().min(limit.max_layers()); + let end = self + .next_glyph + .saturating_add(batch_glyphs) + .min(self.glyphs.len()); + let font = FontRef::new(self.bytes.as_ref()).map_err(|error| { + FormatBackendError::Binary(format!("failed to reopen resident font: {error}")) })?; - let settings = DrawSettings::unhinted(Size::unscaled(), LocationRef::default()); - let mut pen = ShiftPen::default(); - outline.draw(settings, &mut pen).map_err(|e| { - FormatBackendError::Binary(format!( - "failed to draw outline for glyph {glyph_id} (U+{unicode:04X}): {e}" - )) + let hmtx = font.hmtx().map_err(|error| { + FormatBackendError::Binary(format!("failed to read hmtx table: {error}")) })?; + let outlines = font.outline_glyphs(); + let source_id = &self.source_id; + let glyphs = self.glyphs[self.next_glyph..end] + .par_iter() + .map(|record| glyph_from_skrifa(&hmtx, &outlines, source_id, record)) + .collect::>>()?; + self.next_glyph = end; + Ok(glyphs) + } +} - let advance_width = hmtx.advance(glyph_id).ok_or_else(|| { +fn glyph_from_skrifa( + hmtx: &Hmtx<'_>, + outlines: &OutlineGlyphCollection<'_>, + source_id: &SourceId, + record: &BinaryGlyphRecord, +) -> FormatBackendResult { + let advance_width = hmtx.advance(record.raw_id).ok_or_else(|| { + FormatBackendError::Binary(format!("missing advance width for glyph {}", record.raw_id)) + })?; + let mut layer = GlyphLayer::with_width( + LayerId::from_raw(format!("binary{}", record.raw_id.to_u32())), + source_id.clone(), + advance_width as f64, + ); + + if let Some(outline) = outlines.get(record.raw_id) { + let settings = DrawSettings::unhinted(Size::unscaled(), LocationRef::default()); + let mut pen = ShiftPen::new(record.raw_id); + outline.draw(settings, &mut pen).map_err(|error| { FormatBackendError::Binary(format!( - "missing advance width for glyph {glyph_id} (U+{unicode:04X})" + "failed to draw outline for glyph {}: {error}", + record.raw_id )) })?; - - let glyph_name = char::from_u32(unicode) - .map(|c| c.to_string()) - .unwrap_or_else(|| format!("uni{unicode:04X}")); - - let mut glyph = Glyph::with_unicode(glyph_name, unicode); - let mut layer = GlyphLayer::with_width( - LayerId::new(), - default_source_id.clone(), - advance_width as f64, - ); let mut contours = pen.contours(); detect_smooth_points(&mut contours); for contour in contours { layer.add_contour(contour); } - glyph.set_layer(layer); - ir_font.insert_glyph(glyph)?; } - Ok(ir_font) + let mut glyph = Glyph::with_id(record.shift_id.clone(), record.name.clone()); + glyph.set_unicodes(record.unicodes.clone()); + glyph.set_layer(layer); + Ok(glyph) } fn load_variation_metadata(font: &FontRef<'_>, ir_font: &mut Font) -> FormatBackendResult<()> { @@ -324,7 +482,7 @@ fn localized_string(font: &FontRef<'_>, id: StringId) -> Option { #[cfg(test)] mod tests { use super::*; - use shift_font::Point; + use shift_font::CurveSegment; use skrifa::outline::pen::PathElement; use std::path::PathBuf; @@ -337,35 +495,39 @@ mod tests { .join("fixtures/fonts/mutatorsans/MutatorSans.ttf") } - fn quad_at(q0: (f64, f64), q1: (f64, f64), q2: (f64, f64), t: f64) -> (f64, f64) { - let u = 1.0 - t; - ( - u * u * q0.0 + 2.0 * u * t * q1.0 + t * t * q2.0, - u * u * q0.1 + 2.0 * u * t * q1.1 + t * t * q2.1, - ) + fn point_types(contour: &Contour) -> Vec { + contour.points().iter().map(|p| p.point_type()).collect() } - fn cubic_at( - p0: (f64, f64), - c1: (f64, f64), - c2: (f64, f64), - p3: (f64, f64), - t: f64, - ) -> (f64, f64) { - let u = 1.0 - t; - ( - u * u * u * p0.0 + 3.0 * u * u * t * c1.0 + 3.0 * u * t * t * c2.0 + t * t * t * p3.0, - u * u * u * p0.1 + 3.0 * u * u * t * c1.1 + 3.0 * u * t * t * c2.1 + t * t * t * p3.1, - ) + fn deterministic_pen_contours() -> Vec { + let mut pen = ShiftPen::new(GlyphId::new(0x2a)); + pen.move_to(0.0, 0.0); + pen.line_to(100.0, 0.0); + pen.quad_to(100.0, 100.0, 0.0, 0.0); + pen.close(); + pen.move_to(10.0, 20.0); + pen.contours() } - fn point_types(contour: &Contour) -> Vec { - contour.points().iter().map(|p| p.point_type()).collect() + #[test] + fn binary_pen_mints_deterministic_positional_ids() { + let contours = deterministic_pen_contours(); + assert_eq!(contours, deterministic_pen_contours()); + assert_eq!(contours[0].id().as_str(), "contour_b2a_0"); + assert_eq!(contours[1].id().as_str(), "contour_b2a_1"); + assert_eq!(contours[0].points()[0].id().as_str(), "point_b2a_0"); + assert_eq!(contours[0].points()[1].id().as_str(), "point_b2a_1"); + assert_eq!(contours[0].points()[2].id().as_str(), "point_b2a_2"); + assert_eq!( + contours[1].points()[0].id().as_str(), + "point_b2a_4", + "the removed closing endpoint keeps its consumed positional index" + ); } #[test] - fn quad_lifted_to_geometrically_identical_cubic() { - let mut pen = ShiftPen::default(); + fn quad_preserves_control_and_endpoint() { + let mut pen = ShiftPen::new(GlyphId::new(0x2a)); pen.move_to(10.0, 20.0); pen.quad_to(50.0, 90.0, 100.0, 20.0); @@ -373,33 +535,15 @@ mod tests { let points = contours[0].points(); assert_eq!( point_types(&contours[0]), - vec![ - PointType::OnCurve, - PointType::OffCurve, - PointType::OffCurve, - PointType::OnCurve - ] + vec![PointType::OnCurve, PointType::OffCurve, PointType::QCurve] ); - - let q0 = (10.0, 20.0); - let q1 = (50.0, 90.0); - let q2 = (100.0, 20.0); - let c1 = (points[1].x(), points[1].y()); - let c2 = (points[2].x(), points[2].y()); - for step in 0..=20 { - let t = step as f64 / 20.0; - let expected = quad_at(q0, q1, q2, t); - let actual = cubic_at(q0, c1, c2, q2, t); - assert!( - (expected.0 - actual.0).abs() < 1e-9 && (expected.1 - actual.1).abs() < 1e-9, - "lifted cubic diverges from quad at t={t}: {expected:?} vs {actual:?}" - ); - } + assert_eq!((points[1].x(), points[1].y()), (50.0, 90.0)); + assert_eq!((points[2].x(), points[2].y()), (100.0, 20.0)); } #[test] fn close_drops_duplicate_start_point_from_closing_curve() { - let mut pen = ShiftPen::default(); + let mut pen = ShiftPen::new(GlyphId::new(0x2a)); pen.move_to(0.0, 0.0); pen.line_to(100.0, 0.0); pen.quad_to(100.0, 100.0, 0.0, 0.0); @@ -410,19 +554,14 @@ mod tests { assert!(contour.is_closed()); assert_eq!( point_types(contour), - vec![ - PointType::OnCurve, - PointType::OnCurve, - PointType::OffCurve, - PointType::OffCurve - ], - "closing curve's explicit end point should be dropped; the closed contour wraps to the start" + vec![PointType::QCurve, PointType::OnCurve, PointType::OffCurve], + "closing qcurve endpoint should move to the wrapped start point" ); } #[test] fn close_keeps_distinct_last_point() { - let mut pen = ShiftPen::default(); + let mut pen = ShiftPen::new(GlyphId::new(0x2a)); pen.move_to(0.0, 0.0); pen.line_to(100.0, 0.0); pen.line_to(100.0, 100.0); @@ -435,7 +574,7 @@ mod tests { #[test] fn line_after_quad_composes() { - let mut pen = ShiftPen::default(); + let mut pen = ShiftPen::new(GlyphId::new(0x2a)); pen.move_to(0.0, 0.0); pen.quad_to(50.0, 100.0, 100.0, 0.0); pen.line_to(200.0, 0.0); @@ -447,15 +586,14 @@ mod tests { vec![ PointType::OnCurve, PointType::OffCurve, - PointType::OffCurve, - PointType::OnCurve, + PointType::QCurve, PointType::OnCurve ] ); } #[test] - fn imported_cubics_match_source_quadratics() { + fn imported_quadratics_match_source_segments() { let path = mutatorsans_ttf_path(); let bytes = std::fs::read(&path).expect("MutatorSans.ttf fixture should exist"); let font_ref = FontRef::new(&bytes).unwrap(); @@ -475,7 +613,7 @@ mod tests { ) .unwrap(); - let font = font_from_skrifa(&font_ref).unwrap(); + let font = read_font_file(path.to_str().unwrap()).unwrap(); let glyph = font .glyphs_by_unicode('O' as u32) .next() @@ -486,21 +624,18 @@ mod tests { .next() .expect("glyph should have a layer"); - let mut cubic_segments: Vec<[(f64, f64); 4]> = Vec::new(); - for contour in layer.contours_iter() { - let points: Vec<&Point> = contour.points().iter().collect(); - let len = points.len(); - for i in 0..len { - let window: Vec<&Point> = (0..4).map(|offset| points[(i + offset) % len]).collect(); - if window[0].is_on_curve() - && window[1].point_type() == PointType::OffCurve - && window[2].point_type() == PointType::OffCurve - && window[3].is_on_curve() - { - cubic_segments.push([0, 1, 2, 3].map(|p| (window[p].x(), window[p].y()))); - } - } - } + let quadratics = layer + .contours_iter() + .flat_map(|contour| contour.segments()) + .filter_map(|segment| match segment { + CurveSegment::Quad(start, control, end) => Some([ + (start.x(), start.y()), + (control.x(), control.y()), + (end.x(), end.y()), + ]), + CurveSegment::Line(..) | CurveSegment::Cubic(..) => None, + }) + .collect::>(); let mut current = (0.0, 0.0); let mut quads_checked = 0; @@ -510,27 +645,13 @@ mod tests { current = (x as f64, y as f64); } PathElement::QuadTo { cx0, cy0, x, y } => { - let q0 = current; - let q1 = (cx0 as f64, cy0 as f64); - let q2 = (x as f64, y as f64); - let segment = cubic_segments - .iter() - .find(|[p0, _, _, p3]| *p0 == q0 && *p3 == q2) - .unwrap_or_else(|| { - panic!("no imported cubic segment from {q0:?} to {q2:?}") - }); - for step in 1..8 { - let t = step as f64 / 8.0; - let expected = quad_at(q0, q1, q2, t); - let actual = cubic_at(segment[0], segment[1], segment[2], segment[3], t); - assert!( - (expected.0 - actual.0).abs() < 1e-9 - && (expected.1 - actual.1).abs() < 1e-9, - "imported cubic diverges from source quad at t={t}" - ); - } + let expected = [current, (cx0 as f64, cy0 as f64), (x as f64, y as f64)]; + assert!( + quadratics.contains(&expected), + "missing imported quadratic {expected:?}" + ); quads_checked += 1; - current = q2; + current = expected[2]; } PathElement::CurveTo { x, y, .. } => { current = (x as f64, y as f64); diff --git a/crates/shift-backends/src/designspace/error.rs b/crates/shift-backends/src/designspace/error.rs index 6501ac8f..c69a85a9 100644 --- a/crates/shift-backends/src/designspace/error.rs +++ b/crates/shift-backends/src/designspace/error.rs @@ -1,5 +1,7 @@ use std::path::PathBuf; +use crate::errors::FormatBackendError; + pub type DesignspaceResult = Result; #[derive(Debug, thiserror::Error)] @@ -19,9 +21,6 @@ pub enum DesignspaceError { #[error("designspace has no source at the mapped default location")] MissingDefaultSource, - #[error("layer '{layer}' not found in '{filename}'")] - MissingLayer { layer: String, filename: String }, - #[error("failed to read '{path}': {source}")] ReadFile { path: PathBuf, @@ -49,8 +48,12 @@ pub enum DesignspaceError { #[error("failed to save designspace '{path}': {details}")] SaveDesignspace { path: PathBuf, details: String }, - #[error("failed to load UFO '{path}': {details}")] - LoadUfo { path: PathBuf, details: String }, + #[error("failed to load UFO '{path}': {source}")] + LoadUfo { + path: PathBuf, + #[source] + source: Box, + }, #[error("failed to save UFO '{path}': {details}")] SaveUfo { path: PathBuf, details: String }, diff --git a/crates/shift-backends/src/designspace/import.rs b/crates/shift-backends/src/designspace/import.rs new file mode 100644 index 00000000..d3a2bfb0 --- /dev/null +++ b/crates/shift-backends/src/designspace/import.rs @@ -0,0 +1,367 @@ +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + fs, + path::{Path, PathBuf}, +}; + +use norad::designspace::DesignSpaceDocument; +use rayon::prelude::*; +use shift_font::{ + Axis, AxisId, Font, GlyphId, LayerId, Location, MetricDefinition, Source, SourceId, +}; + +use crate::{ + errors::{FormatBackendError, FormatBackendResult}, + import::{GlifGlyph, GlifGlyphStream, GlifLayer}, + metrics::copy_source_metrics, + ufo::{load_header, read_glyph_paths}, +}; + +use super::{ + axis_labels, + error::{DesignspaceError, DesignspaceResult}, + reader::{ + axis_mappings_from_designspace, axisless_source_name, derive_axis_range, + find_default_source_index, location_from_dimensions, named_instances_from_designspace, + parse_axisless_sources, source_name, + }, +}; + +struct SourceDirectory { + source_id: SourceId, + glyphs: BTreeMap, +} + +struct DesignspaceSourceMaterial { + index: usize, + style_name: Option, + metric_definitions: Vec, + metrics: Source, + glyphs: BTreeMap, +} + +pub(crate) fn stream_font(path: &str) -> FormatBackendResult<(Font, GlifGlyphStream)> { + stream_designspace(Path::new(path)).map_err(FormatBackendError::from) +} + +fn stream_designspace(designspace_path: &Path) -> DesignspaceResult<(Font, GlifGlyphStream)> { + let designspace_dir = + designspace_path + .parent() + .ok_or_else(|| DesignspaceError::MissingParent { + path: designspace_path.to_path_buf(), + })?; + let xml = + fs::read_to_string(designspace_path).map_err(|source| DesignspaceError::ReadFile { + path: designspace_path.to_path_buf(), + source, + })?; + let mut labels = axis_labels::parse(&xml)?; + let document = match DesignSpaceDocument::load(designspace_path) { + Ok(document) => document, + Err(error) => { + return stream_axisless_designspace( + designspace_path, + designspace_dir, + &xml, + &error.to_string(), + ); + } + }; + if document.sources.is_empty() { + return Err(DesignspaceError::NoSources); + } + let default_index = + find_default_source_index(&document).ok_or(DesignspaceError::MissingDefaultSource)?; + let default_descriptor = &document.sources[default_index]; + let default_path = designspace_dir.join(&default_descriptor.filename); + let mut header = load_header(&default_path).map_err(|error| DesignspaceError::LoadUfo { + path: default_path.clone(), + source: Box::new(error), + })?; + let default_metrics = header + .default_source() + .cloned() + .ok_or(DesignspaceError::NoSources)?; + let default_metric_definitions = header.metric_definitions().to_vec(); + header.clear_sources(); + + if let Some(family_name) = &default_descriptor.familyname { + header.metadata_mut().family_name = Some(family_name.clone()); + } + add_axes(&mut header, &document, &mut labels)?; + header.set_axis_mappings(axis_mappings_from_designspace(&document, header.axes())?)?; + header.set_named_instances(named_instances_from_designspace(&document, header.axes())?)?; + + let source_order = std::iter::once(default_index) + .chain((0..document.sources.len()).filter(|index| *index != default_index)) + .collect::>(); + let default_style_name = header.metadata().style_name.clone(); + let source_materials = source_order + .into_par_iter() + .map(|index| { + let descriptor = &document.sources[index]; + let ufo_path = designspace_dir.join(&descriptor.filename); + let (style_name, metric_definitions, metrics) = if index == default_index { + ( + default_style_name.clone(), + default_metric_definitions.clone(), + default_metrics.clone(), + ) + } else { + let ufo_header = + load_header(&ufo_path).map_err(|error| DesignspaceError::LoadUfo { + path: ufo_path.clone(), + source: Box::new(error), + })?; + let metrics = ufo_header + .default_source() + .cloned() + .ok_or(DesignspaceError::NoSources)?; + ( + ufo_header.metadata().style_name.clone(), + ufo_header.metric_definitions().to_vec(), + metrics, + ) + }; + let glyphs = + read_glyph_paths(&ufo_path, descriptor.layer.as_deref()).map_err(|error| { + DesignspaceError::LoadUfo { + path: ufo_path, + source: Box::new(error), + } + })?; + Ok(DesignspaceSourceMaterial { + index, + style_name, + metric_definitions, + metrics, + glyphs, + }) + }) + .collect::>>()?; + + let axes = header.axes().to_vec(); + let directories = register_designspace_sources( + &mut header, + source_materials, + default_index, + |index, style_name, source_names| { + let descriptor = &document.sources[index]; + let name = source_name(descriptor, style_name, source_names, index); + let location = location_from_dimensions(&descriptor.location, &document, &axes); + let mut source = Source::with_filename(name, location, descriptor.filename.clone()); + source.set_layer_name(descriptor.layer.clone()); + source + }, + ); + + let (glyph_ids, glyphs) = build_glyph_directory(&directories); + Ok((header, GlifGlyphStream::new(glyph_ids, glyphs))) +} + +fn stream_axisless_designspace( + designspace_path: &Path, + designspace_dir: &Path, + xml: &str, + original_error: &str, +) -> DesignspaceResult<(Font, GlifGlyphStream)> { + let sources = parse_axisless_sources(xml).map_err(|fallback_error| { + DesignspaceError::LoadDesignspace { + path: designspace_path.to_path_buf(), + details: format!( + "{fallback_error}; axisless fallback was used after parser error: {original_error}" + ), + } + })?; + if sources.is_empty() { + return Err(DesignspaceError::NoSources); + } + + let default_path = designspace_dir.join(&sources[0].filename); + let mut header = load_header(&default_path).map_err(|error| DesignspaceError::LoadUfo { + path: default_path, + source: Box::new(error), + })?; + if let Some(family_name) = &sources[0].familyname { + header.metadata_mut().family_name = Some(family_name.clone()); + } + let default_style_name = header.metadata().style_name.clone(); + let default_metrics = header + .default_source() + .cloned() + .ok_or(DesignspaceError::NoSources)?; + let default_metric_definitions = header.metric_definitions().to_vec(); + header.clear_sources(); + + let source_materials = sources + .par_iter() + .enumerate() + .map(|(index, descriptor)| { + let ufo_path = designspace_dir.join(&descriptor.filename); + let (style_name, metric_definitions, imported_metrics) = if index == 0 { + ( + default_style_name.clone(), + default_metric_definitions.clone(), + default_metrics.clone(), + ) + } else { + let ufo_header = + load_header(&ufo_path).map_err(|error| DesignspaceError::LoadUfo { + path: ufo_path.clone(), + source: Box::new(error), + })?; + let metrics = ufo_header + .default_source() + .cloned() + .ok_or(DesignspaceError::NoSources)?; + ( + ufo_header.metadata().style_name.clone(), + ufo_header.metric_definitions().to_vec(), + metrics, + ) + }; + let glyphs = + read_glyph_paths(&ufo_path, descriptor.layer.as_deref()).map_err(|error| { + DesignspaceError::LoadUfo { + path: ufo_path, + source: Box::new(error), + } + })?; + Ok(DesignspaceSourceMaterial { + index, + style_name, + metric_definitions, + metrics: imported_metrics, + glyphs, + }) + }) + .collect::>>()?; + + let directories = register_designspace_sources( + &mut header, + source_materials, + 0, + |index, style_name, source_names| { + let descriptor = &sources[index]; + let name = axisless_source_name(descriptor, style_name, source_names, index); + let mut source = + Source::with_filename(name, Location::new(), descriptor.filename.clone()); + source.set_layer_name(descriptor.layer.clone()); + source + }, + ); + + let (glyph_ids, glyphs) = build_glyph_directory(&directories); + Ok((header, GlifGlyphStream::new(glyph_ids, glyphs))) +} + +fn register_designspace_sources( + header: &mut Font, + source_materials: Vec, + default_index: usize, + mut create_source: impl FnMut(usize, Option<&str>, &HashSet) -> Source, +) -> Vec { + let metric_definitions = header.metric_definitions().to_vec(); + let mut source_names = HashSet::new(); + let mut directories = Vec::with_capacity(source_materials.len()); + for material in source_materials { + let mut source = create_source( + material.index, + material.style_name.as_deref(), + &source_names, + ); + source_names.insert(source.name().to_string()); + copy_source_metrics( + &material.metric_definitions, + &material.metrics, + &metric_definitions, + &mut source, + ); + let source_id = header.add_source(source); + if material.index == default_index { + header.set_default_source_id(source_id.clone()); + } + directories.push(SourceDirectory { + source_id, + glyphs: material.glyphs, + }); + } + directories +} + +fn add_axes( + header: &mut Font, + document: &DesignSpaceDocument, + labels: &mut HashMap>, +) -> DesignspaceResult<()> { + for axis in &document.axes { + let mut imported = if let Some(values) = &axis.values { + let mut values = values + .iter() + .map(|value| f64::from(*value)) + .collect::>(); + values.sort_by(f64::total_cmp); + values.dedup(); + Axis::discrete_with_id( + AxisId::new(), + axis.tag.clone(), + axis.name.clone(), + values, + f64::from(axis.default), + ) + } else { + let (minimum, maximum) = derive_axis_range(axis); + Axis::new( + axis.tag.clone(), + axis.name.clone(), + minimum, + f64::from(axis.default), + maximum, + ) + }; + imported.set_hidden(axis.hidden); + imported.set_labels(labels.remove(axis.name.as_str()).unwrap_or_default()); + imported.validate()?; + header.add_axis(imported)?; + } + Ok(()) +} + +fn build_glyph_directory( + directories: &[SourceDirectory], +) -> (HashMap, Vec) { + let names = directories[0].glyphs.keys().cloned().collect::>(); + let glyph_ids = names + .iter() + .map(|name| (name.clone(), GlyphId::new())) + .collect::>(); + let glyphs = names + .into_iter() + .map(|name| { + let mut layers = Vec::new(); + if let Some(path) = directories[0].glyphs.get(&name) { + layers.push(GlifLayer { + source_id: directories[0].source_id.clone(), + layer_id: LayerId::new(), + path: path.clone(), + }); + } + for directory in directories.iter().skip(1) { + if let Some(path) = directory.glyphs.get(&name) { + layers.push(GlifLayer { + source_id: directory.source_id.clone(), + layer_id: LayerId::new(), + path: path.clone(), + }); + } + } + GlifGlyph { + id: glyph_ids[&name].clone(), + name, + layers, + } + }) + .collect(); + (glyph_ids, glyphs) +} diff --git a/crates/shift-backends/src/designspace/mod.rs b/crates/shift-backends/src/designspace/mod.rs index 11359580..f36849d0 100644 --- a/crates/shift-backends/src/designspace/mod.rs +++ b/crates/shift-backends/src/designspace/mod.rs @@ -1,9 +1,11 @@ mod axis_labels; mod error; +mod import; mod reader; mod writer; pub use error::DesignspaceError; +pub(crate) use import::stream_font; pub use reader::DesignspaceReader; pub use writer::DesignspaceWriter; diff --git a/crates/shift-backends/src/designspace/reader.rs b/crates/shift-backends/src/designspace/reader.rs index 1e230299..81e33109 100644 --- a/crates/shift-backends/src/designspace/reader.rs +++ b/crates/shift-backends/src/designspace/reader.rs @@ -1,18 +1,14 @@ -use super::axis_labels; use super::error::{DesignspaceError, DesignspaceResult}; -use crate::errors::{FormatBackendError, FormatBackendResult}; -use crate::metrics::copy_source_metrics; +use crate::errors::FormatBackendResult; +use crate::import::collect_streamed_font; use crate::traits::FontReader; -use crate::ufo::UfoReader; use norad::designspace::DesignSpaceDocument; use quick_xml::events::{BytesStart, Event}; use quick_xml::Reader; use shift_font::{ - Axis, AxisId, AxisMapping, AxisMappingId, AxisMappingPoint, Component, Font, GlyphLayer, - LayerId, Location, NamedInstance, Source, SourceId, + Axis, AxisId, AxisMapping, AxisMappingId, AxisMappingPoint, Font, Location, NamedInstance, }; use std::collections::{HashMap, HashSet}; -use std::fs; use std::path::Path; pub struct DesignspaceReader; @@ -32,217 +28,17 @@ impl Default for DesignspaceReader { impl FontReader for DesignspaceReader { fn load(&self, path: &str) -> FormatBackendResult { self.load_designspace(path) - .map_err(FormatBackendError::from) } } impl DesignspaceReader { - fn load_designspace(&self, path: &str) -> DesignspaceResult { - let ds_path = Path::new(path); - let ds_dir = ds_path - .parent() - .ok_or_else(|| DesignspaceError::MissingParent { - path: ds_path.to_path_buf(), - })?; - let xml = fs::read_to_string(ds_path).map_err(|source| DesignspaceError::ReadFile { - path: ds_path.to_path_buf(), - source, - })?; - let mut axis_labels = axis_labels::parse(&xml)?; - - let doc = match DesignSpaceDocument::load(ds_path) { - Ok(doc) => doc, - Err(error) => { - let original_error = error.to_string(); - return load_axisless_designspace(ds_path, ds_dir, &original_error).map_err( - |fallback_error| DesignspaceError::LoadDesignspace { - path: ds_path.to_path_buf(), - details: format!("{original_error}; {fallback_error}"), - }, - ); - } - }; - - if doc.sources.is_empty() { - return Err(DesignspaceError::NoSources); - } - - let default_idx = - find_default_source_index(&doc).ok_or(DesignspaceError::MissingDefaultSource)?; - - // Load the default source first to establish the base font. - let default_ds_source = &doc.sources[default_idx]; - let default_ufo_path = ds_dir.join(&default_ds_source.filename); - let default_ufo_str = - default_ufo_path - .to_str() - .ok_or_else(|| DesignspaceError::InvalidPathUtf8 { - path: default_ufo_path.clone(), - })?; - - let ufo_reader = UfoReader::new(); - let mut font = - ufo_reader - .load(default_ufo_str) - .map_err(|source| DesignspaceError::LoadUfo { - path: default_ufo_path.clone(), - details: source.to_string(), - })?; - let default_ufo_source_id = source_layer_id(&font, default_ds_source)?; - let default_ufo_metrics = font - .default_source() - .cloned() - .ok_or(DesignspaceError::NoSources)?; - font.clear_sources(); - - if let Some(ref family) = default_ds_source.familyname { - font.metadata_mut().family_name = Some(family.clone()); - } - - // Add axes. - for ds_axis in &doc.axes { - let mut axis = if let Some(values) = &ds_axis.values { - let mut values = values.iter().map(|value| *value as f64).collect::>(); - values.sort_by(f64::total_cmp); - values.dedup(); - Axis::discrete_with_id( - AxisId::new(), - ds_axis.tag.clone(), - ds_axis.name.clone(), - values, - ds_axis.default as f64, - ) - } else { - let (minimum, maximum) = derive_axis_range(ds_axis); - Axis::new( - ds_axis.tag.clone(), - ds_axis.name.clone(), - minimum, - ds_axis.default as f64, - maximum, - ) - }; - axis.set_hidden(ds_axis.hidden); - axis.set_labels( - axis_labels - .remove(ds_axis.name.as_str()) - .unwrap_or_default(), - ); - axis.validate()?; - font.add_axis(axis)?; - } - font.set_axis_mappings(axis_mappings_from_designspace(&doc, font.axes())?)?; - font.set_named_instances(named_instances_from_designspace(&doc, font.axes())?)?; - - // Register the default source. - let mut source_names = HashSet::new(); - let default_location = - location_from_dimensions(&default_ds_source.location, &doc, font.axes()); - let default_name = source_name( - default_ds_source, - font.metadata().style_name.as_deref(), - &source_names, - default_idx, - ); - source_names.insert(default_name.clone()); - let mut default_source = Source::with_filename( - default_name, - default_location, - default_ds_source.filename.clone(), - ); - default_source.set_layer_name(default_ds_source.layer.clone()); - let metric_definitions = font.metric_definitions().to_vec(); - copy_source_metrics( - &metric_definitions, - &default_ufo_metrics, - &metric_definitions, - &mut default_source, - ); - let default_source_id = font.add_source(default_source); - font.set_default_source_id(default_source_id.clone()); - move_glyph_layers_to_source(&mut font, default_ufo_source_id, default_source_id)?; - - // Cache loaded UFO fonts so we don't re-read the same file for support layers. - let mut ufo_cache: HashMap = HashMap::new(); - - // Load each non-default source. - for (idx, ds_source) in doc.sources.iter().enumerate() { - if idx == default_idx { - continue; - } - - let ufo_path = ds_dir.join(&ds_source.filename); - let ufo_str = ufo_path - .to_str() - .ok_or_else(|| DesignspaceError::InvalidPathUtf8 { - path: ufo_path.clone(), - })? - .to_string(); - - let source_font = match ufo_cache.get(&ufo_str) { - Some(f) => f, - None => { - let loaded = - ufo_reader - .load(&ufo_str) - .map_err(|source| DesignspaceError::LoadUfo { - path: ufo_path.clone(), - details: source.to_string(), - })?; - ufo_cache.insert(ufo_str.clone(), loaded); - ufo_cache.get(&ufo_str).unwrap() - } - }; - - // Determine which layer from the source UFO to read. - let source_source_id = source_layer_id(source_font, ds_source)?; - - let name = source_name( - ds_source, - source_font.metadata().style_name.as_deref(), - &source_names, - idx, - ); - source_names.insert(name.clone()); - let location = location_from_dimensions(&ds_source.location, &doc, font.axes()); - let mut source = Source::with_filename(name, location, ds_source.filename.clone()); - source.set_layer_name(ds_source.layer.clone()); - let imported_metrics = source_font - .default_source() - .ok_or(DesignspaceError::NoSources)?; - copy_source_metrics( - source_font.metric_definitions(), - imported_metrics, - font.metric_definitions(), - &mut source, - ); - let source_id = font.add_source(source); - - // Copy glyphs from the resolved layer into the new layer. - for source_glyph in source_font.glyphs() { - if let Some(source_layer) = source_glyph.layer_for_source(source_source_id.clone()) - { - if let Some(glyph_id) = font.glyph_id_by_name(source_glyph.name()) { - font.insert_glyph_layer( - glyph_id, - clone_layer_with_remapped_components( - source_layer, - &font, - LayerId::new(), - source_id.clone(), - )?, - )?; - } - } - } - } - - remove_glyph_layers_without_source(&mut font)?; - Ok(font) + fn load_designspace(&self, path: &str) -> FormatBackendResult { + let (header, mut stream) = super::stream_font(path)?; + collect_streamed_font(header, &mut stream) } } -fn named_instances_from_designspace( +pub(super) fn named_instances_from_designspace( doc: &DesignSpaceDocument, axes: &[Axis], ) -> DesignspaceResult> { @@ -361,7 +157,7 @@ fn unmap_axis_value(axis: &norad::designspace::Axis, value: f64) -> f64 { value + last_user - last_design } -fn axis_mappings_from_designspace( +pub(super) fn axis_mappings_from_designspace( doc: &DesignSpaceDocument, axes: &[Axis], ) -> DesignspaceResult> { @@ -463,169 +259,15 @@ fn extend_axis_ids(target: &mut Vec, location: &Location) { } #[derive(Clone, Debug)] -struct AxislessSource { - filename: String, - familyname: Option, - stylename: Option, - name: Option, - layer: Option, -} - -fn load_axisless_designspace( - ds_path: &Path, - ds_dir: &Path, - original_error: &str, -) -> DesignspaceResult { - let xml = fs::read_to_string(ds_path).map_err(|source| DesignspaceError::ReadFile { - path: ds_path.to_path_buf(), - source, - })?; - let sources = parse_axisless_sources(&xml).map_err(|fallback_error| { - DesignspaceError::LoadDesignspace { - path: ds_path.to_path_buf(), - details: format!( - "{fallback_error}; axisless fallback was used after parser error: {original_error}" - ), - } - })?; - if sources.is_empty() { - return Err(DesignspaceError::NoSources); - } - - let default_source = &sources[0]; - let default_ufo_path = ds_dir.join(&default_source.filename); - let default_ufo_str = - default_ufo_path - .to_str() - .ok_or_else(|| DesignspaceError::InvalidPathUtf8 { - path: default_ufo_path.clone(), - })?; - - let ufo_reader = UfoReader::new(); - let mut font = - ufo_reader - .load(default_ufo_str) - .map_err(|source| DesignspaceError::LoadUfo { - path: default_ufo_path.clone(), - details: source.to_string(), - })?; - let default_ufo_source_id = font - .default_source_id() - .ok_or(DesignspaceError::NoSources)?; - let default_ufo_source = font - .default_source() - .cloned() - .ok_or(DesignspaceError::NoSources)?; - font.clear_sources(); - - if let Some(family) = &default_source.familyname { - font.metadata_mut().family_name = Some(family.clone()); - } - - let mut source_names = HashSet::new(); - let default_name = axisless_source_name( - default_source, - font.metadata().style_name.as_deref(), - &source_names, - 0, - ); - source_names.insert(default_name.clone()); - let mut new_default_source = Source::with_filename( - default_name, - Location::new(), - default_source.filename.clone(), - ); - let metric_definitions = font.metric_definitions().to_vec(); - copy_source_metrics( - &metric_definitions, - &default_ufo_source, - &metric_definitions, - &mut new_default_source, - ); - let default_source_id = font.add_source(new_default_source); - font.set_default_source_id(default_source_id.clone()); - move_glyph_layers_to_source(&mut font, default_ufo_source_id, default_source_id)?; - - let mut ufo_cache: HashMap = HashMap::new(); - for (idx, ds_source) in sources.iter().enumerate().skip(1) { - let ufo_path = ds_dir.join(&ds_source.filename); - let ufo_str = ufo_path - .to_str() - .ok_or_else(|| DesignspaceError::InvalidPathUtf8 { - path: ufo_path.clone(), - })? - .to_string(); - - let source_font = match ufo_cache.get(&ufo_str) { - Some(f) => f, - None => { - let loaded = - ufo_reader - .load(&ufo_str) - .map_err(|source| DesignspaceError::LoadUfo { - path: ufo_path.clone(), - details: source.to_string(), - })?; - ufo_cache.insert(ufo_str.clone(), loaded); - ufo_cache.get(&ufo_str).unwrap() - } - }; - - let source_source_id = match &ds_source.layer { - Some(layer_name) => find_source_by_external_layer_name(source_font, layer_name) - .ok_or_else(|| DesignspaceError::MissingLayer { - layer: layer_name.clone(), - filename: ds_source.filename.clone(), - })?, - None => source_font - .default_source_id() - .ok_or(DesignspaceError::NoSources)?, - }; - - let name = axisless_source_name( - ds_source, - source_font.metadata().style_name.as_deref(), - &source_names, - idx, - ); - source_names.insert(name.clone()); - let mut source = Source::with_filename(name, Location::new(), ds_source.filename.clone()); - source.set_layer_name(ds_source.layer.clone()); - let imported_source = source_font - .sources() - .iter() - .find(|source| source.id() == source_source_id) - .ok_or(DesignspaceError::NoSources)?; - copy_source_metrics( - source_font.metric_definitions(), - imported_source, - font.metric_definitions(), - &mut source, - ); - let source_id = font.add_source(source); - - for source_glyph in source_font.glyphs() { - if let Some(source_layer) = source_glyph.layer_for_source(source_source_id.clone()) { - if let Some(glyph_id) = font.glyph_id_by_name(source_glyph.name()) { - font.insert_glyph_layer( - glyph_id, - clone_layer_with_remapped_components( - source_layer, - &font, - LayerId::new(), - source_id.clone(), - )?, - )?; - } - } - } - } - - remove_glyph_layers_without_source(&mut font)?; - Ok(font) +pub(super) struct AxislessSource { + pub(super) filename: String, + pub(super) familyname: Option, + pub(super) stylename: Option, + pub(super) name: Option, + pub(super) layer: Option, } -fn axisless_source_name( +pub(super) fn axisless_source_name( source: &AxislessSource, ufo_style_name: Option<&str>, used_names: &HashSet, @@ -641,7 +283,7 @@ fn axisless_source_name( ) } -fn parse_axisless_sources(xml: &str) -> DesignspaceResult> { +pub(super) fn parse_axisless_sources(xml: &str) -> DesignspaceResult> { let mut reader = Reader::from_str(xml); reader.config_mut().trim_text(true); let mut sources = Vec::new(); @@ -702,7 +344,7 @@ fn xml_attr( Ok(None) } -fn source_name( +pub(super) fn source_name( source: &norad::designspace::Source, ufo_style_name: Option<&str>, used_names: &HashSet, @@ -763,7 +405,7 @@ fn imported_source_name( } } -fn location_from_dimensions( +pub(super) fn location_from_dimensions( dimensions: &[norad::designspace::Dimension], doc: &DesignSpaceDocument, axes: &[Axis], @@ -847,7 +489,7 @@ fn map_axis_value(axis: &norad::designspace::Axis, user_value: f64) -> f64 { // design-space locations to the mapped user-space defaults. Layer-backed // sources remain eligible because `layer` describes storage, not source role. // https://fonttools.readthedocs.io/en/stable/designspaceLib/python.html#fontTools.designspaceLib.DesignSpaceDocument.findDefault -fn find_default_source_index(doc: &DesignSpaceDocument) -> Option { +pub(super) fn find_default_source_index(doc: &DesignSpaceDocument) -> Option { doc.sources.iter().position(|source| { doc.axes.iter().all(|axis| { let source_value = source_axis_design_value(&source.location, axis); @@ -862,105 +504,6 @@ fn design_values_equal(left: f64, right: f64) -> bool { (left - right).abs() <= f64::from(f32::EPSILON) * scale * 4.0 } -fn source_layer_id( - font: &Font, - source: &norad::designspace::Source, -) -> DesignspaceResult { - match &source.layer { - Some(layer_name) => find_source_by_external_layer_name(font, layer_name).ok_or_else(|| { - DesignspaceError::MissingLayer { - layer: layer_name.clone(), - filename: source.filename.clone(), - } - }), - None => font.default_source_id().ok_or(DesignspaceError::NoSources), - } -} - -fn find_source_by_external_layer_name(font: &Font, name: &str) -> Option { - font.sources() - .iter() - .find(|source| !source.is_master() && source.name() == name) - .map(Source::id) -} - -fn move_glyph_layers_to_source( - font: &mut Font, - from_source_id: SourceId, - to_source_id: SourceId, -) -> DesignspaceResult<()> { - let layer_moves: Vec<_> = font - .glyphs() - .filter_map(|glyph| { - glyph.layer_for_source(from_source_id.clone()).map(|layer| { - ( - glyph.id(), - layer.id(), - layer.clone_with_identity(LayerId::new(), to_source_id.clone()), - ) - }) - }) - .collect(); - - for (glyph_id, old_layer_id, layer) in layer_moves { - font.remove_glyph_layer(old_layer_id)?; - font.insert_glyph_layer(glyph_id, layer)?; - } - - Ok(()) -} - -fn clone_layer_with_remapped_components( - layer: &GlyphLayer, - font: &Font, - layer_id: LayerId, - source_id: SourceId, -) -> DesignspaceResult { - let mut cloned = layer.clone_with_identity(layer_id, source_id); - cloned.clear_components(); - - for component in layer.components_iter() { - let base_glyph_id = font - .glyph_id_by_name(component.base_glyph_name().as_str()) - .ok_or_else(|| DesignspaceError::LoadUfo { - path: std::path::PathBuf::from(component.base_glyph_name().as_str()), - details: format!( - "component base glyph {:?} does not exist in default font", - component.base_glyph_name() - ), - })?; - cloned.add_component(Component::with_id( - component.id(), - base_glyph_id, - component.base_glyph_name().clone(), - *component.transform(), - )); - } - - Ok(cloned) -} - -fn remove_glyph_layers_without_source(font: &mut Font) -> DesignspaceResult<()> { - let source_ids: Vec<_> = font.sources().iter().map(Source::id).collect(); - let orphan_layer_ids: Vec<_> = font - .glyphs() - .flat_map(|glyph| { - glyph - .layers() - .values() - .filter(|layer| !source_ids.contains(&layer.source_id())) - .map(|layer| layer.id()) - .collect::>() - }) - .collect(); - - for layer_id in orphan_layer_ids { - font.remove_glyph_layer(layer_id)?; - } - - Ok(()) -} - /// Derive (minimum, maximum) for an axis from norad's parsed designspace. /// /// Designspace axis edge cases handled: @@ -970,7 +513,7 @@ fn remove_glyph_layers_without_source(font: &mut Font) -> DesignspaceResult<()> /// - **One-sided** (only min OR max specified): the missing side falls back /// to `default`. Common with slant axes (`min=-15, default=0, max=0`). /// - **Degenerate** (no min/max/values): all three collapse to default. -fn derive_axis_range(ds_axis: &norad::designspace::Axis) -> (f64, f64) { +pub(super) fn derive_axis_range(ds_axis: &norad::designspace::Axis) -> (f64, f64) { let values_range = || { ds_axis .values diff --git a/crates/shift-backends/src/errors.rs b/crates/shift-backends/src/errors.rs index f4de719c..8e4a8320 100644 --- a/crates/shift-backends/src/errors.rs +++ b/crates/shift-backends/src/errors.rs @@ -20,6 +20,9 @@ pub enum BackendError { #[error("unsupported font format for writing: {extension}")] UnsupportedWriteFormat { extension: String }, + #[error("streaming import is not supported for {} fonts", format.name())] + StreamingUnsupported { format: FontFormat }, + #[error("font format adaptor is not registered: {}", format.name())] MissingAdaptor { format: FontFormat }, diff --git a/crates/shift-backends/src/font_loader.rs b/crates/shift-backends/src/font_loader.rs index 05106ff8..6410875f 100644 --- a/crates/shift-backends/src/font_loader.rs +++ b/crates/shift-backends/src/font_loader.rs @@ -8,14 +8,19 @@ use crate::designspace::{DesignspaceReader, DesignspaceWriter}; use crate::errors::{BackendError, BackendResult, FormatBackendError, FormatBackendResult}; use crate::format::FontFormat; use crate::glyphs::GlyphsReader; +use crate::import::{FontImport, GlyphStream}; use crate::traits::{FontReader, FontWriter}; use crate::ufo::{UfoReader, UfoWriter}; use crate::binary::BytesFontAdaptor; -pub trait FontAdaptor { +pub(crate) trait FontAdaptor { fn read_font(&self, path: &str) -> FormatBackendResult; fn write_font(&self, font: &Font, path: &str) -> FormatBackendResult<()>; + + fn stream(&self, _path: &str) -> FormatBackendResult)>> { + Ok(None) + } } struct UfoFontAdaptor; @@ -43,6 +48,11 @@ impl FontAdaptor for UfoFontAdaptor { fn write_font(&self, font: &Font, path: &str) -> FormatBackendResult<()> { UfoWriter::new().save(font, path) } + + fn stream(&self, path: &str) -> FormatBackendResult)>> { + let (header, stream) = crate::ufo::stream_font(path)?; + Ok(Some((header, Box::new(stream)))) + } } impl FontAdaptor for GlyphsFontAdaptor { @@ -63,6 +73,11 @@ impl FontAdaptor for DesignspaceFontAdaptor { fn write_font(&self, font: &Font, path: &str) -> FormatBackendResult<()> { DesignspaceWriter::new().save(font, path) } + + fn stream(&self, path: &str) -> FormatBackendResult)>> { + let (header, stream) = crate::designspace::stream_font(path)?; + Ok(Some((header, Box::new(stream)))) + } } pub struct FontLoader { @@ -101,6 +116,24 @@ fn extension_from_path(path: &Path) -> BackendResult<&str> { }) } +struct ResolvedPath<'a> { + format: FontFormat, + path: &'a str, + path_buf: std::path::PathBuf, + extension: &'a str, +} + +fn resolve(path: &str) -> BackendResult> { + let parsed = Path::new(path); + let extension = extension_from_path(parsed)?; + Ok(ResolvedPath { + format: format_from_extension(extension)?, + path, + path_buf: parsed.to_path_buf(), + extension, + }) +} + impl FontLoader { pub fn new() -> Self { let mut adaptors: HashMap> = HashMap::new(); @@ -118,48 +151,72 @@ impl FontLoader { self.adaptors.keys().collect() } + pub fn stream_font(&self, path: &str) -> BackendResult { + let resolved = resolve(path)?; + let adaptor = self + .adaptors + .get(&resolved.format) + .ok_or(BackendError::MissingAdaptor { + format: resolved.format, + })?; + let streamed = adaptor + .stream(resolved.path) + .map_err(|source| { + BackendError::load(resolved.format, resolved.path_buf.clone(), source) + })? + .ok_or(BackendError::StreamingUnsupported { + format: resolved.format, + })?; + Ok(FontImport::new( + streamed.0, + streamed.1, + resolved.format, + resolved.path_buf, + )) + } + pub fn read_font(&self, path: &str) -> BackendResult { - let path = Path::new(path); - let ext = extension_from_path(path)?; - let format = format_from_extension(ext)?; - let path_buf = path.to_path_buf(); + let resolved = resolve(path)?; let adaptor = self .adaptors - .get(&format) - .ok_or(BackendError::MissingAdaptor { format })?; - let path = path.to_str().ok_or_else(|| BackendError::InvalidPathUtf8 { - path: path_buf.clone(), + .get(&resolved.format) + .ok_or(BackendError::MissingAdaptor { + format: resolved.format, + })?; + let streamed = adaptor.stream(resolved.path).map_err(|source| { + BackendError::load(resolved.format, resolved.path_buf.clone(), source) })?; + if let Some((header, stream)) = streamed { + return FontImport::new(header, stream, resolved.format, resolved.path_buf) + .collect_font(); + } + adaptor - .read_font(path) - .map_err(|source| BackendError::load(format, path_buf, source)) + .read_font(resolved.path) + .map_err(|source| BackendError::load(resolved.format, resolved.path_buf, source)) } pub fn write_font(&self, font: &Font, path: &str) -> BackendResult<()> { - let path = Path::new(path); - let ext = extension_from_path(path)?; - let format = format_from_extension(ext)?; + let resolved = resolve(path)?; - match format { + match resolved.format { FontFormat::Ufo | FontFormat::Designspace | FontFormat::Shift => {} _ => { return Err(BackendError::UnsupportedWriteFormat { - extension: ext.to_string(), + extension: resolved.extension.to_string(), }) } } - let path_buf = path.to_path_buf(); let adaptor = self .adaptors - .get(&format) - .ok_or(BackendError::MissingAdaptor { format })?; - let path = path.to_str().ok_or_else(|| BackendError::InvalidPathUtf8 { - path: path_buf.clone(), - })?; + .get(&resolved.format) + .ok_or(BackendError::MissingAdaptor { + format: resolved.format, + })?; adaptor - .write_font(font, path) - .map_err(|source| BackendError::save(format, path_buf, source)) + .write_font(font, resolved.path) + .map_err(|source| BackendError::save(resolved.format, resolved.path_buf, source)) } } diff --git a/crates/shift-backends/src/import.rs b/crates/shift-backends/src/import.rs new file mode 100644 index 00000000..74573ee9 --- /dev/null +++ b/crates/shift-backends/src/import.rs @@ -0,0 +1,224 @@ +use std::{collections::HashMap, path::PathBuf}; + +use rayon::prelude::*; +use shift_font::{Font, Glyph, GlyphId, GlyphName, LayerId, SourceId}; + +use crate::{BackendError, BackendResult, FontFormat, FormatBackendResult}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GlyphDirectoryEntry { + pub glyph_id: GlyphId, + pub name: GlyphName, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ImportBatchLimit { + max_glyphs: usize, + max_layers: usize, +} + +impl ImportBatchLimit { + pub fn new(max_glyphs: usize, max_layers: usize) -> Self { + Self { + max_glyphs: max_glyphs.max(1), + max_layers: max_layers.max(1), + } + } + + pub fn max_glyphs(self) -> usize { + self.max_glyphs + } + + pub fn max_layers(self) -> usize { + self.max_layers + } +} + +impl Default for ImportBatchLimit { + fn default() -> Self { + Self::new(512, 1_024) + } +} + +#[derive(Clone)] +pub(crate) struct GlifLayer { + pub(crate) source_id: SourceId, + pub(crate) layer_id: LayerId, + pub(crate) path: PathBuf, +} + +#[derive(Clone)] +pub(crate) struct GlifGlyph { + pub(crate) id: GlyphId, + pub(crate) name: String, + pub(crate) layers: Vec, +} + +pub(crate) struct GlifGlyphStream { + glyph_ids: HashMap, + glyphs: Vec, + next_glyph: usize, +} + +impl GlifGlyphStream { + pub(crate) fn new(glyph_ids: HashMap, glyphs: Vec) -> Self { + Self { + glyph_ids, + glyphs, + next_glyph: 0, + } + } +} + +impl GlyphStream for GlifGlyphStream { + fn directory(&self) -> Vec { + self.glyphs + .iter() + .map(|glyph| GlyphDirectoryEntry { + glyph_id: glyph.id.clone(), + name: glyph.name.clone().into(), + }) + .collect() + } + + fn glyph_count(&self) -> usize { + self.glyphs.len() + } + + fn next_batch(&mut self, limit: ImportBatchLimit) -> FormatBackendResult> { + if self.next_glyph == self.glyphs.len() { + return Ok(Vec::new()); + } + + let mut end = self.next_glyph; + let mut layer_count = 0; + while end < self.glyphs.len() && end - self.next_glyph < limit.max_glyphs() { + let next_layers = self.glyphs[end].layers.len(); + if end > self.next_glyph && layer_count + next_layers > limit.max_layers() { + break; + } + layer_count += next_layers; + end += 1; + } + let glyph_ids = &self.glyph_ids; + let glyphs = self.glyphs[self.next_glyph..end] + .par_iter() + .map(|record| load_glif_glyph(record, glyph_ids)) + .collect::>>()?; + self.next_glyph = end; + Ok(glyphs) + } +} + +fn load_glif_glyph( + record: &GlifGlyph, + glyph_ids: &HashMap, +) -> FormatBackendResult { + let mut glyph = Glyph::with_id(record.id.clone(), record.name.clone()); + for (index, layer_record) in record.layers.iter().enumerate() { + let norad_glyph = norad::Glyph::load(&layer_record.path).map_err(|error| { + crate::FormatBackendError::Ufo(format!( + "failed to read glyph {:?} from {}: {error}", + record.name, + layer_record.path.display() + )) + })?; + if index == 0 { + glyph.set_unicodes(norad_glyph.codepoints.iter().map(u32::from).collect()); + if !norad_glyph.lib.is_empty() { + *glyph.lib_mut() = crate::ufo::UfoReader::convert_lib(&norad_glyph.lib); + } + } + let layer = crate::ufo::UfoReader::convert_stream_layer( + &norad_glyph, + layer_record.layer_id.clone(), + layer_record.source_id.clone(), + glyph_ids, + )?; + glyph.set_layer(layer); + } + Ok(glyph) +} + +pub(crate) trait GlyphStream: Send { + fn directory(&self) -> Vec; + fn glyph_count(&self) -> usize; + fn next_batch(&mut self, limit: ImportBatchLimit) -> FormatBackendResult>; +} + +pub(crate) fn collect_streamed_font( + mut header: Font, + stream: &mut dyn GlyphStream, +) -> FormatBackendResult { + let expected_glyph_count = stream.glyph_count(); + loop { + let glyphs = stream.next_batch(ImportBatchLimit::default())?; + if glyphs.is_empty() { + break; + } + for glyph in glyphs { + header.insert_glyph(glyph)?; + } + } + debug_assert_eq!(header.glyph_count(), expected_glyph_count); + Ok(header) +} + +/// A bounded foreign-font import. +/// +/// The header contains top-level authored state but no glyphs. Glyph geometry +/// is materialized only in batches requested by [`Self::next_batch`]. +pub struct FontImport { + header: Font, + stream: Box, + format: FontFormat, + path: PathBuf, +} + +impl FontImport { + pub(crate) fn new( + header: Font, + stream: Box, + format: FontFormat, + path: PathBuf, + ) -> Self { + Self { + header, + stream, + format, + path, + } + } + + pub fn header(&self) -> &Font { + &self.header + } + + pub fn directory(&self) -> Vec { + self.stream.directory() + } + + pub fn glyph_count(&self) -> usize { + self.stream.glyph_count() + } + + pub fn next_batch(&mut self, limit: ImportBatchLimit) -> BackendResult> { + self.stream + .next_batch(limit) + .map_err(|source| BackendError::load(self.format, self.path.clone(), source)) + } + + /// Drains the same bounded stream used by native import into one eager + /// font for compatibility callers. Streaming semantics are therefore the + /// only parsing semantics for formats that implement them. + pub fn collect_font(self) -> BackendResult { + let Self { + header, + mut stream, + format, + path, + } = self; + collect_streamed_font(header, stream.as_mut()) + .map_err(|source| BackendError::load(format, path, source)) + } +} diff --git a/crates/shift-backends/src/lib.rs b/crates/shift-backends/src/lib.rs index d8283c72..4fd2c06a 100644 --- a/crates/shift-backends/src/lib.rs +++ b/crates/shift-backends/src/lib.rs @@ -6,6 +6,7 @@ pub mod export; pub mod font_loader; pub mod format; pub mod glyphs; +pub mod import; mod metrics; mod shift2fontir; mod traits; @@ -14,4 +15,5 @@ pub mod ufo; pub use errors::{BackendError, BackendResult, FormatBackendError, FormatBackendResult}; pub use export::{ExportError, ExportFormat, FontExportRequest, FontExportResult, FontExporter}; pub use format::FontFormat; +pub use import::{FontImport, GlyphDirectoryEntry, ImportBatchLimit}; pub use traits::{FontBackend, FontReader, FontView, FontWriter}; diff --git a/crates/shift-backends/src/ufo/import.rs b/crates/shift-backends/src/ufo/import.rs new file mode 100644 index 00000000..f155b022 --- /dev/null +++ b/crates/shift-backends/src/ufo/import.rs @@ -0,0 +1,193 @@ +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + path::{Path, PathBuf}, +}; + +use norad::{DataRequest, Font as NoradFont}; +use shift_font::{Font, GlyphId, LayerId, Source, SourceId}; + +use crate::{ + errors::{FormatBackendError, FormatBackendResult}, + import::{GlifGlyph, GlifGlyphStream, GlifLayer}, +}; + +use super::UfoReader; + +struct LayerDirectory { + source_id: SourceId, + glyphs: BTreeMap, +} + +pub(crate) fn stream_font(path: &str) -> FormatBackendResult<(Font, GlifGlyphStream)> { + let ufo_path = Path::new(path); + let mut header = load_header(ufo_path)?; + let default_source_id = header.default_source_id().ok_or_else(|| { + FormatBackendError::Ufo("UFO header is missing its default source".into()) + })?; + let layers = load_layer_directories(ufo_path, &mut header, default_source_id)?; + let (glyph_ids, glyphs) = build_glyph_directory(layers); + + Ok((header, GlifGlyphStream::new(glyph_ids, glyphs))) +} + +fn load_layer_directories( + ufo_path: &Path, + header: &mut Font, + default_source_id: SourceId, +) -> FormatBackendResult> { + let mut layer_paths = read_layer_paths(ufo_path)?; + let default_index = layer_paths + .iter() + .position(|(_, path)| path == Path::new("glyphs")) + .ok_or_else(|| FormatBackendError::Ufo("UFO is missing its default glyphs layer".into()))?; + let default_layer = layer_paths.remove(default_index); + layer_paths.insert(0, default_layer); + + let mut layers = Vec::with_capacity(layer_paths.len()); + for (index, (name, relative_path)) in layer_paths.into_iter().enumerate() { + let layer_path = ufo_path.join(&relative_path); + let contents_path = layer_path.join("contents.plist"); + let glyphs: BTreeMap = plist_file(&contents_path)?; + let (color, lib) = read_layer_info(&layer_path)?; + let source_id = if index == 0 { + default_source_id.clone() + } else { + header.add_source(Source::layer(name)) + }; + let source = header.source_mut(source_id.clone()).ok_or_else(|| { + FormatBackendError::Ufo("newly registered UFO source is missing".into()) + })?; + source.set_color(color); + if !lib.is_empty() { + *source.lib_mut() = UfoReader::convert_lib(&lib); + } + layers.push(LayerDirectory { + source_id, + glyphs: glyphs + .into_iter() + .map(|(name, path)| (name, layer_path.join(path))) + .collect(), + }); + } + Ok(layers) +} + +pub(crate) fn load_header(ufo_path: &Path) -> FormatBackendResult { + let request = DataRequest::all().layers(false); + let norad_font = NoradFont::load_requested_data(ufo_path, request) + .map_err(|error| FormatBackendError::Ufo(error.to_string()))?; + UfoReader::convert_header(&norad_font, ufo_path) +} + +pub(crate) fn read_glyph_paths( + ufo_path: &Path, + layer_name: Option<&str>, +) -> FormatBackendResult> { + let layer_paths = read_layer_paths(ufo_path)?; + let relative_path = match layer_name { + Some(layer_name) => layer_paths + .iter() + .find(|(name, _)| name == layer_name) + .map(|(_, path)| path), + None => layer_paths + .iter() + .find(|(_, path)| path == Path::new("glyphs")) + .map(|(_, path)| path), + } + .ok_or_else(|| { + FormatBackendError::Ufo(format!( + "UFO layer {:?} does not exist in {}", + layer_name.unwrap_or("public.default"), + ufo_path.display() + )) + })?; + let layer_path = ufo_path.join(relative_path); + let contents_path = layer_path.join("contents.plist"); + let glyphs: BTreeMap = plist_file(&contents_path)?; + Ok(glyphs + .into_iter() + .map(|(name, path)| (name, layer_path.join(path))) + .collect()) +} + +fn read_layer_paths(ufo_path: &Path) -> FormatBackendResult> { + let path = ufo_path.join("layercontents.plist"); + if !path.exists() { + // UFO 2 predates layercontents.plist and defines only glyphs/ as the + // default layer. Avoid loading a complete norad::Font just to discover + // this standard legacy layout. + return Ok(vec![("public.default".into(), PathBuf::from("glyphs"))]); + } + plist_file(&path) +} + +fn plist_file(path: &Path) -> FormatBackendResult { + plist::from_file(path).map_err(|error| { + FormatBackendError::Ufo(format!("failed to read {}: {error}", path.display())) + }) +} + +fn read_layer_info(layer_path: &Path) -> FormatBackendResult<(Option, plist::Dictionary)> { + let path = layer_path.join("layerinfo.plist"); + if !path.exists() { + return Ok((None, plist::Dictionary::new())); + } + let value = plist::Value::from_file(&path).map_err(|error| { + FormatBackendError::Ufo(format!("failed to read {}: {error}", path.display())) + })?; + let Some(dictionary) = value.as_dictionary() else { + return Err(FormatBackendError::Ufo(format!( + "{} is not a dictionary", + path.display() + ))); + }; + let color = dictionary + .get("color") + .and_then(plist::Value::as_string) + .map(ToOwned::to_owned); + let lib = dictionary + .get("lib") + .and_then(plist::Value::as_dictionary) + .cloned() + .unwrap_or_default(); + Ok((color, lib)) +} + +fn build_glyph_directory( + layers: Vec, +) -> (HashMap, Vec) { + let mut ordered_names = Vec::new(); + let mut seen = HashSet::new(); + for layer in &layers { + for name in layer.glyphs.keys() { + if seen.insert(name.clone()) { + ordered_names.push(name.clone()); + } + } + } + let glyph_ids = ordered_names + .iter() + .map(|name| (name.clone(), GlyphId::new())) + .collect::>(); + let glyphs = ordered_names + .into_iter() + .map(|name| { + let layer_records = layers + .iter() + .filter_map(|layer| { + layer.glyphs.get(&name).map(|path| GlifLayer { + source_id: layer.source_id.clone(), + layer_id: LayerId::new(), + path: path.clone(), + }) + }) + .collect(); + GlifGlyph { + id: glyph_ids[&name].clone(), + name, + layers: layer_records, + } + }) + .collect(); + (glyph_ids, glyphs) +} diff --git a/crates/shift-backends/src/ufo/mod.rs b/crates/shift-backends/src/ufo/mod.rs index df379485..7279994c 100644 --- a/crates/shift-backends/src/ufo/mod.rs +++ b/crates/shift-backends/src/ufo/mod.rs @@ -1,6 +1,8 @@ +mod import; mod reader; mod writer; +pub(crate) use import::{load_header, read_glyph_paths, stream_font}; pub use reader::UfoReader; pub use writer::UfoWriter; @@ -179,7 +181,7 @@ mod tests { let mut broken = create_test_font(); broken.lib_mut().set( "public.objectLibs".to_string(), - shift_font::LibValue::Dict(std::collections::HashMap::new()), + shift_font::LibValue::Dict(std::collections::BTreeMap::new()), ); UfoWriter::new() .save(&broken, ufo_path_str) diff --git a/crates/shift-backends/src/ufo/reader.rs b/crates/shift-backends/src/ufo/reader.rs index a98fdbbc..b7be707d 100644 --- a/crates/shift-backends/src/ufo/reader.rs +++ b/crates/shift-backends/src/ufo/reader.rs @@ -1,13 +1,14 @@ use crate::errors::{FormatBackendError, FormatBackendResult}; +use crate::import::collect_streamed_font; use crate::metrics::set_metric_position; use crate::traits::FontReader; use norad::{Font as NoradFont, Line}; use shift_font::{ - Anchor, Component, Contour, FeatureData, Font, Glyph, GlyphLayer, Guideline, KerningData, - KerningPair, KerningSide, LayerId, LibData, LibValue, MetricKind, PointType, Source, SourceId, + Anchor, Component, Contour, FeatureData, Font, GlyphId, GlyphLayer, Guideline, KerningData, + KerningPair, KerningSide, LayerId, LibData, LibValue, MetricKind, PointType, SourceId, Transform, }; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::path::Path; pub struct UfoReader; @@ -45,7 +46,6 @@ pub(crate) const MAPPED_FONTINFO_KEYS: &[&str] = &[ ]; struct PendingComponent { - layer_id: LayerId, base_glyph_name: String, matrix: Transform, } @@ -65,7 +65,7 @@ impl UfoReader { } } - fn convert_contour(contour: &norad::Contour) -> Contour { + pub(crate) fn convert_contour(contour: &norad::Contour) -> Contour { let mut shift_contour = Contour::new(); let is_closed = contour.is_closed(); @@ -81,9 +81,8 @@ impl UfoReader { shift_contour } - fn convert_component(layer_id: LayerId, component: &norad::Component) -> PendingComponent { + fn convert_component(component: &norad::Component) -> PendingComponent { PendingComponent { - layer_id, base_glyph_name: component.base.to_string(), matrix: Transform { xx: component.transform.x_scale, @@ -96,7 +95,7 @@ impl UfoReader { } } - fn convert_anchor(anchor: &norad::Anchor) -> Anchor { + pub(crate) fn convert_anchor(anchor: &norad::Anchor) -> Anchor { Anchor::new( anchor.name.as_ref().map(|name| name.to_string()), anchor.x, @@ -104,7 +103,7 @@ impl UfoReader { ) } - fn convert_guideline(guideline: &norad::Guideline) -> Guideline { + pub(crate) fn convert_guideline(guideline: &norad::Guideline) -> Guideline { let mut converted = match guideline.line { Line::Horizontal(y) => Guideline::horizontal(y), Line::Vertical(x) => Guideline::vertical(x), @@ -131,7 +130,7 @@ impl UfoReader { LibValue::Array(arr.iter().map(Self::convert_plist_to_lib_value).collect()) } plist::Value::Dictionary(dict) => { - let mut map = HashMap::new(); + let mut map = BTreeMap::new(); for (k, v) in dict.iter() { map.insert(k.clone(), Self::convert_plist_to_lib_value(v)); } @@ -177,21 +176,20 @@ impl UfoReader { } } - fn convert_lib(lib: &plist::Dictionary) -> LibData { - let mut data = HashMap::new(); + pub(crate) fn convert_lib(lib: &plist::Dictionary) -> LibData { + let mut data = BTreeMap::new(); for (k, v) in lib.iter() { data.insert(k.clone(), Self::convert_plist_to_lib_value(v)); } LibData::from_map(data) } - fn convert_glyph_layer( + fn convert_layer_geometry( norad_glyph: &norad::Glyph, layer_id: LayerId, source_id: SourceId, - ) -> (Glyph, Vec) { + ) -> (GlyphLayer, Vec) { let mut glyph_layer = GlyphLayer::with_width(layer_id, source_id, norad_glyph.width); - let layer_id = glyph_layer.id(); let mut pending_components = Vec::new(); if norad_glyph.height != 0.0 { glyph_layer.set_height(Some(norad_glyph.height)); @@ -202,7 +200,7 @@ impl UfoReader { } for component in &norad_glyph.components { - pending_components.push(Self::convert_component(layer_id.clone(), component)); + pending_components.push(Self::convert_component(component)); } for anchor in &norad_glyph.anchors { @@ -217,17 +215,31 @@ impl UfoReader { *glyph_layer.lib_mut() = Self::convert_lib(&norad_glyph.lib); } - let mut glyph = Glyph::new(norad_glyph.name().to_string()); - for codepoint in norad_glyph.codepoints.iter() { - glyph.add_unicode(u32::from(codepoint)); - } + (glyph_layer, pending_components) + } - if !norad_glyph.lib.is_empty() { - *glyph.lib_mut() = Self::convert_lib(&norad_glyph.lib); + pub(crate) fn convert_stream_layer( + norad_glyph: &norad::Glyph, + layer_id: LayerId, + source_id: SourceId, + glyph_ids: &HashMap, + ) -> FormatBackendResult { + let (mut layer, pending_components) = + Self::convert_layer_geometry(norad_glyph, layer_id, source_id); + for pending in pending_components { + let base_glyph_id = glyph_ids.get(&pending.base_glyph_name).ok_or_else(|| { + FormatBackendError::Ufo(format!( + "component base glyph {:?} does not exist", + pending.base_glyph_name + )) + })?; + layer.add_component(Component::with_matrix( + base_glyph_id.clone(), + pending.base_glyph_name, + &pending.matrix, + )); } - - glyph.set_layer(glyph_layer); - (glyph, pending_components) + Ok(layer) } fn convert_kerning(norad_font: &NoradFont) -> KerningData { @@ -283,48 +295,10 @@ impl UfoReader { } } - fn resolve_components( - font: &mut Font, - pending_components: Vec, - ) -> FormatBackendResult<()> { - for pending in pending_components { - let base_glyph_id = - font.glyph_id_by_name(&pending.base_glyph_name) - .ok_or_else(|| { - FormatBackendError::Ufo(format!( - "component base glyph {:?} does not exist", - pending.base_glyph_name - )) - })?; - let layer = font.layer_mut(pending.layer_id.clone()).ok_or_else(|| { - FormatBackendError::Ufo(format!( - "component target layer {} does not exist", - pending.layer_id - )) - })?; - layer.add_component(Component::with_matrix( - base_glyph_id, - pending.base_glyph_name, - &pending.matrix, - )); - } - - Ok(()) - } -} - -impl Default for UfoReader { - fn default() -> Self { - Self::new() - } -} - -impl FontReader for UfoReader { - fn load(&self, path: &str) -> FormatBackendResult { - let norad_font = - NoradFont::load(path).map_err(|e| FormatBackendError::Ufo(e.to_string()))?; - let ufo_path = Path::new(path); - + pub(crate) fn convert_header( + norad_font: &NoradFont, + ufo_path: &Path, + ) -> FormatBackendResult { let mut font = Font::new(); let default_source_id = font .default_source_id() @@ -350,15 +324,14 @@ impl FontReader for UfoReader { font.metadata_mut().description = norad_font.font_info.open_type_name_description.clone(); font.metadata_mut().note = norad_font.font_info.note.clone(); - let upm = norad_font + font.metrics_mut().units_per_em = norad_font .font_info .units_per_em - .map(|n| *n) + .map(|value| *value) .unwrap_or(1000.0); - font.metrics_mut().units_per_em = upm; let metric_definitions = font.metric_definitions().to_vec(); let default_source = font - .source_mut(default_source_id.clone()) + .source_mut(default_source_id) .expect("new font should contain its default source"); set_metric_position( &metric_definitions, @@ -398,69 +371,43 @@ impl FontReader for UfoReader { if let Some(remainder) = Self::convert_fontinfo_remainder(&norad_font.font_info)? { *font.fontinfo_remainder_mut() = remainder; } - - let norad_default_layer_name = norad_font.layers.default_layer().name().clone(); - let mut pending_components = Vec::new(); - for layer in norad_font.layers.iter() { - let source_id = if layer.name() == &norad_default_layer_name { - default_source_id.clone() - } else { - // Non-default UFO layers are carried as layer-only sources: - // they are not designspace masters and must not gain - // `` entries if the font is saved as a designspace. - font.add_source(Source::layer(layer.name().to_string())) - }; - - if let Some(source) = font.source_mut(source_id.clone()) { - source.set_color(layer.color.as_ref().map(|color| color.to_rgba_string())); - if !layer.lib.is_empty() { - *source.lib_mut() = Self::convert_lib(&layer.lib); - } - } - - for norad_glyph in layer.iter() { - let (glyph, glyph_components) = - Self::convert_glyph_layer(norad_glyph, LayerId::new(), source_id.clone()); - pending_components.extend(glyph_components); - - if let Some(glyph_id) = font.glyph_id_by_name(glyph.name()) { - for layer_data in glyph.layers().values() { - font.insert_glyph_layer(glyph_id.clone(), layer_data.as_ref().clone())?; - } - } else { - font.insert_glyph(glyph)?; - } - } - } - Self::resolve_components(&mut font, pending_components)?; - - *font.kerning_mut() = Self::convert_kerning(&norad_font); + *font.kerning_mut() = Self::convert_kerning(norad_font); *font.features_mut() = Self::load_features(ufo_path); - for guideline in norad_font.guidelines() { font.add_guideline(Self::convert_guideline(guideline)); } - if !norad_font.lib.is_empty() { *font.lib_mut() = Self::convert_lib(&norad_font.lib); } - for (data_path, contents) in norad_font.data.iter() { - let bytes = contents.map_err(|e| { - FormatBackendError::Ufo(format!("failed to read data file {data_path:?}: {e}")) + let bytes = contents.map_err(|error| { + FormatBackendError::Ufo(format!("failed to read data file {data_path:?}: {error}")) })?; font.data_files_mut() .insert(data_path.to_string_lossy().into_owned(), bytes.to_vec()); } - for (image_path, contents) in norad_font.images.iter() { - let bytes = contents.map_err(|e| { - FormatBackendError::Ufo(format!("failed to read image file {image_path:?}: {e}")) + let bytes = contents.map_err(|error| { + FormatBackendError::Ufo(format!( + "failed to read image file {image_path:?}: {error}" + )) })?; font.images_mut() .insert(image_path.to_string_lossy().into_owned(), bytes.to_vec()); } - Ok(font) } } + +impl Default for UfoReader { + fn default() -> Self { + Self::new() + } +} + +impl FontReader for UfoReader { + fn load(&self, path: &str) -> FormatBackendResult { + let (header, mut stream) = super::stream_font(path)?; + collect_streamed_font(header, &mut stream) + } +} diff --git a/crates/shift-backends/tests/loading.rs b/crates/shift-backends/tests/loading.rs index 52db4acc..58240c71 100644 --- a/crates/shift-backends/tests/loading.rs +++ b/crates/shift-backends/tests/loading.rs @@ -1,4 +1,7 @@ -use std::path::{Path, PathBuf}; +use std::{ + fs, + path::{Path, PathBuf}, +}; use shift_backends::font_loader::FontLoader; use shift_font::{Contour, Font, Glyph, GlyphLayer, LayerId, MetricKind, PointType}; @@ -12,6 +15,19 @@ fn fixtures_path() -> PathBuf { .join("fixtures") } +fn copy_directory(source: &Path, destination: &Path) { + fs::create_dir_all(destination).unwrap(); + for entry in fs::read_dir(source).unwrap() { + let entry = entry.unwrap(); + let destination_path = destination.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_directory(&entry.path(), &destination_path); + } else { + fs::copy(entry.path(), destination_path).unwrap(); + } + } +} + fn mutatorsans_ufo_path() -> PathBuf { fixtures_path().join("fonts/mutatorsans/MutatorSansLightCondensed.ufo") } @@ -52,6 +68,25 @@ fn load_font(path: &Path) -> Font { .unwrap_or_else(|error| panic!("failed to load {}: {error}", path.display())) } +fn stream_font(path: &Path) -> Font { + let import = FontLoader::new() + .stream_font(path.to_str().unwrap()) + .unwrap_or_else(|error| panic!("failed to stream {}: {error}", path.display())); + assert_eq!(import.header().glyph_count(), 0); + let directory = import.directory(); + let expected_count = import.glyph_count(); + assert_eq!(directory.len(), expected_count); + let font = import.collect_font().unwrap(); + assert_eq!(font.glyph_count(), expected_count); + for entry in directory { + assert_eq!( + font.glyph(entry.glyph_id).map(Glyph::glyph_name), + Some(&entry.name) + ); + } + font +} + fn main_layer(glyph: &Glyph) -> &GlyphLayer { glyph .layers() @@ -176,6 +211,129 @@ fn loads_binary_fonts_with_contours() { } } +#[test] +fn streams_binary_ufo_and_designspace_without_eager_glyphs() { + let binary_path = mutatorsans_ttf_path(); + let binary_bytes = std::fs::read(&binary_path).unwrap(); + let binary = skrifa::FontRef::new(&binary_bytes).unwrap(); + let expected_binary_glyphs = skrifa::raw::TableProvider::maxp(&binary) + .unwrap() + .num_glyphs() as usize; + let streamed_binary = stream_font(&binary_path); + assert_eq!(streamed_binary.glyph_count(), expected_binary_glyphs); + + for path in [binary_path, mutatorsans_ufo_path()] { + let eager = load_font(&path); + let streamed = stream_font(&path); + assert_eq!(streamed.glyph_count(), eager.glyph_count()); + let glyph_a = streamed + .glyphs_by_unicode(65) + .next() + .unwrap_or_else(|| panic!("{} should stream U+0041", path.display())); + assert!(!main_layer(glyph_a).contours().is_empty()); + } + + let path = mutatorsans_designspace_path(); + let eager = load_font(&path); + let streamed = stream_font(&path); + assert_eq!(streamed.glyph_count(), eager.glyph_count()); + assert_eq!(streamed.axes().len(), eager.axes().len()); + assert_eq!(streamed.sources().len(), eager.sources().len()); + assert_eq!( + streamed + .glyphs() + .map(Glyph::name) + .collect::>(), + eager + .glyphs() + .map(Glyph::name) + .collect::>() + ); +} + +#[test] +fn ufo_import_keeps_nondefault_layer_order_when_default_was_not_first() { + let temp = tempfile::tempdir().unwrap(); + let ufo_path = temp.path().join("Reordered.ufo"); + copy_directory(&mutatorsans_ufo_path(), &ufo_path); + let layercontents_path = ufo_path.join("layercontents.plist"); + let mut layercontents = plist::Value::from_file(&layercontents_path).unwrap(); + let layers = layercontents.as_array_mut().unwrap(); + let default = layers.remove(0); + layers.insert(2, default); + layercontents.to_file_xml(&layercontents_path).unwrap(); + + let font = load_font(&ufo_path); + let source_names = font + .sources() + .iter() + .map(|source| source.name().to_string()) + .collect::>(); + + assert_eq!( + &source_names[..3], + &["Regular", "support", "support.crossbar"] + ); +} + +#[test] +fn streaming_batches_preserve_published_directory_order() { + for path in [ + mutatorsans_ttf_path(), + mutatorsans_ufo_path(), + mutatorsans_designspace_path(), + ] { + let mut import = FontLoader::new() + .stream_font(path.to_str().unwrap()) + .unwrap(); + let expected = import + .directory() + .into_iter() + .map(|entry| (entry.glyph_id, entry.name)) + .collect::>(); + let mut actual = Vec::new(); + loop { + let batch = import + .next_batch(shift_backends::ImportBatchLimit::new(7, 20)) + .unwrap(); + if batch.is_empty() { + break; + } + actual.extend( + batch + .into_iter() + .map(|glyph| (glyph.id(), glyph.glyph_name().clone())), + ); + } + + assert_eq!( + actual, + expected, + "stream order changed for {}", + path.display() + ); + } +} + +#[test] +fn streaming_batches_bound_authored_layers_across_sources() { + let path = mutatorsans_designspace_path(); + let mut import = FontLoader::new() + .stream_font(path.to_str().unwrap()) + .unwrap(); + let glyphs = import + .next_batch(shift_backends::ImportBatchLimit::new(512, 4)) + .unwrap(); + let layer_count = glyphs + .iter() + .map(|glyph| glyph.layers().len()) + .sum::(); + + assert!(!glyphs.is_empty()); + assert!(glyphs.len() < import.glyph_count()); + assert!(layer_count <= 4 || glyphs.len() == 1); +} + #[test] fn loads_binary_variable_axes_and_named_instances() { let font = load_font(&host_grotesk_variable_ttf_path()); @@ -207,7 +365,7 @@ fn loads_binary_variable_axes_and_named_instances() { assert_eq!(regular.postscript_name(), Some("HostGrotesk-Regular")); } -fn assert_cubic_point_runs(contour: &Contour, context: &str) { +fn assert_curve_point_runs(contour: &Contour, context: &str) { let points = contour.points(); assert!(!points.is_empty(), "empty contour in {context}"); assert!( @@ -226,16 +384,29 @@ fn assert_cubic_point_runs(contour: &Contour, context: &str) { ); off_run = 0; } - other => panic!("unexpected point type {other:?} in {context}"), + PointType::QCurve => { + assert_eq!( + off_run, 1, + "qcurve point preceded by {off_run} off-curves in {context}" + ); + off_run = 0; + } } } if contour.is_closed() { - assert!( - off_run == 0 || off_run == 2, - "closing segment has {off_run} off-curves in {context}" - ); let first = &points[0]; + match first.point_type() { + PointType::QCurve => assert_eq!( + off_run, 1, + "closing qcurve has {off_run} off-curves in {context}" + ), + PointType::OnCurve => assert!( + off_run == 0 || off_run == 2, + "closing segment has {off_run} off-curves in {context}" + ), + PointType::OffCurve => unreachable!("first point is known to be on-curve"), + } let last = &points[points.len() - 1]; assert!( points.len() == 1 @@ -251,7 +422,7 @@ fn assert_cubic_point_runs(contour: &Contour, context: &str) { } #[test] -fn binary_import_produces_valid_cubic_point_runs() { +fn binary_import_produces_valid_curve_point_runs() { for path in [mutatorsans_ttf_path(), mutatorsans_otf_path()] { let font = load_font(&path); let mut curve_contours = 0; @@ -259,7 +430,7 @@ fn binary_import_produces_valid_cubic_point_runs() { for layer in glyph.layers().values() { for contour in layer.contours_iter() { let context = format!("glyph '{}' in {}", glyph.name(), path.display()); - assert_cubic_point_runs(contour, &context); + assert_curve_point_runs(contour, &context); if contour .points() .iter() @@ -427,6 +598,20 @@ fn loads_designspace_sources_axes_and_default_metadata() { assert_eq!(font.sources()[0].location().get(&width_axis_id), Some(0.0)); assert_eq!(font.sources()[0].location().get(&weight_axis_id), Some(0.0)); assert!(font.sources()[0].filename().is_some()); + let bold_condensed = font + .sources() + .iter() + .find(|source| { + source.filename() == Some("MutatorSansBoldCondensed.ufo") + && source.layer_name().is_none() + }) + .expect("non-default Bold Condensed master should be imported"); + assert_eq!( + font.metric_value(bold_condensed.id(), MetricKind::Ascender) + .expect("non-default master ascender should survive import") + .position, + 800.0 + ); assert_eq!(font.named_instances().len(), 14); let medium_wide = font .named_instances() diff --git a/crates/shift-backends/tests/round_trip/ufo.rs b/crates/shift-backends/tests/round_trip/ufo.rs index 96eb1ced..1573ee65 100644 --- a/crates/shift-backends/tests/round_trip/ufo.rs +++ b/crates/shift-backends/tests/round_trip/ufo.rs @@ -532,7 +532,8 @@ fn ttf_import_saves_spec_valid_ufo_curves() { let norad_font = norad::Font::load(&output_path) .expect("norad should parse every glif written from a TTF import"); - let mut curve_points = 0; + let mut cubic_points = 0; + let mut quadratic_points = 0; for layer in norad_font.layers.iter() { for glyph in layer.iter() { for contour in &glyph.contours { @@ -556,24 +557,32 @@ fn ttf_import_saves_spec_valid_ufo_curves() { "curve point in glyph '{}' is not preceded by exactly two off-curves", glyph.name() ); - curve_points += 1; + cubic_points += 1; } norad::PointType::OffCurve => { let next = &points[(index + 1) % points.len()]; assert!( matches!( next.typ, - norad::PointType::OffCurve | norad::PointType::Curve + norad::PointType::OffCurve + | norad::PointType::Curve + | norad::PointType::QCurve ), - "off-curve in glyph '{}' is not part of a cubic curve segment", + "off-curve in glyph '{}' is not part of a curve segment", glyph.name() ); } norad::PointType::QCurve => { - panic!( - "TTF import should not produce qcurve points (glyph '{}')", + let len = points.len(); + let prev = &points[(index + len - 1) % len]; + let prev_prev = &points[(index + len - 2) % len]; + assert!( + prev.typ == norad::PointType::OffCurve + && prev_prev.typ != norad::PointType::OffCurve, + "qcurve point in glyph '{}' is not preceded by exactly one off-curve", glyph.name() ); + quadratic_points += 1; } norad::PointType::Line | norad::PointType::Move => {} } @@ -582,7 +591,11 @@ fn ttf_import_saves_spec_valid_ufo_curves() { } } assert!( - curve_points > 0, + cubic_points + quadratic_points > 0, "TTF import round trip should contain curve segments" ); + assert!( + quadratic_points > 0, + "TrueType quadratic segments should remain qcurves after UFO export" + ); } diff --git a/crates/shift-bridge/__test__/index.spec.mjs b/crates/shift-bridge/__test__/index.spec.mjs index 42da4d0b..ba3a4390 100644 --- a/crates/shift-bridge/__test__/index.spec.mjs +++ b/crates/shift-bridge/__test__/index.spec.mjs @@ -3,6 +3,7 @@ import { createRequire } from "module"; import { existsSync, mkdtempSync, rmSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; +import { fileURLToPath } from "url"; const require = createRequire(import.meta.url); const { Bridge } = require("../index.js"); @@ -10,6 +11,7 @@ const { Bridge } = require("../index.js"); const shortIdAlphabet = "useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict"; const shortIdLength = 10; const shortIdAlphabetMask = shortIdAlphabet.length - 1; +const repositoryRoot = fileURLToPath(new URL("../../..", import.meta.url)); function mintId(prefix) { const bytes = new Uint8Array(shortIdLength); @@ -147,6 +149,37 @@ describe("Bridge", () => { expect(existsSync(outputPath)).toBe(true); }); + function resumeMutatorPackage() { + const sourcePath = join(repositoryRoot, "fixtures/fonts/mutatorsans/MutatorSans.ttf"); + const storePath = join(tempDir, "mutatorsans.sqlite"); + const packagePath = join(tempDir, "MutatorSans.shift"); + bridge.openWorkspace(sourcePath, storePath); + bridge.saveWorkspaceAs(packagePath); + bridge.closeWorkspace(); + bridge.resumeWorkspaceForSource(storePath, packagePath); + } + + it("acquires every lazy glyph payload before preparing the complete Slug atlas", () => { + resumeMutatorPackage(); + + const atlas = bridge.prepareSlugAtlas(256); + expect(atlas.glyphs).toHaveLength(bridge.getGlyphs().length); + expect(atlas.curveCount).toBeGreaterThan(0); + expect(atlas.layout.totalLength).toBeGreaterThan(0); + }); + + it("acquires lazy glyph payloads before preparing a Slug atlas patch", () => { + resumeMutatorPackage(); + + const glyph = bridge.getGlyphs().find((record) => record.name === "S"); + expect(glyph).toBeDefined(); + + const atlas = bridge.prepareSlugAtlasPage([glyph.id], 256); + expect(atlas.glyphs).toContainEqual(expect.objectContaining({ glyphId: glyph.id })); + expect(atlas.curveCount).toBeGreaterThan(0); + expect(atlas.layout.totalLength).toBeGreaterThan(0); + }); + it("adds a point and echoes structure, values, and the client-minted ids", () => { const layerId = createGlyphLayer(); const contourId = addContour(layerId); diff --git a/crates/shift-bridge/docs/DOCS.md b/crates/shift-bridge/docs/DOCS.md index ed08f1b2..a0f2f215 100644 --- a/crates/shift-bridge/docs/DOCS.md +++ b/crates/shift-bridge/docs/DOCS.md @@ -12,7 +12,9 @@ NAPI bindings that expose the Rust font engine to Node.js and Electron as a `Bri **Architecture Invariant:** Bulk position updates use `Float64Array` values plus typed node descriptors. **WHY:** Drag updates keep the hot path in flat numeric buffers while IDs remain branded strings at the API boundary. -**Architecture Invariant:** Export uses a clone/COW `FontSaveSnapshot` of the current workspace font. **WHY:** Async export can run from a stable view without coupling output generation to renderer focus state. +**Architecture Invariant:** Export explicitly acquires all persisted layers before taking a clone/COW `FontSaveSnapshot`. **WHY:** Async export gets a complete stable view while ordinary workspace open remains directory-first and lazy. + +**Architecture Invariant:** Glyph snapshot, projection, preview, and Slug preparation methods are explicit acquisition boundaries. They may load requested BLOBs in the serialized utility process; retained renderer glyph objects and synchronous getters never initiate I/O. Component closure comes from the relational reference index rather than BLOB scans, and acquired payloads remain resident in the workspace cache. **Architecture Invariant:** `shift-font` constructs typed glyph and source-metric interpolation; renderer code only evaluates their flattened transport snapshots. **WHY:** Per-location canvas work stays cheap without moving variation-model construction or value-layout ownership into transport code. @@ -57,12 +59,12 @@ crates/shift-bridge/ 2. JS calls a mutation such as `closeContour(layerId, contourId)`. 3. `Bridge` parses boundary strings and asks `FontWorkspace` to run the edit against that layer. 4. The bridge returns a `shift-wire` change DTO and bumps the live version. -5. Full glyph snapshots include authored layer state plus the same `GlyphProjection` used by lightweight reads. `getGlyphProjections()` batches roots and transitive component projections without resolving a design location. Source reads expose master sources only; layer-only/background sources remain native authoring details and never enter renderer interpolation. +5. Full glyph snapshots first acquire requested layer payloads, then include authored state plus the same `GlyphProjection` used by lightweight reads. `getGlyphProjections()` and previews expand transitive component identities through SQLite indexes, acquire those layers, and only then project without further I/O. Source reads expose master sources only; layer-only/background sources remain native authoring details and never enter renderer interpolation. 6. `saveWorkspace()` / `saveWorkspaceAs(path)` update the `.shift` source package target and record the persisted version. 7. `inspectPackage(path)` and `inspectPackageDraft(storePath)` expose source/package identity for the utility process without choosing a recovery policy. 8. `closeWorkspace()` drops the live Rust workspace handle before the utility process deletes a clean or discarded SQLite document. 9. `exportWorkspace(request)` creates a `FontSaveSnapshot` and exports asynchronously through `shift-backends`. -10. `prepareSlugAtlasPage(glyphIds, alignment)` compiles only ordered roots and their component closures; its stream/discard methods retain the bounded complete-atlas transport contract. `prepareSlugAtlas(alignment)` remains the complete-font diagnostic path. Every font edit invalidates an unconsumed generation or page. +10. `prepareSlugAtlas(alignment)` acquires all layers for the catalog's complete resident generation. `prepareSlugAtlasPage(glyphIds, alignment)` acquires the ordered roots and their indexed component closures for a local edit patch. Acquired payloads remain in the workspace cache, while stream/discard methods retain the bounded atlas transport contract. Every font edit invalidates an unconsumed generation or patch. ## Type Boundary diff --git a/crates/shift-bridge/src/bridge.rs b/crates/shift-bridge/src/bridge.rs index e4dbf204..729a61f9 100644 --- a/crates/shift-bridge/src/bridge.rs +++ b/crates/shift-bridge/src/bridge.rs @@ -32,7 +32,8 @@ use shift_wire::{ SourceMetricsInterpolationSnapshot, }; use shift_workspace::{ - FontWorkspace, NewWorkspace, PackageDraft, PackageIdentity, WorkspaceError, WorkspaceSource, + AcquireScope, FontWorkspace, NewWorkspace, PackageDraft, PackageIdentity, WorkspaceError, + WorkspaceSource, }; use std::{ collections::{HashSet, VecDeque}, @@ -543,13 +544,19 @@ impl Bridge { #[napi] pub fn get_glyphs(&self) -> errors::Result> { - let mut records: Vec<_> = self - .font()? + let workspace = self.workspace()?; + let mut component_references = workspace.glyph_component_references()?; + let mut records = workspace + .font() .glyphs() - .map(GlyphRecord::from) - .map(Into::into) - .collect(); - records.sort_by(|a: &NapiGlyphRecord, b: &NapiGlyphRecord| a.name.cmp(&b.name)); + .map(|glyph| { + let mut record = GlyphRecord::from(glyph); + record.component_base_glyph_ids = + component_references.remove(&glyph.id()).unwrap_or_default(); + NapiGlyphRecord::from(record) + }) + .collect::>(); + records.sort_by(|a, b| a.name.cmp(&b.name)); Ok(records) } @@ -623,8 +630,8 @@ impl Bridge { .map(|touched| touched.layer.id()) .collect(); let dependents = self - .font()? - .dependents_of_layers(&touched_layer_ids) + .workspace()? + .dependent_glyph_ids_for_layers(&touched_layer_ids)? .into_iter() .map(|name| name.to_string()) .collect(); @@ -636,8 +643,8 @@ impl Bridge { layer_id: touched.layer.id().to_string(), structure: touched .structural - .then(|| GlyphStructure::from(&touched.layer).into()), - values: shift_wire::values_from_layer(&touched.layer).into(), + .then(|| GlyphStructure::from(touched.layer.as_ref()).into()), + values: shift_wire::values_from_layer(touched.layer.as_ref()).into(), changed: GlyphChangedEntities::default().into(), }) .collect(); @@ -714,14 +721,20 @@ impl Bridge { /// Glyph-addressed snapshots for renderer-local synchronous font state. #[napi] pub fn get_glyph_snapshots( - &self, + &mut self, requests: Vec, ) -> errors::Result> { - let font = self.font()?; + let requests = requests + .into_iter() + .map(GlyphSnapshotRequest::from) + .collect::>(); + let glyph_ids = requests + .iter() + .map(|request| request.glyph_id.clone()) + .collect::>(); + let font = self.acquire_and_font(&glyph_ids, AcquireScope::Glyphs)?; let mut snapshots = Vec::new(); - for request in requests { - let request = GlyphSnapshotRequest::from(request); let glyph_id = request.glyph_id; let Some(glyph) = font.glyph(glyph_id.clone()) else { continue; @@ -757,15 +770,16 @@ impl Bridge { /// renderer can evaluate design-location changes without further IPC. #[napi(ts_args_type = "glyphIds: Array")] pub fn get_glyph_projections( - &self, + &mut self, glyph_ids: Vec, ) -> errors::Result> { - let font = self.font()?; - let mut projections = Vec::new(); - let mut pending = glyph_ids + let glyph_ids = glyph_ids .iter() .map(|glyph_id| parse::(glyph_id)) - .collect::>>()?; + .collect::>>()?; + let font = self.acquire_and_font(&glyph_ids, AcquireScope::ComponentClosure)?; + let mut projections = Vec::new(); + let mut pending = glyph_ids.into_iter().collect::>(); let mut seen = HashSet::new(); while let Some(glyph_id) = pending.pop_front() { @@ -796,17 +810,16 @@ impl Bridge { /// `get_glyph_snapshots`. #[napi(ts_args_type = "glyphIds: Array, location: NapiLocation")] pub fn get_glyph_previews( - &self, + &mut self, glyph_ids: Vec, location: NapiLocation, ) -> errors::Result> { - let font = self.font()?; let glyph_ids = glyph_ids .iter() .map(|glyph_id| parse::(glyph_id)) .collect::>>()?; let location = map_location(location)?; - + let font = self.acquire_and_font(&glyph_ids, AcquireScope::ComponentClosure)?; let mut projection = font.projection(&location); let previews = projection .glyphs(&glyph_ids)? @@ -827,6 +840,7 @@ impl Bridge { /// geometry remains native until `stream_slug_atlas` emits bounded chunks. #[napi] pub fn prepare_slug_atlas(&mut self, alignment: u32) -> errors::Result { + self.workspace_mut()?.acquire_all_layers()?; let authored = build_authored_atlas(self.font()?, shift_slug::DEFAULT_BAND_COUNT)?; let layout = authored.atlas().layout(alignment as usize)?; self.slug_generation = self @@ -857,8 +871,8 @@ impl Bridge { .iter() .map(|glyph_id| parse::(glyph_id)) .collect::>>()?; - let authored = - build_authored_atlas_page(self.font()?, &glyph_ids, shift_slug::DEFAULT_BAND_COUNT)?; + let font = self.acquire_and_font(&glyph_ids, AcquireScope::ComponentClosure)?; + let authored = build_authored_atlas_page(font, &glyph_ids, shift_slug::DEFAULT_BAND_COUNT)?; let layout = authored.atlas().layout(alignment as usize)?; self.slug_generation = self .slug_generation @@ -1053,7 +1067,8 @@ impl Bridge { ) } - fn save_snapshot(&self) -> BridgeResult { + fn save_snapshot(&mut self) -> BridgeResult { + self.workspace_mut()?.acquire_all_layers()?; Ok(FontSaveSnapshot::new(self.font()?.clone(), None)) } @@ -1077,6 +1092,15 @@ impl Bridge { }) } + fn acquire_and_font( + &mut self, + glyph_ids: &[GlyphId], + scope: AcquireScope, + ) -> BridgeResult<&Font> { + self.workspace_mut()?.acquire_glyphs(glyph_ids, scope)?; + self.font() + } + fn font(&self) -> BridgeResult<&Font> { Ok(self.workspace()?.font()) } @@ -2025,7 +2049,7 @@ mod tests { assert!(result.is_err()); // atomicity: the valid point from the rejected set must NOT exist - let state = glyph_state(&bridge, "A"); + let state = glyph_state(&mut bridge, "A"); assert_eq!(state.structure.contours[0].points.len(), 1); } @@ -2159,7 +2183,7 @@ mod tests { None, ); assert!(result.is_err()); - let state = glyph_state(&bridge, "A"); + let state = glyph_state(&mut bridge, "A"); assert_eq!(state.structure.contours[0].points.len(), 1); assert_eq!(state.structure.anchors.len(), 1); @@ -2186,7 +2210,7 @@ mod tests { ) .unwrap(); assert_eq!(removed.layers.len(), 1); - let state = glyph_state(&bridge, "A"); + let state = glyph_state(&mut bridge, "A"); assert_eq!(state.structure.contours[0].points.len(), 0); assert!(state.structure.anchors.is_empty()); } @@ -2206,15 +2230,15 @@ mod tests { Some("Add Anchor".to_string()), ) .unwrap(); - assert_eq!(glyph_state(&bridge, "A").structure.anchors.len(), 1); + assert_eq!(glyph_state(&mut bridge, "A").structure.anchors.len(), 1); let undone = bridge.undo().unwrap().expect("one entry to undo"); assert_eq!(undone.layers.len(), 1); - assert!(glyph_state(&bridge, "A").structure.anchors.is_empty()); + assert!(glyph_state(&mut bridge, "A").structure.anchors.is_empty()); let redone = bridge.redo().unwrap().expect("one entry to redo"); assert_eq!(redone.layers.len(), 1); - let state = glyph_state(&bridge, "A"); + let state = glyph_state(&mut bridge, "A"); assert_eq!(state.structure.anchors.len(), 1); assert_eq!(state.structure.anchors[0].id, a1); assert_eq!(state.structure.anchors[0].name.as_deref(), Some("top")); @@ -2251,10 +2275,10 @@ mod tests { assert!(result.is_err()); // atomicity: the valid anchor from the rejected set must NOT exist - assert_eq!(glyph_state(&bridge, "A").structure.anchors.len(), 1); + assert_eq!(glyph_state(&mut bridge, "A").structure.anchors.len(), 1); } - fn glyph_state(bridge: &Bridge, name: &str) -> NapiGlyphState { + fn glyph_state(bridge: &mut Bridge, name: &str) -> NapiGlyphState { let record = bridge .get_glyphs() .unwrap() @@ -2267,7 +2291,7 @@ mod tests { } fn glyph_source_state( - bridge: &Bridge, + bridge: &mut Bridge, glyph_id: &str, source_id: &str, ) -> Option { @@ -2289,7 +2313,7 @@ mod tests { .map(|layer| layer.state) } - fn contour_point_count(bridge: &Bridge) -> usize { + fn contour_point_count(bridge: &mut Bridge) -> usize { glyph_state(bridge, "A") .structure .contours @@ -2315,15 +2339,58 @@ mod tests { Some("Add Point".to_string()), ) .unwrap(); - assert_eq!(contour_point_count(&bridge), 1); + assert_eq!(contour_point_count(&mut bridge), 1); let undone = bridge.undo().unwrap().expect("one entry to undo"); assert_eq!(undone.layers.len(), 1); - assert_eq!(contour_point_count(&bridge), 0); + assert!(undone.layers[0].structure.is_some()); + assert_eq!(contour_point_count(&mut bridge), 0); let redone = bridge.redo().unwrap().expect("one entry to redo"); assert_eq!(redone.layers.len(), 1); - assert_eq!(contour_point_count(&bridge), 1); + assert!(redone.layers[0].structure.is_some()); + assert_eq!(contour_point_count(&mut bridge), 1); + } + + #[test] + fn position_only_undo_redo_echoes_values_without_structure() { + let mut bridge = bridge_with_workspace(); + let (layer_id, contour_id) = pen_setup(&mut bridge); + let point_id = shift_font::PointId::new().to_string(); + bridge + .apply( + vec![add_points_intent( + &layer_id, + &contour_id, + None, + vec![seed(&point_id, 10.0, 20.0)], + )], + None, + ) + .unwrap(); + bridge + .apply( + vec![NapiFontIntent { + move_points: Some(NapiMovePointsIntent { + layer_id, + point_ids: vec![point_id], + coords: vec![30.0, 40.0], + }), + ..skeleton_intent("movePoints") + }], + None, + ) + .unwrap(); + + let undone = bridge.undo().unwrap().expect("position edit should undo"); + assert!(undone.layers[0].structure.is_none()); + assert_eq!(&undone.layers[0].values[1..3], &[10.0, 20.0]); + assert_eq!(&glyph_state(&mut bridge, "A").values[1..3], &[10.0, 20.0]); + + let redone = bridge.redo().unwrap().expect("position edit should redo"); + assert!(redone.layers[0].structure.is_none()); + assert_eq!(&redone.layers[0].values[1..3], &[30.0, 40.0]); + assert_eq!(&glyph_state(&mut bridge, "A").values[1..3], &[30.0, 40.0]); } #[test] @@ -2368,7 +2435,7 @@ mod tests { .unwrap(); assert!(bridge.redo().unwrap().is_none()); - assert_eq!(contour_point_count(&bridge), 1); + assert_eq!(contour_point_count(&mut bridge), 1); } #[test] @@ -2447,13 +2514,13 @@ mod tests { None, ) .unwrap(); - assert_eq!(contour_point_count(&bridge), 2); + assert_eq!(contour_point_count(&mut bridge), 2); bridge .undo() .unwrap() .expect("removePoints must be undoable"); - assert_eq!(contour_point_count(&bridge), 3); + assert_eq!(contour_point_count(&mut bridge), 3); } fn test_paths(label: &str) -> (String, String) { @@ -2560,6 +2627,28 @@ mod tests { ); } + #[test] + fn get_glyphs_reads_component_references_without_payload_acquisition() { + let source_path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../fixtures/fonts/mutatorsans-variable/MutatorSans.designspace"); + let (_, store_path) = test_paths("component-directory"); + let mut bridge = Bridge::new(); + bridge + .open_workspace(source_path.to_string_lossy().into_owned(), store_path) + .unwrap(); + + assert_eq!(bridge.workspace().unwrap().loaded_layer_count(), 0); + assert!( + bridge + .get_glyphs() + .unwrap() + .iter() + .any(|glyph| !glyph.component_base_glyph_ids.is_empty()), + "directory records should retain component edges without loading payloads" + ); + assert_eq!(bridge.workspace().unwrap().loaded_layer_count(), 0); + } + fn default_source_id(bridge: &Bridge) -> String { bridge.get_sources().unwrap()[0].id.clone() } @@ -2856,7 +2945,7 @@ mod tests { ) .unwrap(); - let state = glyph_source_state(&bridge, &glyph_id, "source_bold") + let state = glyph_source_state(&mut bridge, &glyph_id, "source_bold") .expect("the explicit layer must resolve by glyph and source"); assert_eq!(state.layer_id, layer_id); assert_eq!(applied.layers[0].layer_id, layer_id); @@ -2919,9 +3008,10 @@ mod tests { ) .unwrap(); - let source = glyph_source_state(&bridge, &glyph_id, &default_source_id(&bridge)) + let source_id = default_source_id(&bridge); + let source = glyph_source_state(&mut bridge, &glyph_id, &source_id) .expect("source layer should be readable"); - let cloned = glyph_source_state(&bridge, &glyph_id, "source_bold") + let cloned = glyph_source_state(&mut bridge, &glyph_id, "source_bold") .expect("cloned layer should be readable"); assert_eq!(applied.layers[0].layer_id, layer_id); @@ -3040,7 +3130,7 @@ mod tests { None, ) .unwrap(); - assert!(glyph_source_state(&bridge, &glyph_id, "source_bold").is_some()); + assert!(glyph_source_state(&mut bridge, &glyph_id, "source_bold").is_some()); let applied = bridge .apply(vec![delete_source_intent("source_bold")], None) @@ -3057,8 +3147,9 @@ mod tests { .expect("deleteSource layer removal must echo glyph records"); assert_eq!(glyphs[0].layers.len(), 1); assert!(applied.layers.is_empty()); - assert!(glyph_source_state(&bridge, &glyph_id, "source_bold").is_none()); - let default_state = glyph_source_state(&bridge, &glyph_id, &default_source_id(&bridge)) + assert!(glyph_source_state(&mut bridge, &glyph_id, "source_bold").is_none()); + let source_id = default_source_id(&bridge); + let default_state = glyph_source_state(&mut bridge, &glyph_id, &source_id) .expect("default source must keep its layer"); assert_eq!(default_state.layer_id, default_layer_id); } @@ -3246,7 +3337,7 @@ mod tests { ) .unwrap(); - let state = glyph_state(&bridge, "A"); + let state = glyph_state(&mut bridge, "A"); assert_eq!(state.layer_id, layer_id); assert_eq!(state.structure.contours.len(), 1); @@ -3256,7 +3347,7 @@ mod tests { #[test] fn get_glyph_snapshots_returns_none_for_missing_glyph() { - let bridge = bridge_with_workspace(); + let mut bridge = bridge_with_workspace(); let missing_glyph_id = shift_font::GlyphId::new().to_string(); let snapshots = bridge diff --git a/crates/shift-font/docs/DOCS.md b/crates/shift-font/docs/DOCS.md index 97f41e41..13cae11a 100644 --- a/crates/shift-font/docs/DOCS.md +++ b/crates/shift-font/docs/DOCS.md @@ -42,6 +42,8 @@ crates/shift-font/src/ - `SourceMetricInterpolation` owns metric identity, optional technical-field participation, variation regions, and delta ordering for source-owned metrics. - `Glyph` is a glyph concept identified by `GlyphId`. - `GlyphLayer` is authored editable data for one glyph at one source. +- `LibData` and recursive `LibValue::Dict` use ordered maps so equal domain values serialize deterministically. +- `GlyphEntityId` gives `FontIndex` one typed set for contour, point, component, anchor, and guideline identity. - `InterpolationBasis` is coordinate-independent variation math for an ordered source set. It contains normalized supports and source coefficient rows, never glyph coordinates or metrics. - `GlyphInterpolation` combines a reusable basis with one glyph's compatible authored source values. The glyph's default-source layer owns topology when present; otherwise a deterministic master-backed reference layer allows sparse glyph interpolation. - `LayerCompatibility` records every hard structural difference between an interpolation reference layer and another source layer. `LayerDifference` retains ordered path, node, anchor, and component evidence for diagnostics. @@ -68,7 +70,7 @@ Stable IDs are identity. Names and Unicode values are editable metadata. - Own font authoring data structures such as `Font`, `Glyph`, `GlyphLayer`, `Contour`, `Point`, `Source`, and `Axis`. - Keep object-level mutation behavior near the objects it mutates. -- Provide model-native helpers for layer editing, component resolution, variation behavior, axis mapping evaluation, and geometry-derived behavior. +- Provide model-native helpers for layer editing, component resolution, variation behavior, axis mapping evaluation, and geometry-derived behavior. `Font::replace_glyph_layers` retains previous layers through cheap `Arc` references, validates a complete hydration/eviction batch into one `HashSet`, moves that set into the final index, mutates uniquely owned fonts in place, and preserves shared snapshots through copy-on-write. - Own canonical glyph and source-metric interpolation value ordering, variation-model construction, interpolation evaluation, and location-bound glyph resolution. - Stay independent of TypeScript, NAPI, and bridge DTOs. @@ -90,7 +92,7 @@ Coordinates, advance width, smooth flags, anchor positions, and component transf `shift-wire` may translate native bases, source values, and projections into transport DTOs, but it must not rebuild source samples, define value ordering or topology compatibility, or evaluate variation models. -`shift-font` should not perform SQLite persistence. Durable working-store reads and writes belong in `shift-store`. +`shift-font` should not perform SQLite persistence or define a durable binary encoding. Durable working-store reads, writes, MessagePack encoding, compression, and compatibility policy belong in `shift-store`. `shift-font` should not own Electron, NAPI, or editor state. The TypeScript editor owns UI interaction, selection, hover, camera, tools, and command history. diff --git a/crates/shift-font/src/changes.rs b/crates/shift-font/src/changes.rs index 414751c3..8799cf18 100644 --- a/crates/shift-font/src/changes.rs +++ b/crates/shift-font/src/changes.rs @@ -134,12 +134,8 @@ impl FontChange { }) } - pub fn glyph_layer_created( - glyph_id: GlyphId, - name: Option, - layer: &GlyphLayer, - ) -> Self { - Self::GlyphLayerCreated(GlyphLayerCreated::from_layer(glyph_id, name, layer)) + pub fn glyph_layer_created(glyph_id: GlyphId, layer: &GlyphLayer) -> Self { + Self::GlyphLayerCreated(GlyphLayerCreated::from_layer(glyph_id, layer)) } pub fn glyph_layer_deleted(glyph_id: GlyphId, layer: &GlyphLayer) -> Self { @@ -203,6 +199,40 @@ impl FontChange { layer: GlyphLayerValue::from(layer), }) } + + /// Layer identity touched by a payload-affecting change. + /// + /// Workspace persistence uses this exhaustive match as a safety boundary: + /// an unloaded directory placeholder must never be written back over its + /// canonical payload. + pub fn layer_id(&self) -> Option<&LayerId> { + match self { + Self::GlyphLayerCreated(change) => Some(&change.layer_id), + Self::GlyphLayerDeleted(change) => Some(&change.layer_id), + Self::LayerMetricsChanged(change) => Some(&change.layer_id), + Self::ContourAdded(change) => Some(&change.layer_id), + Self::ContourOpenClosedChanged(change) => Some(&change.layer_id), + Self::PointsAdded(change) => Some(&change.layer_id), + Self::PointsDeleted(change) => Some(&change.layer_id), + Self::PointSmoothChanged(change) => Some(&change.layer_id), + Self::PointPositionsChanged(change) => Some(&change.layer_id), + Self::AnchorPositionsChanged(change) => Some(&change.layer_id), + Self::LayerGeometryReplaced(change) => Some(&change.layer_id), + Self::FontMetadataUpdated(_) + | Self::AxisCreated(_) + | Self::AxisUpdated(_) + | Self::AxisDeleted(_) + | Self::AxisMappingsUpdated(_) + | Self::MetricDefinitionsUpdated(_) + | Self::NamedInstancesUpdated(_) + | Self::SourceCreated(_) + | Self::SourceUpdated(_) + | Self::SourceDeleted(_) + | Self::GlyphCreated(_) + | Self::GlyphDeleted(_) + | Self::GlyphIdentityChanged(_) => None, + } + } } /// Complete authored metadata snapshot after one replacement. @@ -347,18 +377,16 @@ pub struct GlyphLayerCreated { pub glyph_id: GlyphId, pub source_id: SourceId, pub layer_id: LayerId, - pub name: Option, pub width: f64, pub height: Option, } impl GlyphLayerCreated { - pub fn from_layer(glyph_id: GlyphId, name: Option, layer: &GlyphLayer) -> Self { + pub fn from_layer(glyph_id: GlyphId, layer: &GlyphLayer) -> Self { Self { glyph_id, source_id: layer.source_id(), layer_id: layer.id(), - name, width: layer.width(), height: layer.height(), } diff --git a/crates/shift-font/src/error.rs b/crates/shift-font/src/error.rs index 9367ddd7..eabe4328 100644 --- a/crates/shift-font/src/error.rs +++ b/crates/shift-font/src/error.rs @@ -79,6 +79,13 @@ pub enum CoreError { #[error("layer {0} not found")] LayerNotFound(LayerId), + #[error("layer {layer_id} source is {actual_source_id}, expected {expected_source_id}")] + LayerSourceMismatch { + layer_id: LayerId, + expected_source_id: SourceId, + actual_source_id: SourceId, + }, + #[error("layer {layer_id} belongs to glyph {actual_glyph_id}, not glyph {glyph_id}")] LayerGlyphMismatch { layer_id: LayerId, diff --git a/crates/shift-font/src/intents.rs b/crates/shift-font/src/intents.rs index e3db1a30..e2184570 100644 --- a/crates/shift-font/src/intents.rs +++ b/crates/shift-font/src/intents.rs @@ -17,6 +17,7 @@ use crate::ir::{ use crate::layer_edit::BulkNodePositionUpdates; use crate::source::source_locations_equal; use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; /// A point to create, with stable identity minted by a trusted caller. #[derive(Clone, Debug)] @@ -241,6 +242,56 @@ impl FontIntent { } } + /// Complete authored-layer read set required before applying this intent. + /// + /// Keeping this exhaustive match beside the intent vocabulary prevents a + /// new operation from accidentally mutating a directory placeholder. The + /// workspace still guards the persistence boundary if a dependency is + /// ever omitted here. + pub fn required_layer_ids(&self, font: &Font) -> Vec { + match self { + Self::AddPoints { layer_id, .. } + | Self::AddContour { layer_id, .. } + | Self::SetContourClosed { layer_id, .. } + | Self::MovePoints { layer_id, .. } + | Self::SetPointSmooth { layer_id, .. } + | Self::RemovePoints { layer_id, .. } + | Self::AddAnchors { layer_id, .. } + | Self::MoveAnchors { layer_id, .. } + | Self::RemoveAnchors { layer_id, .. } + | Self::ReverseContour { layer_id, .. } + | Self::TranslatePoints { layer_id, .. } + | Self::SetXAdvance { layer_id, .. } + | Self::ApplyBooleanOp { layer_id, .. } => vec![layer_id.clone()], + Self::CloneGlyphLayer { from_layer_id, .. } + | Self::MaterializeGlyphLayer { from_layer_id, .. } => { + vec![from_layer_id.clone()] + } + Self::DeleteSource { source_id } => font + .glyphs() + .filter_map(|glyph| { + glyph + .layer_for_source(source_id.clone()) + .map(GlyphLayer::id) + }) + .collect(), + Self::CreateGlyph { .. } + | Self::UpdateGlyph { .. } + | Self::UpdateFontMetadata { .. } + | Self::CreateAxis { .. } + | Self::UpdateAxis { .. } + | Self::DeleteAxis { .. } + | Self::SetAxisMappings { .. } + | Self::SetMetricDefinitions { .. } + | Self::CreateNamedInstance { .. } + | Self::UpdateNamedInstance { .. } + | Self::DeleteNamedInstance { .. } + | Self::CreateSource { .. } + | Self::UpdateSource { .. } + | Self::CreateGlyphLayer { .. } => Vec::new(), + } + } + /// Whether applying this intent changes layer structure (vs values only). /// Smooth flags live in structure, so they count. fn structural(&self) -> bool { @@ -261,7 +312,7 @@ pub struct FontIntentSet { /// One touched layer after an intent set applied. pub struct TouchedLayer { - pub layer: GlyphLayer, + pub layer: Arc, pub structural: bool, } @@ -316,9 +367,11 @@ impl Font { .into_iter() .map(|(layer_id, structural)| { let layer = self - .layer(layer_id.clone()) - .ok_or(CoreError::LayerNotFound(layer_id))? - .clone(); + .glyph_id_by_layer(layer_id.clone()) + .and_then(|glyph_id| self.glyph(glyph_id)) + .and_then(|glyph| glyph.layers().get(&layer_id)) + .cloned() + .ok_or(CoreError::LayerNotFound(layer_id))?; Ok(TouchedLayer { layer, structural }) }) .collect::>>()?; @@ -652,11 +705,6 @@ impl Font { source_id: SourceId, changes: &mut FontChangeSet, ) -> CoreResult> { - let glyph_name = self - .glyph(glyph_id.clone()) - .ok_or_else(|| CoreError::GlyphNotFound(glyph_id.clone()))? - .glyph_name() - .clone(); if self.glyph_id_by_layer(layer_id.clone()).is_some() { return Err(CoreError::DuplicateLayerId(layer_id)); } @@ -672,11 +720,7 @@ impl Font { let layer = GlyphLayer::with_width(layer_id.clone(), source_id, self.default_layer_width()); self.insert_glyph_layer(glyph_id.clone(), layer.clone())?; - changes.push(FontChange::glyph_layer_created( - glyph_id, - Some(glyph_name), - &layer, - )); + changes.push(FontChange::glyph_layer_created(glyph_id, &layer)); Ok(vec![layer_id]) } @@ -689,15 +733,11 @@ impl Font { from_layer_id: LayerId, changes: &mut FontChangeSet, ) -> CoreResult> { - let (glyph_name, layer) = + let layer = self.cloned_glyph_layer(layer_id.clone(), glyph_id.clone(), source_id, from_layer_id)?; self.insert_glyph_layer(glyph_id.clone(), layer.clone())?; - changes.push(FontChange::glyph_layer_created( - glyph_id, - Some(glyph_name), - &layer, - )); + changes.push(FontChange::glyph_layer_created(glyph_id, &layer)); Ok(vec![layer_id]) } @@ -711,16 +751,12 @@ impl Font { values: &GlyphInterpolationValues, changes: &mut FontChangeSet, ) -> CoreResult> { - let (glyph_name, mut layer) = + let mut layer = self.cloned_glyph_layer(layer_id.clone(), glyph_id.clone(), source_id, from_layer_id)?; layer.apply_interpolation_values(values)?; self.insert_glyph_layer(glyph_id.clone(), layer.clone())?; - changes.push(FontChange::glyph_layer_created( - glyph_id, - Some(glyph_name), - &layer, - )); + changes.push(FontChange::glyph_layer_created(glyph_id, &layer)); Ok(vec![layer_id]) } @@ -731,13 +767,7 @@ impl Font { glyph_id: GlyphId, source_id: SourceId, from_layer_id: LayerId, - ) -> CoreResult<(GlyphName, GlyphLayer)> { - let glyph_name = self - .glyph(glyph_id.clone()) - .ok_or_else(|| CoreError::GlyphNotFound(glyph_id.clone()))? - .glyph_name() - .clone(); - + ) -> CoreResult { if self.glyph_id_by_layer(layer_id.clone()).is_some() { return Err(CoreError::DuplicateLayerId(layer_id)); } @@ -767,7 +797,7 @@ impl Font { .ok_or(CoreError::LayerNotFound(from_layer_id))?; let layer = source_layer.clone_with_fresh_ids(layer_id, source_id); - Ok((glyph_name, layer)) + Ok(layer) } fn apply_update_glyph( @@ -1125,40 +1155,6 @@ impl Font { } } - /// Ids of glyphs whose components reference the glyphs owning the - /// given layers — the composites a renderer must re-render after an - /// edit. Stable ids (not names): references survive renames. Sorted, - /// deduplicated, excludes the touched glyphs themselves. - pub fn dependents_of_layers(&self, layer_ids: &[LayerId]) -> Vec { - let touched: Vec = self - .glyphs() - .filter(|glyph| { - glyph - .layers() - .keys() - .any(|layer_id| layer_ids.contains(layer_id)) - }) - .map(|glyph| glyph.id()) - .collect(); - - let mut dependents: Vec = self - .glyphs() - .filter(|glyph| !touched.contains(&glyph.id())) - .filter(|glyph| { - glyph.layers().values().any(|layer| { - layer - .components_iter() - .any(|component| touched.contains(&component.base_glyph_id())) - }) - }) - .map(|glyph| glyph.id()) - .collect(); - - dependents.sort(); - dependents.dedup(); - dependents - } - fn layer_mut_or_err(&mut self, layer_id: &LayerId) -> CoreResult<&mut GlyphLayer> { self.layer_mut(layer_id.clone()) .ok_or(CoreError::LayerNotFound(layer_id.clone())) diff --git a/crates/shift-font/src/ir/entity.rs b/crates/shift-font/src/ir/entity.rs index 9998e30e..21766ffb 100644 --- a/crates/shift-font/src/ir/entity.rs +++ b/crates/shift-font/src/ir/entity.rs @@ -1,4 +1,5 @@ -use serde::{Deserialize, Serialize}; +use serde::de::Error as _; +use serde::{Deserialize, Deserializer, Serialize}; const SHORT_ID_ALPHABET: &[u8; 64] = b"useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict"; @@ -36,7 +37,7 @@ impl std::fmt::Display for EntityId { macro_rules! typed_id { ($name:ident, $prefix:literal) => { - #[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)] + #[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize)] pub struct $name(String); impl Default for $name { @@ -84,6 +85,16 @@ macro_rules! typed_id { } } } + + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + value.parse().map_err(D::Error::custom) + } + } }; } @@ -133,6 +144,45 @@ typed_id!(AxisMappingId, "axisMapping"); typed_id!(NamedInstanceId, "namedInstance"); typed_id!(MetricId, "metric"); +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub enum GlyphEntityId { + Contour(ContourId), + Point(PointId), + Component(ComponentId), + Anchor(AnchorId), + Guideline(GuidelineId), +} + +impl From for GlyphEntityId { + fn from(id: ContourId) -> Self { + Self::Contour(id) + } +} + +impl From for GlyphEntityId { + fn from(id: PointId) -> Self { + Self::Point(id) + } +} + +impl From for GlyphEntityId { + fn from(id: ComponentId) -> Self { + Self::Component(id) + } +} + +impl From for GlyphEntityId { + fn from(id: AnchorId) -> Self { + Self::Anchor(id) + } +} + +impl From for GlyphEntityId { + fn from(id: GuidelineId) -> Self { + Self::Guideline(id) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/shift-font/src/ir/font.rs b/crates/shift-font/src/ir/font.rs index 9c750b04..135b09e0 100644 --- a/crates/shift-font/src/ir/font.rs +++ b/crates/shift-font/src/ir/font.rs @@ -2,13 +2,13 @@ use crate::axis::{Axis, AxisMapping, Location}; use crate::binary_data::BinaryData; use crate::collection::EntityList; use crate::entity::{ - AnchorId, AxisId, ComponentId, ContourId, GlyphId, GuidelineId, LayerId, MetricId, PointId, - SourceId, + AnchorId, AxisId, ContourId, GlyphEntityId, GlyphId, LayerId, MetricId, PointId, SourceId, }; use crate::error::{CoreError, CoreResult}; use crate::features::FeatureData; use crate::glyph::{Glyph, GlyphLayer}; use crate::guideline::Guideline; +use crate::interpolation::GlyphInterpolationValues; use crate::kerning::KerningData; use crate::lib_data::LibData; use crate::metrics::{FontMetrics, MetricDefinition, MetricKind, MetricValue}; @@ -126,11 +126,7 @@ struct FontIndex { layer_owner: HashMap, layer_by_glyph_source: HashMap<(GlyphId, SourceId), LayerId>, glyphs_by_unicode: HashMap>, - contour_ids: HashSet, - point_ids: HashSet, - component_ids: HashSet, - anchor_ids: HashSet, - guideline_ids: HashSet, + entity_ids: HashSet, } impl FontIndex { @@ -158,11 +154,7 @@ impl FontIndex { } let mut local_sources = HashSet::new(); - let mut local_contours = HashSet::new(); - let mut local_points = HashSet::new(); - let mut local_components = HashSet::new(); - let mut local_anchors = HashSet::new(); - let mut local_guidelines = HashSet::new(); + let mut local_entities = HashSet::new(); for layer in glyph.layers().values().map(Arc::as_ref) { if self.layer_owner.contains_key(&layer.id()) { @@ -180,38 +172,10 @@ impl FontIndex { }); } - for contour in layer.contours_iter() { - if self.contour_ids.contains(&contour.id()) || !local_contours.insert(contour.id()) + for entity_id in glyph_entity_ids(layer) { + if self.entity_ids.contains(&entity_id) || !local_entities.insert(entity_id.clone()) { - return Err(CoreError::DuplicateContourId(contour.id())); - } - - for point in contour.points() { - if self.point_ids.contains(&point.id()) || !local_points.insert(point.id()) { - return Err(CoreError::DuplicatePointId(point.id())); - } - } - } - - for component in layer.components_iter() { - if self.component_ids.contains(&component.id()) - || !local_components.insert(component.id()) - { - return Err(CoreError::DuplicateComponentId(component.id())); - } - } - - for anchor in layer.anchors_iter() { - if self.anchor_ids.contains(&anchor.id()) || !local_anchors.insert(anchor.id()) { - return Err(CoreError::DuplicateAnchorId(anchor.id())); - } - } - - for guideline in layer.guidelines() { - if self.guideline_ids.contains(&guideline.id()) - || !local_guidelines.insert(guideline.id()) - { - return Err(CoreError::DuplicateGuidelineId(guideline.id())); + return Err(duplicate_entity_error(entity_id)); } } } @@ -234,45 +198,37 @@ impl FontIndex { }); } - let mut contour_ids = HashSet::new(); - let mut point_ids = HashSet::new(); - let mut component_ids = HashSet::new(); - let mut anchor_ids = HashSet::new(); - let mut guideline_ids = HashSet::new(); - - for contour in layer.contours_iter() { - if self.contour_ids.contains(&contour.id()) || !contour_ids.insert(contour.id()) { - return Err(CoreError::DuplicateContourId(contour.id())); - } - - for point in contour.points() { - if self.point_ids.contains(&point.id()) || !point_ids.insert(point.id()) { - return Err(CoreError::DuplicatePointId(point.id())); - } + let mut local_entities = HashSet::new(); + for entity_id in glyph_entity_ids(layer) { + if self.entity_ids.contains(&entity_id) || !local_entities.insert(entity_id.clone()) { + return Err(duplicate_entity_error(entity_id)); } } - for component in layer.components_iter() { - if self.component_ids.contains(&component.id()) || !component_ids.insert(component.id()) - { - return Err(CoreError::DuplicateComponentId(component.id())); - } - } + Ok(()) + } - for anchor in layer.anchors_iter() { - if self.anchor_ids.contains(&anchor.id()) || !anchor_ids.insert(anchor.id()) { - return Err(CoreError::DuplicateAnchorId(anchor.id())); - } - } + fn validate_layer_replacements( + &self, + replacements: &[(GlyphId, Arc, Arc)], + ) -> CoreResult> { + let removed_ids = replacements + .iter() + .flat_map(|(_, previous, _)| glyph_entity_ids(previous)) + .collect::>(); + let mut replacement_ids = HashSet::new(); - for guideline in layer.guidelines() { - if self.guideline_ids.contains(&guideline.id()) || !guideline_ids.insert(guideline.id()) - { - return Err(CoreError::DuplicateGuidelineId(guideline.id())); + for (_, _, replacement) in replacements { + for entity_id in glyph_entity_ids(replacement) { + if (self.entity_ids.contains(&entity_id) && !removed_ids.contains(&entity_id)) + || !replacement_ids.insert(entity_id.clone()) + { + return Err(duplicate_entity_error(entity_id)); + } } } - Ok(()) + Ok(replacement_ids) } fn insert_layer(&mut self, glyph_id: GlyphId, layer: &GlyphLayer) { @@ -280,17 +236,7 @@ impl FontIndex { self.layer_by_glyph_source .insert((glyph_id, layer.source_id()), layer.id()); - for contour in layer.contours_iter() { - self.contour_ids.insert(contour.id()); - self.point_ids - .extend(contour.points().iter().map(|point| point.id())); - } - self.component_ids - .extend(layer.components_iter().map(|component| component.id())); - self.anchor_ids - .extend(layer.anchors_iter().map(|anchor| anchor.id())); - self.guideline_ids - .extend(layer.guidelines().iter().map(Guideline::id)); + self.entity_ids.extend(glyph_entity_ids(layer)); } fn remove_layer(&mut self, glyph_id: GlyphId, layer: &GlyphLayer) { @@ -298,20 +244,8 @@ impl FontIndex { self.layer_by_glyph_source .remove(&(glyph_id, layer.source_id())); - for contour in layer.contours_iter() { - self.contour_ids.remove(&contour.id()); - for point in contour.points() { - self.point_ids.remove(&point.id()); - } - } - for component in layer.components_iter() { - self.component_ids.remove(&component.id()); - } - for anchor in layer.anchors_iter() { - self.anchor_ids.remove(&anchor.id()); - } - for guideline in layer.guidelines() { - self.guideline_ids.remove(&guideline.id()); + for entity_id in glyph_entity_ids(layer) { + self.entity_ids.remove(&entity_id); } } @@ -349,6 +283,44 @@ impl FontIndex { } } +fn glyph_entity_ids(layer: &GlyphLayer) -> impl Iterator + '_ { + layer + .contours_iter() + .map(|contour| GlyphEntityId::from(contour.id())) + .chain( + layer + .contours_iter() + .flat_map(|contour| contour.points().iter()) + .map(|point| GlyphEntityId::from(point.id())), + ) + .chain( + layer + .components_iter() + .map(|component| GlyphEntityId::from(component.id())), + ) + .chain( + layer + .anchors_iter() + .map(|anchor| GlyphEntityId::from(anchor.id())), + ) + .chain( + layer + .guidelines() + .iter() + .map(|guideline| GlyphEntityId::from(guideline.id())), + ) +} + +fn duplicate_entity_error(id: GlyphEntityId) -> CoreError { + match id { + GlyphEntityId::Contour(id) => CoreError::DuplicateContourId(id), + GlyphEntityId::Point(id) => CoreError::DuplicatePointId(id), + GlyphEntityId::Component(id) => CoreError::DuplicateComponentId(id), + GlyphEntityId::Anchor(id) => CoreError::DuplicateAnchorId(id), + GlyphEntityId::Guideline(id) => CoreError::DuplicateGuidelineId(id), + } +} + impl FontState { fn from_data(data: FontData) -> CoreResult { let index = FontIndex::from_glyphs(&data.glyphs)?; @@ -958,45 +930,66 @@ impl Font { /// Returns whether a contour identity is already in use anywhere in the font. pub(crate) fn has_contour_id(&self, contour_id: &ContourId) -> bool { - self.index().contour_ids.contains(contour_id) + self.index() + .entity_ids + .contains(&GlyphEntityId::from(contour_id.clone())) } /// Returns whether a point identity is already in use anywhere in the font. pub(crate) fn has_point_id(&self, point_id: &PointId) -> bool { - self.index().point_ids.contains(point_id) + self.index() + .entity_ids + .contains(&GlyphEntityId::from(point_id.clone())) } /// Returns whether an anchor identity is already in use anywhere in the font. pub(crate) fn has_anchor_id(&self, anchor_id: &AnchorId) -> bool { - self.index().anchor_ids.contains(anchor_id) + self.index() + .entity_ids + .contains(&GlyphEntityId::from(anchor_id.clone())) } /// Records a contour minted by an in-place layer edit. pub(crate) fn record_contour_id(&mut self, contour_id: ContourId) { - self.state_mut().index.contour_ids.insert(contour_id); + self.state_mut() + .index + .entity_ids + .insert(GlyphEntityId::from(contour_id)); } /// Records points minted by an in-place layer edit. pub(crate) fn record_point_ids(&mut self, point_ids: impl IntoIterator) { - self.state_mut().index.point_ids.extend(point_ids); + self.state_mut() + .index + .entity_ids + .extend(point_ids.into_iter().map(GlyphEntityId::from)); } /// Records anchors minted by an in-place layer edit. pub(crate) fn record_anchor_ids(&mut self, anchor_ids: impl IntoIterator) { - self.state_mut().index.anchor_ids.extend(anchor_ids); + self.state_mut() + .index + .entity_ids + .extend(anchor_ids.into_iter().map(GlyphEntityId::from)); } /// Forgets point identities removed by an in-place layer edit. pub(crate) fn forget_point_ids(&mut self, point_ids: &[PointId]) { for point_id in point_ids { - self.state_mut().index.point_ids.remove(point_id); + self.state_mut() + .index + .entity_ids + .remove(&GlyphEntityId::from(point_id.clone())); } } /// Forgets anchor identities removed by an in-place layer edit. pub(crate) fn forget_anchor_ids(&mut self, anchor_ids: &[AnchorId]) { for anchor_id in anchor_ids { - self.state_mut().index.anchor_ids.remove(anchor_id); + self.state_mut() + .index + .entity_ids + .remove(&GlyphEntityId::from(anchor_id.clone())); } } @@ -1106,6 +1099,94 @@ impl Font { Ok(()) } + /// Replaces one layer's canonical numeric values without changing its structure or indexes. + /// + /// Value count and finiteness are validated before the layer changes. + pub fn replace_glyph_layer_values( + &mut self, + layer_id: LayerId, + values: &GlyphInterpolationValues, + ) -> CoreResult<()> { + let glyph_id = self + .glyph_id_by_layer(layer_id.clone()) + .ok_or(CoreError::LayerNotFound(layer_id.clone()))?; + let state = self.state_mut(); + let glyph = state + .data + .glyphs + .get_mut(&glyph_id) + .expect("layer owner was resolved before mutation"); + let layer = Arc::make_mut(glyph) + .layer_mut(layer_id.clone()) + .ok_or(CoreError::LayerNotFound(layer_id))?; + + layer.apply_interpolation_values(values) + } + + /// Atomically replaces existing layers after validating the complete batch. + /// Uniquely owned fonts mutate in place; shared snapshots retain copy-on-write semantics. + /// Already shared layer snapshots transfer without cloning their geometry. + pub fn replace_glyph_layers(&mut self, layers: Vec) -> CoreResult<()> + where + L: Into>, + { + if layers.is_empty() { + return Ok(()); + } + + let mut seen_layer_ids = HashSet::with_capacity(layers.len()); + let mut replacements = Vec::with_capacity(layers.len()); + for layer in layers { + let layer = layer.into(); + let layer_id = layer.id(); + if !seen_layer_ids.insert(layer_id.clone()) { + return Err(CoreError::DuplicateLayerId(layer_id)); + } + + let glyph_id = self + .glyph_id_by_layer(layer_id.clone()) + .ok_or(CoreError::LayerNotFound(layer_id.clone()))?; + let previous = self + .data() + .glyphs + .get(&glyph_id) + .and_then(|glyph| glyph.layers().get(&layer_id)) + .cloned() + .ok_or(CoreError::LayerNotFound(layer_id.clone()))?; + if layer.source_id() != previous.source_id() { + return Err(CoreError::LayerSourceMismatch { + layer_id, + expected_source_id: previous.source_id(), + actual_source_id: layer.source_id(), + }); + } + replacements.push((glyph_id, previous, layer)); + } + + let replacement_ids = self.index().validate_layer_replacements(&replacements)?; + + let state = self.state_mut(); + for (glyph_id, previous, _) in &replacements { + state.index.remove_layer(glyph_id.clone(), previous); + } + for (glyph_id, _, layer) in replacements { + state.index.layer_owner.insert(layer.id(), glyph_id.clone()); + state + .index + .layer_by_glyph_source + .insert((glyph_id.clone(), layer.source_id()), layer.id()); + let glyph = state + .data + .glyphs + .get_mut(&glyph_id) + .expect("replacement owner was resolved before mutation"); + Arc::make_mut(glyph).set_layer(layer); + } + state.index.entity_ids.extend(replacement_ids); + + Ok(()) + } + pub fn remove_glyph_layer(&mut self, layer_id: LayerId) -> CoreResult { let glyph_id = self .glyph_id_by_layer(layer_id.clone()) @@ -1364,8 +1445,8 @@ fn validate_axis_mappings(axes: &[Axis], mappings: &[AxisMapping]) -> CoreResult mod tests { use super::*; use crate::{ - test_support::sample_font, AxisMappingPoint, AxisRole, Contour, ContourId, GlyphLayer, - LayerId, PointId, PointType, + test_support::sample_font, Anchor, AxisMappingPoint, AxisRole, Component, ComponentId, + Contour, ContourId, GlyphLayer, GuidelineId, LayerId, PointId, PointType, }; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -1663,6 +1744,269 @@ mod tests { ); } + #[test] + fn glyph_layer_batch_replacement_is_atomic() { + let mut font = Font::new(); + let source_id = font.default_source_id().unwrap(); + let first_layer = GlyphLayer::with_width(LayerId::new(), source_id.clone(), 500.0); + let first_layer_id = first_layer.id(); + let mut first_glyph = Glyph::new("A"); + first_glyph.set_layer(first_layer); + font.insert_glyph(first_glyph).unwrap(); + + let second_layer = GlyphLayer::with_width(LayerId::new(), source_id.clone(), 600.0); + let second_layer_id = second_layer.id(); + let mut second_glyph = Glyph::new("B"); + second_glyph.set_layer(second_layer); + font.insert_glyph(second_glyph).unwrap(); + + let contour_id = ContourId::new(); + let mut first_contour = Contour::with_id(contour_id.clone()); + first_contour.add_point(0.0, 0.0, PointType::OnCurve, false); + let mut first_replacement = + GlyphLayer::with_width(first_layer_id.clone(), source_id.clone(), 700.0); + first_replacement.add_contour(first_contour); + + let mut second_contour = Contour::with_id(contour_id.clone()); + second_contour.add_point(10.0, 10.0, PointType::OnCurve, false); + let mut second_replacement = + GlyphLayer::with_width(second_layer_id.clone(), source_id.clone(), 800.0); + second_replacement.add_contour(second_contour); + + assert!(matches!( + font.replace_glyph_layers(vec![first_replacement, second_replacement]), + Err(CoreError::DuplicateContourId(id)) if id == contour_id + )); + assert_eq!(font.layer(first_layer_id.clone()).unwrap().width(), 500.0); + assert_eq!(font.layer(second_layer_id.clone()).unwrap().width(), 600.0); + + font.replace_glyph_layers(vec![ + GlyphLayer::with_width(first_layer_id.clone(), source_id.clone(), 700.0), + GlyphLayer::with_width(second_layer_id.clone(), source_id, 800.0), + ]) + .unwrap(); + assert_eq!(font.layer(first_layer_id).unwrap().width(), 700.0); + assert_eq!(font.layer(second_layer_id).unwrap().width(), 800.0); + } + + #[test] + fn glyph_layer_value_replacement_preserves_structure_indexes_and_snapshots() { + let mut font = Font::new(); + let source_id = font.default_source_id().unwrap(); + let contour_id = ContourId::new(); + let point_id = PointId::new(); + let anchor_id = AnchorId::new(); + let component_id = ComponentId::new(); + let mut contour = Contour::with_id(contour_id.clone()); + contour.add_point_with_id(point_id.clone(), 10.0, 20.0, PointType::OnCurve, false); + let mut layer = GlyphLayer::with_width(LayerId::new(), source_id, 500.0); + layer.add_contour(contour); + layer.add_anchor(Anchor::with_id( + anchor_id.clone(), + Some("top".to_string()), + 30.0, + 40.0, + )); + layer.add_component(Component::with_id( + component_id.clone(), + GlyphId::new(), + "base", + crate::DecomposedTransform::default(), + )); + let layer_id = layer.id(); + let mut glyph = Glyph::new("A"); + glyph.set_layer(layer); + font.insert_glyph(glyph).unwrap(); + + let snapshot = font.clone(); + let mut expected = font.layer(layer_id.clone()).unwrap().clone(); + expected.set_width(700.0); + expected.contours_iter_mut().next().unwrap().points_mut()[0].set_position(50.0, 60.0); + expected + .anchors_iter_mut() + .next() + .unwrap() + .set_position(70.0, 80.0); + expected + .components_iter_mut() + .next() + .unwrap() + .translate(90.0, 100.0); + let values = expected.interpolation_values(); + + font.replace_glyph_layer_values(layer_id.clone(), &values) + .unwrap(); + + assert_eq!(font.layer(layer_id.clone()).unwrap(), &expected); + assert_ne!(snapshot.layer(layer_id.clone()).unwrap(), &expected); + assert!(font.has_contour_id(&contour_id)); + assert!(font.has_point_id(&point_id)); + assert!(font.has_anchor_id(&anchor_id)); + assert!(font + .index() + .entity_ids + .contains(&GlyphEntityId::from(component_id))); + + let committed = font.layer(layer_id.clone()).unwrap().clone(); + let mut invalid = values.into_vec(); + invalid[0] = f64::NAN; + assert!(matches!( + font.replace_glyph_layer_values( + layer_id.clone(), + &GlyphInterpolationValues::new(invalid) + ), + Err(CoreError::InvalidPositionUpdateInput { .. }) + )); + assert_eq!(font.layer(layer_id).unwrap(), &committed); + } + + #[test] + fn glyph_layer_batch_replacement_preserves_snapshots_and_identity_indexes() { + let mut font = Font::new(); + let source_id = font.default_source_id().unwrap(); + let contour_id = ContourId::new(); + let point_id = PointId::new(); + let component_id = ComponentId::new(); + let anchor_id = AnchorId::new(); + let guideline_id = GuidelineId::new(); + let base_glyph_id = GlyphId::new(); + let mut contour = Contour::with_id(contour_id.clone()); + contour.add_point_with_id(point_id.clone(), 0.0, 0.0, PointType::OnCurve, false); + let mut first_layer = GlyphLayer::with_width(LayerId::new(), source_id.clone(), 500.0); + first_layer.add_contour(contour); + first_layer.add_component(Component::with_id( + component_id.clone(), + base_glyph_id.clone(), + "base", + crate::DecomposedTransform::default(), + )); + first_layer.add_anchor(Anchor::with_id( + anchor_id.clone(), + Some("top".to_string()), + 0.0, + 100.0, + )); + first_layer.add_guideline(Guideline::with_id( + guideline_id.clone(), + None, + Some(0.0), + None, + None, + None, + )); + let first_layer_id = first_layer.id(); + let mut first_glyph = Glyph::new("A"); + first_glyph.set_layer(first_layer); + font.insert_glyph(first_glyph).unwrap(); + + let second_layer = GlyphLayer::with_width(LayerId::new(), source_id.clone(), 600.0); + let second_layer_id = second_layer.id(); + let mut second_glyph = Glyph::new("B"); + second_glyph.set_layer(second_layer); + font.insert_glyph(second_glyph).unwrap(); + + let snapshot = font.clone(); + let mut replacement = font.layer(first_layer_id.clone()).unwrap().clone(); + replacement.set_width(700.0); + font.replace_glyph_layers(vec![replacement]).unwrap(); + + assert_eq!(font.layer(first_layer_id.clone()).unwrap().width(), 700.0); + assert_eq!( + snapshot.layer(first_layer_id.clone()).unwrap().width(), + 500.0 + ); + assert!(font.has_contour_id(&contour_id)); + assert!(font.has_point_id(&point_id)); + assert!(font.has_anchor_id(&anchor_id)); + + let mut conflicting_contour = Contour::new(); + conflicting_contour.add_point_with_id( + point_id.clone(), + 10.0, + 10.0, + PointType::OnCurve, + false, + ); + let mut conflicting_layer = + GlyphLayer::with_width(second_layer_id.clone(), source_id.clone(), 800.0); + conflicting_layer.add_contour(conflicting_contour); + assert!(matches!( + font.replace_glyph_layers(vec![conflicting_layer]), + Err(CoreError::DuplicatePointId(id)) if id == point_id + )); + + let mut conflicting_layer = + GlyphLayer::with_width(second_layer_id.clone(), source_id.clone(), 800.0); + conflicting_layer.add_component(Component::with_id( + component_id.clone(), + base_glyph_id.clone(), + "base", + crate::DecomposedTransform::default(), + )); + assert!(matches!( + font.replace_glyph_layers(vec![conflicting_layer]), + Err(CoreError::DuplicateComponentId(id)) if id == component_id + )); + + let mut conflicting_layer = + GlyphLayer::with_width(second_layer_id.clone(), source_id.clone(), 800.0); + conflicting_layer.add_anchor(Anchor::with_id(anchor_id.clone(), None, 10.0, 10.0)); + assert!(matches!( + font.replace_glyph_layers(vec![conflicting_layer]), + Err(CoreError::DuplicateAnchorId(id)) if id == anchor_id + )); + + let mut conflicting_layer = + GlyphLayer::with_width(second_layer_id.clone(), source_id.clone(), 800.0); + conflicting_layer.add_guideline(Guideline::with_id( + guideline_id.clone(), + Some(10.0), + None, + None, + None, + None, + )); + assert!(matches!( + font.replace_glyph_layers(vec![conflicting_layer]), + Err(CoreError::DuplicateGuidelineId(id)) if id == guideline_id + )); + assert_eq!(font.layer(second_layer_id.clone()).unwrap().width(), 600.0); + + let emptied = GlyphLayer::with_width(first_layer_id, source_id.clone(), 700.0); + let mut transferred_contour = Contour::with_id(contour_id.clone()); + transferred_contour.add_point_with_id( + point_id.clone(), + 20.0, + 20.0, + PointType::OnCurve, + false, + ); + let mut transferred = GlyphLayer::with_width(second_layer_id.clone(), source_id, 800.0); + transferred.add_contour(transferred_contour); + transferred.add_component(Component::with_id( + component_id, + base_glyph_id, + "base", + crate::DecomposedTransform::default(), + )); + transferred.add_anchor(Anchor::with_id(anchor_id.clone(), None, 20.0, 20.0)); + transferred.add_guideline(Guideline::with_id( + guideline_id, + None, + Some(20.0), + None, + None, + None, + )); + font.replace_glyph_layers(vec![emptied, transferred]) + .unwrap(); + + assert_eq!(font.layer(second_layer_id).unwrap().width(), 800.0); + assert!(font.has_contour_id(&contour_id)); + assert!(font.has_point_id(&point_id)); + assert!(font.has_anchor_id(&anchor_id)); + } + #[test] fn font_remove_insert_glyph() { let mut font = Font::new(); diff --git a/crates/shift-font/src/ir/glyph.rs b/crates/shift-font/src/ir/glyph.rs index 571ad873..eaec54ac 100644 --- a/crates/shift-font/src/ir/glyph.rs +++ b/crates/shift-font/src/ir/glyph.rs @@ -386,8 +386,12 @@ impl Glyph { self.layer_mut(layer_id).expect("layer was just inserted") } - pub fn set_layer(&mut self, layer: GlyphLayer) { - self.layers.insert(layer.id(), Arc::new(layer)); + pub fn set_layer(&mut self, layer: L) + where + L: Into>, + { + let layer = layer.into(); + self.layers.insert(layer.id(), layer); } pub fn layer_for_source(&self, source_id: SourceId) -> Option<&GlyphLayer> { diff --git a/crates/shift-font/src/ir/glyph_name.rs b/crates/shift-font/src/ir/glyph_name.rs index c50667e9..2f43a3e4 100644 --- a/crates/shift-font/src/ir/glyph_name.rs +++ b/crates/shift-font/src/ir/glyph_name.rs @@ -1,9 +1,10 @@ -use serde::{Deserialize, Serialize}; +use serde::de::Error as _; +use serde::{Deserialize, Deserializer, Serialize}; use std::borrow::Borrow; use std::fmt; use std::ops::Deref; -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] #[serde(transparent)] pub struct GlyphName(String); @@ -30,6 +31,16 @@ impl GlyphName { } } +impl<'de> Deserialize<'de> for GlyphName { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(D::Error::custom) + } +} + impl From for GlyphName { fn from(value: String) -> Self { Self::new(value).expect("glyph name must not be empty") diff --git a/crates/shift-font/src/ir/lib_data.rs b/crates/shift-font/src/ir/lib_data.rs index a47f5541..7c542a97 100644 --- a/crates/shift-font/src/ir/lib_data.rs +++ b/crates/shift-font/src/ir/lib_data.rs @@ -1,9 +1,10 @@ +use std::collections::BTreeMap; + use serde::{Deserialize, Serialize}; -use std::collections::HashMap; #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct LibData { - data: HashMap, + data: BTreeMap, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -16,7 +17,7 @@ pub enum LibValue { Float(f64), Boolean(bool), Array(Vec), - Dict(HashMap), + Dict(BTreeMap), Data(Vec), /// A plist date, carried as the RFC 3339 / ISO 8601 string used by XML /// plists (`plist::Date::to_xml_format`), e.g. `2024-02-02T02:02:02Z`. @@ -29,7 +30,7 @@ pub enum LibValue { impl LibData { pub fn new() -> Self { Self { - data: HashMap::new(), + data: BTreeMap::new(), } } @@ -61,17 +62,19 @@ impl LibData { self.data.iter() } - pub fn into_inner(self) -> HashMap { + pub fn into_inner(self) -> BTreeMap { self.data } - pub fn from_map(data: HashMap) -> Self { - Self { data } + pub fn from_map(data: impl IntoIterator) -> Self { + Self { + data: data.into_iter().collect(), + } } } -impl From> for LibData { - fn from(data: HashMap) -> Self { +impl From> for LibData { + fn from(data: BTreeMap) -> Self { Self { data } } } diff --git a/crates/shift-font/src/ir/mod.rs b/crates/shift-font/src/ir/mod.rs index dd1f6a41..1d72ecb7 100644 --- a/crates/shift-font/src/ir/mod.rs +++ b/crates/shift-font/src/ir/mod.rs @@ -29,8 +29,8 @@ pub use boolean::{boolean, BooleanOp}; pub use component::{Component, DecomposedTransform, Transform}; pub use contour::{Contour, Contours}; pub use entity::{ - AnchorId, AxisId, AxisLabelId, AxisMappingId, ComponentId, ContourId, EntityId, GlyphId, - GuidelineId, LayerId, MetricId, NamedInstanceId, PointId, SourceId, + AnchorId, AxisId, AxisLabelId, AxisMappingId, ComponentId, ContourId, EntityId, GlyphEntityId, + GlyphId, GuidelineId, LayerId, MetricId, NamedInstanceId, PointId, SourceId, }; pub use features::FeatureData; pub use font::{Font, FontMetadata}; diff --git a/crates/shift-font/src/test_support.rs b/crates/shift-font/src/test_support.rs index d19d15c1..5deb8e78 100644 --- a/crates/shift-font/src/test_support.rs +++ b/crates/shift-font/src/test_support.rs @@ -7,7 +7,7 @@ use crate::{ LibValue, Location, MetricKind, MetricValue, NamedInstance, NamedInstanceId, Point, PointId, PointType, Source, SourceId, }; -use std::collections::HashMap; +use std::collections::BTreeMap; /// Builds a kitchen-sink font corpus for semantic round-trip tests. /// @@ -58,7 +58,7 @@ pub fn sample_font() -> Font { font_guideline.set_color(Some("green".to_string())); font.add_guideline(font_guideline); - let mut nested = HashMap::new(); + let mut nested = BTreeMap::new(); nested.insert( "nestedString".to_string(), LibValue::String("value".to_string()), @@ -90,7 +90,7 @@ pub fn sample_font() -> Font { ); font.fontinfo_remainder_mut() .set("openTypeOS2WeightClass".to_string(), LibValue::Integer(700)); - let mut woff_metadata = HashMap::new(); + let mut woff_metadata = BTreeMap::new(); woff_metadata.insert( "id".to_string(), LibValue::String("dogfood-unique-id".to_string()), diff --git a/crates/shift-glyph-codec/Cargo.toml b/crates/shift-glyph-codec/Cargo.toml deleted file mode 100644 index 8b76d1ce..00000000 --- a/crates/shift-glyph-codec/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "shift-glyph-codec" -version = "0.1.0" -edition = "2021" -description = "Strict packed glyph payload codecs shared by Shift transports and stores" -license = "MIT OR Apache-2.0" - -[lib] -crate-type = ["rlib"] diff --git a/crates/shift-glyph-codec/SPECIFICATION.md b/crates/shift-glyph-codec/SPECIFICATION.md deleted file mode 100644 index 9134f373..00000000 --- a/crates/shift-glyph-codec/SPECIFICATION.md +++ /dev/null @@ -1,120 +0,0 @@ -# Shift glyph-codec specification - -Status: normative for `shift.glyph-outline.v1` - -## Compatibility and ownership - -The glyph-codec family uses one common frame and independently versioned payload -kinds. The byte specification and shared golden vectors define compatibility; -implementations do not define the format by accident. - -- `shift.glyph-outline.v1` (`payloadKind = 0x01`, `formatVersion = 0x01`) is a - derived, flattened preview/rendering outline. Components are already resolved, - coordinates are drawing coordinates, and f64-to-f32 loss is permitted. It is - never authoritative editable glyph state. -- `payloadKind = 0x02` is reserved for a future `shift.glyph-layer.v1` format. - That separate canonical format must preserve stable identities, authored point - semantics and ordering, contour closure, smooth flags, components and - transforms, anchors, metadata, and authored f64 precision. Outline bytes must - never be persisted or written back as a glyph layer. - -A future incompatible interpretation uses a new version scoped to its payload -kind. Unknown kinds, versions, and flag bits are errors. Codec implementations -own framing, validation, errors, and opaque packed values; they do not own font -semantics, transport, persistence, DOM, canvas, or SQLite behavior. - -## Common frame - -All multibyte values are little-endian. - -```text -u8[4] magic ASCII "SHFT" -u8 payloadKind 0x01 = glyph-outline; 0x02 reserved for glyph-layer -u8 formatVersion version scoped to payloadKind -u16 flags zero in outline v1 -``` - -## `shift.glyph-outline.v1` - -One buffer represents one resolved outline. - -```text -u8[4] magic "SHFT" -u8 payloadKind 0x01 -u8 formatVersion 0x01 -u16 flags 0 -u32 commandCount -u32 coordCount number of f32 values, not points -u8[commandCount] commands -u8[...] zero padding coordinate section begins at a 4-byte offset -f32[coordCount] coordinates -``` - -Its exact byte length is: - -```text -16 + align4(commandCount) + 4 * coordCount -``` - -No trailing bytes are permitted. - -### Commands - -| Byte | Command | Coordinates consumed | -| ---: | --- | ---: | -| `0` | `Move(x, y)` | 2 | -| `1` | `Line(x, y)` | 2 | -| `2` | `Quad(cx, cy, x, y)` | 4 | -| `3` | `Cubic(c1x, c1y, c2x, c2y, x, y)` | 6 | -| `4` | `Close` | 0 | - -The general segment set is intentional. Any quadratic-only Slug reduction is a -consumer transformation after decode. - -### Command state - -- An empty outline is valid only when both counts are zero. -- Every non-empty contour begins with `Move`. -- `Line`, `Quad`, and `Cubic` require an active contour. -- `Close` requires an active contour with at least one drawing segment and ends - that contour. -- After `Close`, only `Move` or the end of the stream is legal. -- A new `Move` may end an open contour and begin another. -- Open contours are legal. -- The command arity sum equals `coordCount` exactly. - -### Coordinates - -Coordinates are finite IEEE-754 binary32 values. An encoder receiving f64 -coordinates rejects non-finite inputs and finite inputs whose conversion is not -a finite f32. NaN and either infinity are invalid on decode. - -### Strict validation - -A decoder returns either one complete validated view or an error. It rejects: - -- truncated framing or body sections; -- wrong magic, kind, version, or non-zero flags; -- unknown commands and illegal state transitions; -- an arity sum different from `coordCount`; -- non-zero alignment padding; -- non-finite coordinates; -- arithmetic overflow and configured implementation-limit violations; -- lengths inconsistent with counts and all trailing bytes. - -V1 implementations in this repository apply the same explicit limits before -allocation or command iteration: - -- at most 1,000,000 commands; -- at most 6,000,000 coordinates; -- at most 32 MiB total payload bytes. - -These are implementation safety limits, not permission for future v1 encoders -to alter framing. - -## Golden vectors - -Canonical binary vectors are checked in at -`fixtures/glyph-codec/outline-v1/`. Rust and TypeScript tests both consume those -same files and compare decoded commands and f32 coordinates. Re-encoding any -accepted canonical vector must reproduce every byte, including zero padding. diff --git a/crates/shift-glyph-codec/examples/outline_benchmark.rs b/crates/shift-glyph-codec/examples/outline_benchmark.rs deleted file mode 100644 index 344f3c80..00000000 --- a/crates/shift-glyph-codec/examples/outline_benchmark.rs +++ /dev/null @@ -1,60 +0,0 @@ -use std::{hint::black_box, time::Instant}; - -use shift_glyph_codec::{pack_outline, OutlineCommand}; - -fn representative_outline() -> Vec> { - let mut commands = vec![OutlineCommand::Move { x: 0.0, y: 0.0 }]; - for index in 0..80 { - let x = f64::from(index) * 7.25; - match index % 3 { - 0 => commands.push(OutlineCommand::Line { x, y: x * 0.5 }), - 1 => commands.push(OutlineCommand::Quad { - cx: x - 2.0, - cy: x * 0.25, - x, - y: x * 0.5, - }), - _ => commands.push(OutlineCommand::Cubic { - c1x: x - 4.0, - c1y: x * 0.2, - c2x: x - 2.0, - c2y: x * 0.4, - x, - y: x * 0.5, - }), - } - } - commands.push(OutlineCommand::Close); - commands -} - -fn main() { - const ITERATIONS: usize = 20_000; - const WARMUP_GLYPHS: usize = 400; - let commands = representative_outline(); - let packed = pack_outline(&commands).expect("representative outline should encode"); - - for _ in 0..1_000 { - black_box(pack_outline(black_box(&commands)).unwrap()); - } - - let start = Instant::now(); - for _ in 0..ITERATIONS { - black_box(pack_outline(black_box(&commands)).unwrap()); - } - let elapsed = start.elapsed(); - let per_glyph_micros = elapsed.as_secs_f64() * 1_000_000.0 / ITERATIONS as f64; - - let warmup_start = Instant::now(); - for _ in 0..WARMUP_GLYPHS { - black_box(pack_outline(black_box(&commands)).unwrap()); - } - - println!( - "payload_bytes={} commands={} encode_us_per_glyph={:.3} encode_400_ms={:.3}", - packed.as_bytes().len(), - commands.len(), - per_glyph_micros, - warmup_start.elapsed().as_secs_f64() * 1_000.0, - ); -} diff --git a/crates/shift-glyph-codec/src/lib.rs b/crates/shift-glyph-codec/src/lib.rs deleted file mode 100644 index 08f09b6b..00000000 --- a/crates/shift-glyph-codec/src/lib.rs +++ /dev/null @@ -1,680 +0,0 @@ -//! Packed glyph payload codecs. -//! -//! This crate owns byte framing and strict validation. It intentionally has no -//! dependency on `shift-font`: authored geometry adaptation belongs to the -//! domain/transport boundary, not to the codec. - -use std::{error::Error, fmt}; - -const MAGIC: &[u8; 4] = b"SHFT"; -const OUTLINE_KIND: u8 = 0x01; -const OUTLINE_VERSION: u8 = 0x01; -const HEADER_LEN: usize = 16; - -/// Maximum command count accepted by the v1 implementation. -pub const MAX_COMMAND_COUNT: usize = 1_000_000; -/// Maximum coordinate count accepted by the v1 implementation. -pub const MAX_COORD_COUNT: usize = 6_000_000; -/// Maximum total payload size accepted by the v1 implementation. -pub const MAX_PAYLOAD_BYTES: usize = 32 * 1024 * 1024; - -/// One codec-owned drawing command. -/// -/// Encoders accept `OutlineCommand` and perform the specified finite f64 -/// to f32 conversion. Decoded iterators yield `OutlineCommand` directly -/// from the validated payload. -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum OutlineCommand { - Move { - x: T, - y: T, - }, - Line { - x: T, - y: T, - }, - Quad { - cx: T, - cy: T, - x: T, - y: T, - }, - Cubic { - c1x: T, - c1y: T, - c2x: T, - c2y: T, - x: T, - y: T, - }, - Close, -} - -impl OutlineCommand { - fn byte(&self) -> u8 { - match self { - Self::Move { .. } => 0, - Self::Line { .. } => 1, - Self::Quad { .. } => 2, - Self::Cubic { .. } => 3, - Self::Close => 4, - } - } - - fn arity(&self) -> usize { - match self { - Self::Move { .. } | Self::Line { .. } => 2, - Self::Quad { .. } => 4, - Self::Cubic { .. } => 6, - Self::Close => 0, - } - } -} - -/// The command-stream rule violated by an encoder or decoder. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum CommandOrderError { - DrawingCommandWithoutContour, - CloseWithoutContour, - CloseWithoutDrawingSegment, -} - -impl fmt::Display for CommandOrderError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::DrawingCommandWithoutContour => { - f.write_str("drawing command requires an active contour") - } - Self::CloseWithoutContour => f.write_str("close requires an active contour"), - Self::CloseWithoutDrawingSegment => { - f.write_str("close requires at least one drawing segment") - } - } - } -} - -/// Strict outline codec failure. -#[derive(Debug, Eq, PartialEq)] -pub enum CodecError { - HeaderTruncated { - actual: usize, - }, - WrongMagic, - UnsupportedPayloadKind(u8), - UnsupportedVersion(u8), - UnknownFlags(u16), - UnknownCommand { - index: usize, - command: u8, - }, - InvalidCommandOrder { - index: usize, - reason: CommandOrderError, - }, - CoordinateCountMismatch { - expected: usize, - actual: usize, - }, - NonZeroPadding { - offset: usize, - }, - NonFiniteCoordinate { - index: usize, - }, - NonFiniteInputCoordinate { - index: usize, - }, - CoordinateOutOfF32Range { - index: usize, - }, - LengthOverflow, - LengthMismatch { - expected: usize, - actual: usize, - }, - LimitExceeded { - field: &'static str, - limit: usize, - actual: usize, - }, -} - -impl fmt::Display for CodecError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::HeaderTruncated { actual } => { - write!( - f, - "outline header is truncated: {actual} of {HEADER_LEN} bytes" - ) - } - Self::WrongMagic => f.write_str("wrong glyph-codec magic"), - Self::UnsupportedPayloadKind(kind) => { - write!(f, "unsupported glyph-codec payload kind {kind:#04x}") - } - Self::UnsupportedVersion(version) => { - write!(f, "unsupported glyph-outline version {version}") - } - Self::UnknownFlags(flags) => write!(f, "unknown glyph-outline flags {flags:#06x}"), - Self::UnknownCommand { index, command } => { - write!(f, "unknown outline command {command:#04x} at index {index}") - } - Self::InvalidCommandOrder { index, reason } => { - write!( - f, - "invalid outline command ordering at index {index}: {reason}" - ) - } - Self::CoordinateCountMismatch { expected, actual } => write!( - f, - "outline command arity requires {expected} coordinates, header declares {actual}" - ), - Self::NonZeroPadding { offset } => { - write!(f, "non-zero outline alignment padding at byte {offset}") - } - Self::NonFiniteCoordinate { index } => { - write!(f, "non-finite outline coordinate at index {index}") - } - Self::NonFiniteInputCoordinate { index } => { - write!(f, "non-finite input coordinate at index {index}") - } - Self::CoordinateOutOfF32Range { index } => { - write!( - f, - "input coordinate at index {index} is outside finite f32 range" - ) - } - Self::LengthOverflow => f.write_str("outline payload length arithmetic overflowed"), - Self::LengthMismatch { expected, actual } => write!( - f, - "outline payload length mismatch: expected {expected} bytes, got {actual}" - ), - Self::LimitExceeded { - field, - limit, - actual, - } => write!( - f, - "outline {field} exceeds implementation limit {limit}: got {actual}" - ), - } - } -} - -impl Error for CodecError {} - -/// An owned, canonical `shift.glyph-outline.v1` payload. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct PackedGlyphOutline(Vec); - -impl PackedGlyphOutline { - /// Validates and takes ownership of an existing payload. - pub fn from_bytes(bytes: Vec) -> Result { - decode_outline(&bytes)?; - Ok(Self(bytes)) - } - - /// Returns the canonical encoded bytes. - pub fn as_bytes(&self) -> &[u8] { - &self.0 - } - - /// Consumes the opaque value and returns its canonical bytes. - pub fn into_bytes(self) -> Vec { - self.0 - } - - /// Returns a validated, allocation-free view over this payload. - pub fn view(&self) -> OutlineView<'_> { - // Construction and decoding are the only ways to create this type. - decode_outline(&self.0).expect("PackedGlyphOutline invariant violated") - } -} - -/// Incremental canonical encoder used by domain adapters to avoid allocating an -/// intermediate command-object graph. -#[derive(Debug, Default)] -pub struct OutlineEncoder { - commands: Vec, - coordinates: Vec, - state: CommandState, -} - -impl OutlineEncoder { - pub fn new() -> Self { - Self::default() - } - - /// Creates an encoder with checked capacities. - pub fn with_capacity( - command_count: usize, - coordinate_count: usize, - ) -> Result { - check_limit("command count", command_count, MAX_COMMAND_COUNT)?; - check_limit("coordinate count", coordinate_count, MAX_COORD_COUNT)?; - - Ok(Self { - commands: Vec::with_capacity(command_count), - coordinates: Vec::with_capacity(coordinate_count), - state: CommandState::default(), - }) - } - - /// Adds one command after validating ordering and every converted coordinate. - pub fn push(&mut self, command: OutlineCommand) -> Result<(), CodecError> { - let command_index = self.commands.len(); - check_limit( - "command count", - command_index - .checked_add(1) - .ok_or(CodecError::LengthOverflow)?, - MAX_COMMAND_COUNT, - )?; - let mut next_state = self.state; - validate_order(&mut next_state, command.byte(), command_index)?; - - let coordinate_index = self.coordinates.len(); - let arity = command.arity(); - check_limit( - "coordinate count", - coordinate_index - .checked_add(arity) - .ok_or(CodecError::LengthOverflow)?, - MAX_COORD_COUNT, - )?; - - let mut converted = [0.0_f32; 6]; - let count = match command { - OutlineCommand::Move { x, y } | OutlineCommand::Line { x, y } => { - converted[0] = convert_coordinate(x, coordinate_index)?; - converted[1] = convert_coordinate(y, coordinate_index + 1)?; - 2 - } - OutlineCommand::Quad { cx, cy, x, y } => { - for (slot, (value, index)) in converted.iter_mut().zip( - [cx, cy, x, y] - .into_iter() - .zip(coordinate_index..coordinate_index + 4), - ) { - *slot = convert_coordinate(value, index)?; - } - 4 - } - OutlineCommand::Cubic { - c1x, - c1y, - c2x, - c2y, - x, - y, - } => { - for (slot, (value, index)) in converted.iter_mut().zip( - [c1x, c1y, c2x, c2y, x, y] - .into_iter() - .zip(coordinate_index..coordinate_index + 6), - ) { - *slot = convert_coordinate(value, index)?; - } - 6 - } - OutlineCommand::Close => 0, - }; - - self.state = next_state; - self.commands.push(command.byte()); - self.coordinates.extend_from_slice(&converted[..count]); - Ok(()) - } - - /// Finishes the payload with canonical zero padding and little-endian f32s. - pub fn finish(self) -> Result { - let command_count = self.commands.len(); - let coordinate_count = self.coordinates.len(); - let aligned_commands = align4(command_count).ok_or(CodecError::LengthOverflow)?; - let coordinate_bytes = coordinate_count - .checked_mul(4) - .ok_or(CodecError::LengthOverflow)?; - let total_len = HEADER_LEN - .checked_add(aligned_commands) - .and_then(|value| value.checked_add(coordinate_bytes)) - .ok_or(CodecError::LengthOverflow)?; - check_limit("payload byte length", total_len, MAX_PAYLOAD_BYTES)?; - - let command_count = u32::try_from(command_count).map_err(|_| CodecError::LengthOverflow)?; - let coordinate_count = - u32::try_from(coordinate_count).map_err(|_| CodecError::LengthOverflow)?; - let mut bytes = Vec::with_capacity(total_len); - bytes.extend_from_slice(MAGIC); - bytes.push(OUTLINE_KIND); - bytes.push(OUTLINE_VERSION); - bytes.extend_from_slice(&0_u16.to_le_bytes()); - bytes.extend_from_slice(&command_count.to_le_bytes()); - bytes.extend_from_slice(&coordinate_count.to_le_bytes()); - bytes.extend_from_slice(&self.commands); - bytes.resize(HEADER_LEN + aligned_commands, 0); - for coordinate in self.coordinates { - bytes.extend_from_slice(&coordinate.to_le_bytes()); - } - - debug_assert_eq!(bytes.len(), total_len); - Ok(PackedGlyphOutline(bytes)) - } -} - -/// Canonically encodes a codec-owned command slice. -pub fn pack_outline(commands: &[OutlineCommand]) -> Result { - let coordinate_count = commands.iter().try_fold(0_usize, |count, command| { - count - .checked_add(command.arity()) - .ok_or(CodecError::LengthOverflow) - })?; - let mut encoder = OutlineEncoder::with_capacity(commands.len(), coordinate_count)?; - for command in commands { - encoder.push(*command)?; - } - encoder.finish() -} - -/// A validated, borrowed outline payload view. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct OutlineView<'a> { - bytes: &'a [u8], - command_count: usize, - coordinate_count: usize, - coordinate_offset: usize, -} - -impl<'a> OutlineView<'a> { - pub fn command_count(&self) -> usize { - self.command_count - } - - pub fn coordinate_count(&self) -> usize { - self.coordinate_count - } - - /// Iterates decoded commands without allocating points or offset tables. - pub fn commands(&self) -> OutlineCommandIter<'a> { - OutlineCommandIter { - bytes: self.bytes, - command_count: self.command_count, - command_index: 0, - coordinate_offset: self.coordinate_offset, - } - } - - /// Iterates all coordinates in command-stream order. - pub fn coordinates(&self) -> CoordinateIter<'a> { - CoordinateIter { - bytes: self.bytes, - offset: self.coordinate_offset, - remaining: self.coordinate_count, - } - } -} - -/// Strictly validates one outline buffer and returns an allocation-free view. -pub fn decode_outline(bytes: &[u8]) -> Result, CodecError> { - if bytes.len() < HEADER_LEN { - return Err(CodecError::HeaderTruncated { - actual: bytes.len(), - }); - } - check_limit("payload byte length", bytes.len(), MAX_PAYLOAD_BYTES)?; - - if &bytes[..4] != MAGIC { - return Err(CodecError::WrongMagic); - } - if bytes[4] != OUTLINE_KIND { - return Err(CodecError::UnsupportedPayloadKind(bytes[4])); - } - if bytes[5] != OUTLINE_VERSION { - return Err(CodecError::UnsupportedVersion(bytes[5])); - } - - let flags = u16::from_le_bytes([bytes[6], bytes[7]]); - if flags != 0 { - return Err(CodecError::UnknownFlags(flags)); - } - - let command_count = read_u32(bytes, 8) as usize; - let coordinate_count = read_u32(bytes, 12) as usize; - check_limit("command count", command_count, MAX_COMMAND_COUNT)?; - check_limit("coordinate count", coordinate_count, MAX_COORD_COUNT)?; - - let aligned_commands = align4(command_count).ok_or(CodecError::LengthOverflow)?; - let coordinate_bytes = coordinate_count - .checked_mul(4) - .ok_or(CodecError::LengthOverflow)?; - let expected_len = HEADER_LEN - .checked_add(aligned_commands) - .and_then(|value| value.checked_add(coordinate_bytes)) - .ok_or(CodecError::LengthOverflow)?; - check_limit("payload byte length", expected_len, MAX_PAYLOAD_BYTES)?; - if bytes.len() != expected_len { - return Err(CodecError::LengthMismatch { - expected: expected_len, - actual: bytes.len(), - }); - } - - let commands_end = HEADER_LEN + command_count; - let mut state = CommandState::default(); - let mut expected_coordinate_count = 0_usize; - for (index, command) in bytes[HEADER_LEN..commands_end].iter().copied().enumerate() { - let arity = command_arity(command).ok_or(CodecError::UnknownCommand { index, command })?; - validate_order(&mut state, command, index)?; - expected_coordinate_count = expected_coordinate_count - .checked_add(arity) - .ok_or(CodecError::LengthOverflow)?; - } - if expected_coordinate_count != coordinate_count { - return Err(CodecError::CoordinateCountMismatch { - expected: expected_coordinate_count, - actual: coordinate_count, - }); - } - - let coordinate_offset = HEADER_LEN + aligned_commands; - for (relative_offset, byte) in bytes[commands_end..coordinate_offset] - .iter() - .copied() - .enumerate() - { - if byte != 0 { - return Err(CodecError::NonZeroPadding { - offset: commands_end + relative_offset, - }); - } - } - - for index in 0..coordinate_count { - let coordinate = read_f32(bytes, coordinate_offset + index * 4); - if !coordinate.is_finite() { - return Err(CodecError::NonFiniteCoordinate { index }); - } - } - - Ok(OutlineView { - bytes, - command_count, - coordinate_count, - coordinate_offset, - }) -} - -/// Iterator over commands in a validated outline view. -pub struct OutlineCommandIter<'a> { - bytes: &'a [u8], - command_count: usize, - command_index: usize, - coordinate_offset: usize, -} - -impl Iterator for OutlineCommandIter<'_> { - type Item = OutlineCommand; - - fn next(&mut self) -> Option { - if self.command_index >= self.command_count { - return None; - } - - let command = self.bytes[HEADER_LEN + self.command_index]; - self.command_index += 1; - let mut next = || { - let value = read_f32(self.bytes, self.coordinate_offset); - self.coordinate_offset += 4; - value - }; - Some(match command { - 0 => OutlineCommand::Move { - x: next(), - y: next(), - }, - 1 => OutlineCommand::Line { - x: next(), - y: next(), - }, - 2 => OutlineCommand::Quad { - cx: next(), - cy: next(), - x: next(), - y: next(), - }, - 3 => OutlineCommand::Cubic { - c1x: next(), - c1y: next(), - c2x: next(), - c2y: next(), - x: next(), - y: next(), - }, - 4 => OutlineCommand::Close, - _ => unreachable!("validated command byte"), - }) - } - - fn size_hint(&self) -> (usize, Option) { - let remaining = self.command_count - self.command_index; - (remaining, Some(remaining)) - } -} - -impl ExactSizeIterator for OutlineCommandIter<'_> {} - -/// Iterator over f32 coordinates in a validated outline view. -pub struct CoordinateIter<'a> { - bytes: &'a [u8], - offset: usize, - remaining: usize, -} - -impl Iterator for CoordinateIter<'_> { - type Item = f32; - - fn next(&mut self) -> Option { - if self.remaining == 0 { - return None; - } - - let value = read_f32(self.bytes, self.offset); - self.offset += 4; - self.remaining -= 1; - Some(value) - } - - fn size_hint(&self) -> (usize, Option) { - (self.remaining, Some(self.remaining)) - } -} - -impl ExactSizeIterator for CoordinateIter<'_> {} - -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -enum CommandState { - #[default] - BetweenContours, - ActiveWithoutSegment, - ActiveWithSegment, -} - -fn validate_order(state: &mut CommandState, command: u8, index: usize) -> Result<(), CodecError> { - match command { - 0 => *state = CommandState::ActiveWithoutSegment, - 1..=3 => match state { - CommandState::BetweenContours => { - return Err(CodecError::InvalidCommandOrder { - index, - reason: CommandOrderError::DrawingCommandWithoutContour, - }); - } - CommandState::ActiveWithoutSegment | CommandState::ActiveWithSegment => { - *state = CommandState::ActiveWithSegment; - } - }, - 4 => match state { - CommandState::BetweenContours => { - return Err(CodecError::InvalidCommandOrder { - index, - reason: CommandOrderError::CloseWithoutContour, - }); - } - CommandState::ActiveWithoutSegment => { - return Err(CodecError::InvalidCommandOrder { - index, - reason: CommandOrderError::CloseWithoutDrawingSegment, - }); - } - CommandState::ActiveWithSegment => *state = CommandState::BetweenContours, - }, - _ => unreachable!("command byte validated before ordering"), - } - - Ok(()) -} - -fn command_arity(command: u8) -> Option { - match command { - 0 | 1 => Some(2), - 2 => Some(4), - 3 => Some(6), - 4 => Some(0), - _ => None, - } -} - -fn convert_coordinate(value: f64, index: usize) -> Result { - if !value.is_finite() { - return Err(CodecError::NonFiniteInputCoordinate { index }); - } - - let converted = value as f32; - if !converted.is_finite() { - return Err(CodecError::CoordinateOutOfF32Range { index }); - } - - Ok(converted) -} - -fn check_limit(field: &'static str, actual: usize, limit: usize) -> Result<(), CodecError> { - if actual > limit { - return Err(CodecError::LimitExceeded { - field, - limit, - actual, - }); - } - Ok(()) -} - -fn align4(value: usize) -> Option { - value.checked_add(3).map(|value| value & !3) -} - -fn read_u32(bytes: &[u8], offset: usize) -> u32 { - u32::from_le_bytes(bytes[offset..offset + 4].try_into().expect("fixed width")) -} - -fn read_f32(bytes: &[u8], offset: usize) -> f32 { - f32::from_le_bytes(bytes[offset..offset + 4].try_into().expect("fixed width")) -} diff --git a/crates/shift-glyph-codec/tests/outline.rs b/crates/shift-glyph-codec/tests/outline.rs deleted file mode 100644 index ad3f196e..00000000 --- a/crates/shift-glyph-codec/tests/outline.rs +++ /dev/null @@ -1,386 +0,0 @@ -use shift_glyph_codec::{ - decode_outline, pack_outline, CodecError, CommandOrderError, OutlineCommand, MAX_COMMAND_COUNT, -}; - -type Command64 = OutlineCommand; -type Command32 = OutlineCommand; - -const EMPTY: &[u8] = include_bytes!("../../../fixtures/glyph-codec/outline-v1/empty.bin"); -const CLOSED_LINE: &[u8] = - include_bytes!("../../../fixtures/glyph-codec/outline-v1/closed-line.bin"); -const OPEN: &[u8] = include_bytes!("../../../fixtures/glyph-codec/outline-v1/open.bin"); -const CURVES: &[u8] = - include_bytes!("../../../fixtures/glyph-codec/outline-v1/quadratic-cubic.bin"); -const MULTIPLE: &[u8] = - include_bytes!("../../../fixtures/glyph-codec/outline-v1/multiple-contours.bin"); - -fn golden_vectors() -> Vec<(&'static str, &'static [u8], Vec)> { - vec![ - ("empty", EMPTY, vec![]), - ( - "closed-line", - CLOSED_LINE, - vec![ - Command64::Move { x: 0.0, y: 0.0 }, - Command64::Line { x: 100.0, y: 200.0 }, - Command64::Close, - ], - ), - ( - "open", - OPEN, - vec![ - Command64::Move { x: -12.5, y: 0.25 }, - Command64::Line { - x: 123.75, - y: -456.5, - }, - ], - ), - ( - "quadratic-cubic", - CURVES, - vec![ - Command64::Move { x: -1.5, y: 2.25 }, - Command64::Quad { - cx: 0.5, - cy: -0.75, - x: 1000.125, - y: -2000.5, - }, - Command64::Cubic { - c1x: -3.0, - c1y: 4.5, - c2x: 5.25, - c2y: -6.75, - x: 1.0e20, - y: -1.0e20, - }, - Command64::Close, - ], - ), - ( - "multiple-contours", - MULTIPLE, - vec![ - Command64::Move { x: 0.0, y: 0.0 }, - Command64::Line { x: 10.0, y: 0.0 }, - Command64::Close, - Command64::Move { - x: 100.5, - y: -100.25, - }, - Command64::Line { x: 200.0, y: 300.0 }, - ], - ), - ] -} - -fn as_f32(command: Command64) -> Command32 { - match command { - Command64::Move { x, y } => Command32::Move { - x: x as f32, - y: y as f32, - }, - Command64::Line { x, y } => Command32::Line { - x: x as f32, - y: y as f32, - }, - Command64::Quad { cx, cy, x, y } => Command32::Quad { - cx: cx as f32, - cy: cy as f32, - x: x as f32, - y: y as f32, - }, - Command64::Cubic { - c1x, - c1y, - c2x, - c2y, - x, - y, - } => Command32::Cubic { - c1x: c1x as f32, - c1y: c1y as f32, - c2x: c2x as f32, - c2y: c2y as f32, - x: x as f32, - y: y as f32, - }, - Command64::Close => Command32::Close, - } -} - -fn as_f64(command: Command32) -> Command64 { - match command { - Command32::Move { x, y } => Command64::Move { - x: f64::from(x), - y: f64::from(y), - }, - Command32::Line { x, y } => Command64::Line { - x: f64::from(x), - y: f64::from(y), - }, - Command32::Quad { cx, cy, x, y } => Command64::Quad { - cx: f64::from(cx), - cy: f64::from(cy), - x: f64::from(x), - y: f64::from(y), - }, - Command32::Cubic { - c1x, - c1y, - c2x, - c2y, - x, - y, - } => Command64::Cubic { - c1x: f64::from(c1x), - c1y: f64::from(c1y), - c2x: f64::from(c2x), - c2y: f64::from(c2y), - x: f64::from(x), - y: f64::from(y), - }, - Command32::Close => Command64::Close, - } -} - -fn raw_outline(commands: &[u8], coordinates: &[f32]) -> Vec { - let aligned_commands = (commands.len() + 3) & !3; - let mut bytes = Vec::with_capacity(16 + aligned_commands + coordinates.len() * 4); - bytes.extend_from_slice(b"SHFT"); - bytes.extend_from_slice(&[1, 1, 0, 0]); - bytes.extend_from_slice(&(commands.len() as u32).to_le_bytes()); - bytes.extend_from_slice(&(coordinates.len() as u32).to_le_bytes()); - bytes.extend_from_slice(commands); - bytes.resize(16 + aligned_commands, 0); - for coordinate in coordinates { - bytes.extend_from_slice(&coordinate.to_le_bytes()); - } - bytes -} - -fn canonicalize(bytes: &[u8]) -> Option> { - let view = decode_outline(bytes).ok()?; - let commands = view.commands().map(as_f64).collect::>(); - Some(pack_outline(&commands).unwrap().into_bytes()) -} - -#[test] -fn shared_golden_vectors_decode_and_reencode_byte_for_byte() { - for (name, bytes, expected) in golden_vectors() { - let view = decode_outline(bytes).unwrap_or_else(|error| panic!("{name}: {error}")); - let expected_f32 = expected.iter().copied().map(as_f32).collect::>(); - - assert_eq!(view.commands().collect::>(), expected_f32, "{name}"); - assert_eq!(pack_outline(&expected).unwrap().as_bytes(), bytes, "{name}"); - assert_eq!(canonicalize(bytes).as_deref(), Some(bytes), "{name}"); - } -} - -#[test] -fn golden_vectors_cover_every_command_alignment_padding_length() { - let remainders = golden_vectors() - .into_iter() - .map(|(_, bytes, _)| decode_outline(bytes).unwrap().command_count() % 4) - .collect::>(); - - assert_eq!(remainders, [0, 1, 2, 3].into_iter().collect()); -} - -#[test] -fn decoder_rejects_truncation_and_trailing_bytes() { - for length in 0..CURVES.len() { - assert!( - decode_outline(&CURVES[..length]).is_err(), - "length {length}" - ); - } - - let mut trailing = CURVES.to_vec(); - trailing.push(0); - assert!(matches!( - decode_outline(&trailing), - Err(CodecError::LengthMismatch { .. }) - )); -} - -#[test] -fn decoder_rejects_wrong_framing_and_unknown_flags() { - let mut bytes = OPEN.to_vec(); - bytes[0] = b'X'; - assert_eq!(decode_outline(&bytes), Err(CodecError::WrongMagic)); - - let mut bytes = OPEN.to_vec(); - bytes[4] = 2; - assert_eq!( - decode_outline(&bytes), - Err(CodecError::UnsupportedPayloadKind(2)) - ); - - let mut bytes = OPEN.to_vec(); - bytes[5] = 2; - assert_eq!( - decode_outline(&bytes), - Err(CodecError::UnsupportedVersion(2)) - ); - - let mut bytes = OPEN.to_vec(); - bytes[6] = 1; - assert_eq!(decode_outline(&bytes), Err(CodecError::UnknownFlags(1))); -} - -#[test] -fn decoder_rejects_unknown_commands_and_every_illegal_ordering_boundary() { - let unknown = raw_outline(&[9], &[]); - assert!(matches!( - decode_outline(&unknown), - Err(CodecError::UnknownCommand { .. }) - )); - - let drawing_first = raw_outline(&[1], &[1.0, 2.0]); - assert_eq!( - decode_outline(&drawing_first), - Err(CodecError::InvalidCommandOrder { - index: 0, - reason: CommandOrderError::DrawingCommandWithoutContour, - }) - ); - - let close_first = raw_outline(&[4], &[]); - assert_eq!( - decode_outline(&close_first), - Err(CodecError::InvalidCommandOrder { - index: 0, - reason: CommandOrderError::CloseWithoutContour, - }) - ); - - let empty_close = raw_outline(&[0, 4], &[0.0, 0.0]); - assert_eq!( - decode_outline(&empty_close), - Err(CodecError::InvalidCommandOrder { - index: 1, - reason: CommandOrderError::CloseWithoutDrawingSegment, - }) - ); - - let after_close = raw_outline(&[0, 1, 4, 1], &[0.0, 0.0, 1.0, 1.0, 2.0, 2.0]); - assert!(matches!( - decode_outline(&after_close), - Err(CodecError::InvalidCommandOrder { index: 3, .. }) - )); -} - -#[test] -fn decoder_rejects_arity_padding_and_non_finite_coordinates() { - let wrong_arity = raw_outline(&[0, 1], &[0.0, 0.0]); - assert_eq!( - decode_outline(&wrong_arity), - Err(CodecError::CoordinateCountMismatch { - expected: 4, - actual: 2, - }) - ); - - let mut non_zero_padding = OPEN.to_vec(); - non_zero_padding[18] = 1; - assert_eq!( - decode_outline(&non_zero_padding), - Err(CodecError::NonZeroPadding { offset: 18 }) - ); - - for value in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { - let non_finite = raw_outline(&[0], &[value, 0.0]); - assert_eq!( - decode_outline(&non_finite), - Err(CodecError::NonFiniteCoordinate { index: 0 }) - ); - } -} - -#[test] -fn encoder_rejects_illegal_command_ordering() { - assert!(matches!( - pack_outline(&[Command64::Line { x: 1.0, y: 2.0 }]), - Err(CodecError::InvalidCommandOrder { - reason: CommandOrderError::DrawingCommandWithoutContour, - .. - }) - )); - assert!(matches!( - pack_outline(&[Command64::Close]), - Err(CodecError::InvalidCommandOrder { - reason: CommandOrderError::CloseWithoutContour, - .. - }) - )); - assert!(matches!( - pack_outline(&[Command64::Move { x: 0.0, y: 0.0 }, Command64::Close]), - Err(CodecError::InvalidCommandOrder { - reason: CommandOrderError::CloseWithoutDrawingSegment, - .. - }) - )); -} - -#[test] -fn encoder_rejects_non_finite_and_out_of_range_input() { - for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { - assert!(matches!( - pack_outline(&[Command64::Move { x: value, y: 0.0 }]), - Err(CodecError::NonFiniteInputCoordinate { index: 0 }) - )); - } - - assert!(matches!( - pack_outline(&[Command64::Move { - x: f64::MAX, - y: 0.0 - }]), - Err(CodecError::CoordinateOutOfF32Range { index: 0 }) - )); -} - -#[test] -fn decoder_applies_limits_before_body_iteration() { - let mut bytes = EMPTY.to_vec(); - bytes[8..12].copy_from_slice(&((MAX_COMMAND_COUNT as u32) + 1).to_le_bytes()); - - assert!(matches!( - decode_outline(&bytes), - Err(CodecError::LimitExceeded { - field: "command count", - .. - }) - )); -} - -#[test] -fn mutated_payloads_never_panic_and_any_accepted_payload_is_canonical() { - let seed_commands = golden_vectors() - .into_iter() - .flat_map(|(_, _, commands)| commands) - .collect::>(); - let seed = pack_outline(&seed_commands).unwrap().into_bytes(); - let mut random = 0x7a11_4e29_u32; - - for iteration in 0..10_000 { - random ^= random << 13; - random ^= random >> 17; - random ^= random << 5; - let mut candidate = seed.clone(); - let offset = random as usize % candidate.len(); - candidate[offset] ^= (random >> 24) as u8 | 1; - if iteration % 7 == 0 { - candidate.truncate(random as usize % candidate.len()); - } else if iteration % 11 == 0 { - candidate.push((random >> 16) as u8); - } - - if let Some(canonical) = canonicalize(&candidate) { - assert_eq!(canonical, candidate); - } - } -} diff --git a/crates/shift-slug/Cargo.toml b/crates/shift-slug/Cargo.toml index 4ff7a6e3..cb5275ba 100644 --- a/crates/shift-slug/Cargo.toml +++ b/crates/shift-slug/Cargo.toml @@ -14,7 +14,6 @@ wgpu-benchmark = ["dep:naga", "dep:pollster", "dep:wgpu"] [dependencies] shift-font = { workspace = true } -shift-glyph-codec = { workspace = true } naga = { version = "30.0.0", optional = true, features = ["wgsl-in"] } pollster = { version = "0.4", optional = true } diff --git a/crates/shift-slug/docs/DOCS.md b/crates/shift-slug/docs/DOCS.md index c9feb3c1..3500fcaf 100644 --- a/crates/shift-slug/docs/DOCS.md +++ b/crates/shift-slug/docs/DOCS.md @@ -11,12 +11,14 @@ GPU-independent preprocessing for the experimental Slug home/catalog glyph grid. - **Bands are location-bound.** The static builder bands one resolved shape. The variable path resolves and re-bands only visible glyphs after every weight update, so current-location membership stays exact without geometry upload. - **Deterministic topology conversion.** Lines become quadratics. Cubics use the conservative third-derivative error bound from Kurbo's `CubicBez::to_quads`, with a one-font-unit tolerance and equal parameter intervals. Compatible authored sources freeze the maximum subdivision count required by any source so variable topology remains identical. - **No shaping.** The grid addresses glyphs by dense atlas index and does not need a text shaper. +- **Command ownership.** `OutlineCommand` is a Slug preprocessing input. No standalone packed-outline storage format exists. ## Codemap ```text src/ lib.rs public boundary + outline.rs Slug-owned drawing command type curve.rs outline commands -> quadratic curves atlas.rs glyph bounds, horizontal/vertical bands, atlas assembly authored.rs stable shift-font point/segment topology and source derivation @@ -25,7 +27,7 @@ src/ render.rs shared uniform and visible-instance byte layouts variable.rs multi-source resident base/delta model and aligned packing variable/component.rs component records, CPU oracle, and component packing - resident.rs complete-font/page authored atlases, identity maps, and weight-basis deduplication + resident.rs complete authored atlas, edit patches, identity maps, and basis deduplication error.rs strict conversion and size failures shaders/ slug.wgsl shared static native-wgpu/Electron-WebGPU renderer @@ -56,7 +58,7 @@ Each glyph owns `band_count` horizontal ranges followed by `band_count` vertical ## Resident variable execution -`build_authored_atlas()` remains the complete-font diagnostic boundary. Product residency uses `build_authored_atlas_page()`: it compiles an ordered root-glyph batch plus transitive component geometry, preserves explicit `GlyphId` mapping, and deduplicates interpolation bases only across that page closure. Layerless root records receive zero-curve/zero-advance descriptors so one incomplete draft cannot disable the rest of the grid. Pages are location-independent; axis movement changes only each page's shared weight vector and visible instances. Complete-font construction delegates to one page containing every root, so page and diagnostic output share the same compiler and packed layout. +`build_authored_atlas()` is the product complete-font residency boundary. It delegates to `build_authored_atlas_page()` with every root so complete residency and local edit patches share one compiler and packed layout. A patch compiles an ordered root-glyph batch plus transitive component geometry, preserves explicit `GlyphId` mapping, and deduplicates interpolation bases across that closure. Layerless root records receive zero-curve/zero-advance descriptors so one incomplete draft cannot disable the rest of the grid. Complete atlases and patches are location-independent; axis movement changes only their shared weight vectors and visible instances. The variable model keeps one base quadratic array plus base-relative `f32` source deltas. Each 8-byte source descriptor remains dense by default; only a source whose unchanged curves make sparse storage strictly smaller receives a tagged offset into a compact side table of sorted glyph-local indexes. Dense fonts therefore pay no sparse metadata tax. Each source references a global weight index so equal interpolation bases share a small per-frame weight vector. The complete packed atlas remains one logical byte stream but may span two `array` storage bindings. A split offset and every logical section offset fit in the existing 64-byte uniform; one accessor selects the physical buffer without changing packed bytes. Typed WGSL decoders preserve the exact little-endian resident layout while using exactly eight storage bindings in the resolve entry point, matching WebGPU's baseline binding-count and 128 MiB storage-binding limits. Compute preserves the full weighted-source equation as `base × sum(weights) + Σ(weight × delta)`, rather than assuming weights always sum to one. A one-bit-per-curve resident mask marks controls generated from authored lines: after endpoint interpolation, compute regenerates those controls with Slug's normalized perpendicular epsilon because that operation is nonlinear and cannot be represented exactly by source control deltas. One workgroup per visible glyph resolves curves and reduces exact current-location bounds into scratch; a second pass rebuilds the eight horizontal and vertical bands using those bounds. Fragment band selection reads the same scratch bounds. Cell sizing remains a consumer-owned metrics/advance transform, so neither loose all-location bounds nor current-location geometry can shrink or jump the grid layout. Offscreen glyphs perform none of this work until visible. diff --git a/crates/shift-slug/examples/analyze_font.rs b/crates/shift-slug/examples/analyze_font.rs index 2efc7c9c..5959be0d 100644 --- a/crates/shift-slug/examples/analyze_font.rs +++ b/crates/shift-slug/examples/analyze_font.rs @@ -1,7 +1,6 @@ use std::{env, error::Error, fs, path::PathBuf, time::Instant}; -use shift_glyph_codec::OutlineCommand; -use shift_slug::{AtlasBuilder, CurveIndexEncoding, DEFAULT_BAND_COUNT}; +use shift_slug::{AtlasBuilder, CurveIndexEncoding, OutlineCommand, DEFAULT_BAND_COUNT}; use skrifa::{ outline::{DrawSettings, OutlinePen}, prelude::{LocationRef, Size}, diff --git a/crates/shift-slug/examples/benchmark_variable_wgpu.rs b/crates/shift-slug/examples/benchmark_variable_wgpu.rs index 2e77fdd5..77c5e51d 100644 --- a/crates/shift-slug/examples/benchmark_variable_wgpu.rs +++ b/crates/shift-slug/examples/benchmark_variable_wgpu.rs @@ -1,9 +1,8 @@ use std::{env, error::Error, fs, path::PathBuf, sync::mpsc, time::Instant}; -use shift_glyph_codec::OutlineCommand; use shift_slug::{ - pack_render_instances, pack_render_params, pack_variable_params, Bounds, Curve, RenderInstance, - RenderParams, SlugError, VariableAtlas, VariableAtlasBuilder, VariableParams, + pack_render_instances, pack_render_params, pack_variable_params, Bounds, Curve, OutlineCommand, + RenderInstance, RenderParams, SlugError, VariableAtlas, VariableAtlasBuilder, VariableParams, DEFAULT_BAND_COUNT, VARIABLE_SLUG_WGSL, }; use skrifa::{ diff --git a/crates/shift-slug/examples/benchmark_wgpu.rs b/crates/shift-slug/examples/benchmark_wgpu.rs index b4c43328..a59b5db2 100644 --- a/crates/shift-slug/examples/benchmark_wgpu.rs +++ b/crates/shift-slug/examples/benchmark_wgpu.rs @@ -8,10 +8,9 @@ use std::{ time::{Duration, Instant}, }; -use shift_glyph_codec::OutlineCommand; use shift_slug::{ pack_render_instances, pack_render_params, Atlas, AtlasBuilder, Bounds, CurveIndexEncoding, - Layout, RenderInstance, RenderParams, Section, DEFAULT_BAND_COUNT, SLUG_WGSL, + Layout, OutlineCommand, RenderInstance, RenderParams, Section, DEFAULT_BAND_COUNT, SLUG_WGSL, }; use skrifa::{ outline::{DrawSettings, OutlinePen}, diff --git a/crates/shift-slug/src/atlas.rs b/crates/shift-slug/src/atlas.rs index 0aabbdcd..7ec968d1 100644 --- a/crates/shift-slug/src/atlas.rs +++ b/crates/shift-slug/src/atlas.rs @@ -1,9 +1,8 @@ use std::cmp::Ordering; -use shift_glyph_codec::OutlineCommand; - use crate::{ - curve::curves_from_commands, Bounds, Curve, CurveIndexEncoding, Layout, PackedAtlas, SlugError, + curve::curves_from_commands, Bounds, Curve, CurveIndexEncoding, Layout, OutlineCommand, + PackedAtlas, SlugError, }; pub const DEFAULT_BAND_COUNT: u32 = 8; diff --git a/crates/shift-slug/src/curve.rs b/crates/shift-slug/src/curve.rs index 64ca4f1a..5aea591c 100644 --- a/crates/shift-slug/src/curve.rs +++ b/crates/shift-slug/src/curve.rs @@ -1,6 +1,4 @@ -use shift_glyph_codec::OutlineCommand; - -use crate::SlugError; +use crate::{OutlineCommand, SlugError}; /// Maximum font-space distance between a source cubic and its quadratic approximation. pub const CUBIC_APPROXIMATION_ACCURACY: f32 = 1.0; diff --git a/crates/shift-slug/src/lib.rs b/crates/shift-slug/src/lib.rs index 3a808647..1b4ad59c 100644 --- a/crates/shift-slug/src/lib.rs +++ b/crates/shift-slug/src/lib.rs @@ -9,6 +9,7 @@ mod atlas; mod authored; mod curve; mod error; +mod outline; mod pack; mod render; mod resident; @@ -24,6 +25,7 @@ pub use authored::{ }; pub use curve::{Bounds, Curve, Point, CUBIC_APPROXIMATION_ACCURACY, LINE_EPSILON}; pub use error::SlugError; +pub use outline::OutlineCommand; pub use pack::{CurveIndexEncoding, Layout, PackedAtlas, Section}; pub use render::{ pack_render_instances, pack_render_params, RenderInstance, RenderParams, RENDER_INSTANCE_BYTES, diff --git a/crates/shift-slug/src/outline.rs b/crates/shift-slug/src/outline.rs new file mode 100644 index 00000000..e96b9db8 --- /dev/null +++ b/crates/shift-slug/src/outline.rs @@ -0,0 +1,27 @@ +/// One drawing command consumed by Slug preprocessing. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum OutlineCommand { + Move { + x: T, + y: T, + }, + Line { + x: T, + y: T, + }, + Quad { + cx: T, + cy: T, + x: T, + y: T, + }, + Cubic { + c1x: T, + c1y: T, + c2x: T, + c2y: T, + x: T, + y: T, + }, + Close, +} diff --git a/crates/shift-slug/src/resident.rs b/crates/shift-slug/src/resident.rs index 624c5523..5590fba5 100644 --- a/crates/shift-slug/src/resident.rs +++ b/crates/shift-slug/src/resident.rs @@ -64,8 +64,8 @@ pub fn build_authored_atlas( /// Compiles an ordered root-glyph batch and its transitive component geometry. /// /// Root identities are deduplicated in caller order. Interpolation bases are -/// collected only from those roots and their component closures, so viewport -/// work does not scan or compile unrelated glyphs. +/// collected only from those roots and their component closures, so a local +/// edit patch does not scan or compile unrelated glyphs. pub fn build_authored_atlas_page( font: &Font, glyph_ids: &[GlyphId], diff --git a/crates/shift-slug/src/variable.rs b/crates/shift-slug/src/variable.rs index 3f87e211..dda18abc 100644 --- a/crates/shift-slug/src/variable.rs +++ b/crates/shift-slug/src/variable.rs @@ -1,11 +1,9 @@ -use shift_glyph_codec::OutlineCommand; - use crate::{ curve::{ cubic_subdivision_counts_from_commands, curves_and_line_flags_from_commands_with_subdivisions, }, - Bounds, Curve, Point, Section, SlugError, LINE_EPSILON, + Bounds, Curve, OutlineCommand, Point, Section, SlugError, LINE_EPSILON, }; const CURVE_BYTES: usize = 24; diff --git a/crates/shift-slug/tests/atlas.rs b/crates/shift-slug/tests/atlas.rs index 342f2736..4cda6130 100644 --- a/crates/shift-slug/tests/atlas.rs +++ b/crates/shift-slug/tests/atlas.rs @@ -1,4 +1,4 @@ -use shift_glyph_codec::OutlineCommand; +use shift_slug::OutlineCommand; #[cfg(feature = "wgpu-benchmark")] use shift_slug::SLUG_WGSL; use shift_slug::{ diff --git a/crates/shift-slug/tests/variable_model.rs b/crates/shift-slug/tests/variable_model.rs index 588f0eb3..2d7155a3 100644 --- a/crates/shift-slug/tests/variable_model.rs +++ b/crates/shift-slug/tests/variable_model.rs @@ -1,8 +1,7 @@ use shift_font::{ test_support::sample_variable_font, CurveSegment, CurveSegmentIter, GlyphId, Location, }; -use shift_glyph_codec::OutlineCommand; -use shift_slug::{AtlasBuilder, VariableAtlasBuilder}; +use shift_slug::{AtlasBuilder, OutlineCommand, VariableAtlasBuilder}; #[test] fn resident_two_source_model_matches_shift_projection_at_midpoint() { diff --git a/crates/shift-source/src/package.rs b/crates/shift-source/src/package.rs index 97734958..b5978ca6 100644 --- a/crates/shift-source/src/package.rs +++ b/crates/shift-source/src/package.rs @@ -1605,7 +1605,7 @@ impl TryFrom for LibValue { values .into_iter() .map(|(key, value)| value.try_into().map(|value| (key, value))) - .collect::, _>>()?, + .collect::, _>>()?, ), LibValueDoc::Data(value) => Self::Data(value), LibValueDoc::Date(value) => Self::Date(value), diff --git a/crates/shift-store/Cargo.toml b/crates/shift-store/Cargo.toml index 08da32a1..f964fa81 100644 --- a/crates/shift-store/Cargo.toml +++ b/crates/shift-store/Cargo.toml @@ -5,9 +5,14 @@ edition = "2024" [dependencies] shift-font = { workspace = true } +blake3 = "1.8" rusqlite = { version = "0.37.0", features = ["blob", "backup", "limits"] } +rayon = "1.10" +rmp-serde = "1.3" +serde = "1.0" serde_json = "1.0" thiserror = "2" +zstd = "0.13" [dev-dependencies] shift-font = { workspace = true, features = ["test-support"] } diff --git a/crates/shift-store/README.md b/crates/shift-store/README.md index 504bcea9..584b94f8 100644 --- a/crates/shift-store/README.md +++ b/crates/shift-store/README.md @@ -1,40 +1,67 @@ # shift-store -`shift-store` owns Shift's durable source database. +`shift-store` owns Shift's durable SQLite working database. -The crate provides the storage boundary between the rest of the Rust application and SQLite. Callers should use typed store APIs exposed by this crate instead of preparing SQL statements or opening database connections directly. +Callers use typed APIs from this crate rather than preparing SQL or opening a second application-level persistence path. -## Responsibilities +## Storage boundary + +- Font metadata and the glyph/layer directory remain relational and load eagerly. Layer rows reference their glyph rather than duplicating its editable name. +- Each editable glyph layer is one independently addressable, store-private MessagePack payload identified as `shift.glyph-layer.v1`; the store independently wraps it as `none` or `zstd.v1` without changing authored semantics. +- Component dependency rows contain only the earned query index: component ID, owner layer ID, base glyph ID, and ordinal. +- Points, contours, anchors, transforms, guidelines, and layer lib values are canonical only inside the layer BLOB; they are not duplicated as normalized SQL rows. +- Payload replacement, directory facts, component rows, and workspace revision state commit in one transaction. + +`ShiftStore::load_font_directory` never reads payload BLOBs. `load_glyph_layer` bounds stored and decoded lengths before fetching and decompressing one payload. `load_glyph_layers` accepts a complete requested set, deduplicates it, and partitions it through the single count- and decoded-byte-aware planner. Each internal batch performs three ordered directory/payload/reference scans for at most 512 layers and 256 MiB decoded bytes, then decompresses and validates with Rayon. Every decoded payload must match its exact declared length and 32-byte BLAKE3 before strict MessagePack decoding. Full-font materialization and workspace acquisition use that same planner; no caller maintains a weaker count-only loop. All payload paths cross-check canonical bytes against relational facts. + +## Outer payload contract + +`glyph_layer_payloads` stores `inner_format`, `compression`, `stored_byte_length`, +`decoded_byte_length`, `decoded_blake3`, and `payload`. `inner_format` remains +`shift.glyph-layer.v1`. Its bytes are `rmp-serde`'s compact serialization of `GlyphLayer`; struct field order and enum Serde attributes are format-bearing and require a new identifier if changed. Compression is either `none` or `zstd.v1`; `zstd.v1` means one complete independent Zstandard frame with no dictionary. Level 1 is the current write policy, not a decoding +requirement. Writers retain compressed bytes only when they are smaller than the canonical input. + +`decoded_blake3` is the 32-byte BLAKE3 of the exact canonical bytes. Reads bound both lengths before +fetch/allocation, reject trailing Zstandard data, require the exact decoded length and hash, and +only then invoke strict MessagePack decoding. A layer replacement may move between `none` and +`zstd.v1` without changing its inner bytes or touching another layer. -- open and configure the SQLite connection; -- create and update the database schema; -- expose typed Rust APIs for source-store reads and writes; -- keep raw SQL local to this crate; -- preserve stable internal IDs for source entities; -- separate canonical source data from derived/index data as the store grows. +## Import boundary -## Design Goals +`FontImportWriter` accepts bounded replacement glyph batches without requiring a complete in-memory `Font`. `encode_glyph_batch` MessagePack-encodes, BLAKE3-hashes, and independently compresses an owned batch with Rayon without borrowing the SQLite transaction, allowing the workspace to overlap parsing, encoding/compression, and one stable-order SQLite writer. Streaming inserts, full-state replacement, and change-set replacement share one write implementation parameterized only by insert/upsert mode. Change sets supplied with a committed post-edit font skip incremental decode/re-encode for touched existing layers and write each final layer once. Secondary query indexes are dropped inside the stream transaction and rebuilt in bulk by `finish`; dropping an unfinished writer restores them with the rest of the rollback. Prefix-redundant Unicode-glyph and component-layer indexes are omitted. `finish` is the only stream commit point. -- incremental writes; -- durability and crash-safe recovery; -- stable IDs; -- performant reads and writes for large fonts; -- clear handling of CJK-scale glyph inventories; -- efficient queries for references to and from source entities; -- support for components, kerning, and feature-related source data; -- clear separation between canonical source data and derived/index data; -- import/export support through typed store APIs rather than ad hoc SQL access. +`ShiftStore::open_for_import` uses rollback-capable in-memory journaling and disabled synchronous writes only while the foreign source remains authoritative and the path is disposable. `finish_import` syncs the completed database and restores WAL + NORMAL. Normal edits never use import pragmas. `FontWorkspace` opens imports at a sibling staging path, writes workspace state there, makes the staged database durable, closes it, removes the closed staging WAL/SHM sidecars, atomically installs the main database at the requested destination, and syncs the parent directory. A failure before installation leaves the previous destination untouched. -## Shape +## Schema policy + +Shift has not shipped this working-store schema. Schema changes therefore update the version-1 baseline directly; there is intentionally no compatibility migration for earlier development databases. + +## Responsibilities + +- configure SQLite for WAL-backed transactional work and a separately finalized disposable-import mode; +- own the baseline schema and raw SQL; +- preserve stable authored IDs and exact values; +- provide directory-first and bounded canonical payload APIs rather than row-level outline/component compatibility models; +- keep canonical payloads and relational query indexes atomic; +- support CJK-scale directories without BLOB scans. + +## Main modules ```text src/ - connection.rs # opening and configuring SQLite - error.rs # store error type - glyph.rs # glyph persistence API - schema.rs # schema creation and versioning entry point - store.rs # ShiftStore connection owner - types.rs # storage-facing IDs and small value types + connection.rs # normal WAL opening plus disposable import/finalization posture + schema.rs # pre-release version-1 baseline + font_state.rs # eager metadata/directory and explicit full materialization + import_writer.rs # pipelined Rayon encode/compress plus one SQLite transaction owner + layer/ + format.rs # strict MessagePack encoding and domain validation + payload.rs # independent Zstd/BLAKE3 storage representation + directory.rs # relational metadata and row cross-validation + load.rs # bounded single and batch BLOB reads + write.rs # transactional payload and reference-index writes + references.rs # component dependency queries and index validation + tests.rs # integrated SQLite layer behavior + write_mode.rs # shared insert/upsert policy for canonical write paths + change_set.rs # transactional workspace changes and one final touched-layer write + workspace_state.rs ``` - -Tests live in `tests/store_test.rs` and should start as small persistence checks before broader integration tests are added. diff --git a/crates/shift-store/src/change_set.rs b/crates/shift-store/src/change_set.rs index f7ac53bb..360288cf 100644 --- a/crates/shift-store/src/change_set.rs +++ b/crates/shift-store/src/change_set.rs @@ -1,16 +1,54 @@ +use std::collections::HashSet; + use rusqlite::{Transaction, params}; use shift_font as font; use crate::{ - ShiftStore, StoreError, source::SourceKind, workspace_state::mark_workspace_dirty_in_tx, + ShiftStore, StoreError, + layer::{create_empty_layer_in_tx, rewrite_layer_in_tx, write_layer_in_tx}, + source::SourceKind, + workspace_state::mark_workspace_dirty_in_tx, + write_mode::WriteMode, }; impl ShiftStore { pub fn apply_change_set(&mut self, change_set: &font::FontChangeSet) -> Result<(), StoreError> { + self.apply_change_set_inner(change_set, None) + } + + /// Applies relational changes and takes touched layer payloads from the + /// committed post-edit font. This preserves complete clone/materialize + /// results even when a compact change record only carries directory facts. + pub fn apply_change_set_with_font( + &mut self, + change_set: &font::FontChangeSet, + post_font: &font::Font, + ) -> Result<(), StoreError> { + self.apply_change_set_inner(change_set, Some(post_font)) + } + + fn apply_change_set_inner( + &mut self, + change_set: &font::FontChangeSet, + post_font: Option<&font::Font>, + ) -> Result<(), StoreError> { let tx = self.conn.transaction()?; + let mut touched_layer_ids = HashSet::new(); for change in &change_set.changes { - apply_change(&tx, change)?; + if post_font.is_none() || !post_font_supersedes_incremental_layer_write(change) { + apply_change(&tx, change)?; + } + if let Some(layer_id) = change.layer_id() { + touched_layer_ids.insert(layer_id.clone()); + } + } + if let Some(post_font) = post_font { + for layer_id in touched_layer_ids { + if let Some(layer) = post_font.layer(layer_id) { + rewrite_layer_in_tx(&tx, layer)?; + } + } } mark_workspace_dirty_in_tx(&tx)?; @@ -20,117 +58,163 @@ impl ShiftStore { pub fn replace_font_state(&mut self, font: &font::Font) -> Result<(), StoreError> { let tx = self.conn.transaction()?; + replace_font_header_in_tx(&tx, font)?; + + for (order_index, glyph) in font.glyphs().enumerate() { + write_glyph_in_tx(&tx, glyph, order_index as i64)?; + } + + tx.commit()?; + Ok(()) + } +} + +pub(crate) fn replace_font_header_in_tx( + tx: &Transaction<'_>, + font: &font::Font, +) -> Result<(), StoreError> { + tx.execute("DELETE FROM glyph_lib", [])?; + tx.execute("DELETE FROM font_lib", [])?; + tx.execute("DELETE FROM fontinfo_remainder", [])?; + tx.execute("DELETE FROM font_binaries", [])?; + tx.execute("DELETE FROM kerning_pairs", [])?; + tx.execute("DELETE FROM kerning_group_members", [])?; + tx.execute("DELETE FROM kerning_groups", [])?; + tx.execute("DELETE FROM feature_text", [])?; + tx.execute("DELETE FROM font_guidelines", [])?; + tx.execute("DELETE FROM glyph_components", [])?; + tx.execute("DELETE FROM glyph_layer_payloads", [])?; + tx.execute("DELETE FROM glyph_layers", [])?; + tx.execute("DELETE FROM glyph_unicodes", [])?; + tx.execute("DELETE FROM glyphs", [])?; + tx.execute("DELETE FROM source_locations", [])?; + tx.execute("DELETE FROM source_metric_values", [])?; + tx.execute("DELETE FROM source_lib", [])?; + tx.execute("DELETE FROM sources", [])?; + tx.execute("DELETE FROM metric_definitions", [])?; + tx.execute("DELETE FROM axis_mappings", [])?; + tx.execute("DELETE FROM named_instances", [])?; + tx.execute("DELETE FROM axes", [])?; + + upsert_font_info(tx, font)?; + replace_feature_text(tx, font.features().fea_source())?; + replace_font_guidelines(tx, font.guidelines())?; + replace_lib_data(tx, "font_lib", "key", None, font.lib())?; + replace_lib_data( + tx, + "fontinfo_remainder", + "key", + None, + font.fontinfo_remainder(), + )?; + replace_font_binaries(tx, "data", font.data_files())?; + replace_font_binaries(tx, "image", font.images())?; + replace_kerning(tx, font.kerning())?; - tx.execute("DELETE FROM glyph_layer_lib", [])?; - tx.execute("DELETE FROM glyph_lib", [])?; - tx.execute("DELETE FROM font_lib", [])?; - tx.execute("DELETE FROM fontinfo_remainder", [])?; - tx.execute("DELETE FROM font_binaries", [])?; - tx.execute("DELETE FROM kerning_pairs", [])?; - tx.execute("DELETE FROM kerning_group_members", [])?; - tx.execute("DELETE FROM kerning_groups", [])?; - tx.execute("DELETE FROM feature_text", [])?; - tx.execute("DELETE FROM glyph_layer_guidelines", [])?; - tx.execute("DELETE FROM font_guidelines", [])?; - tx.execute("DELETE FROM glyph_layer_points", [])?; - tx.execute("DELETE FROM glyph_layer_contours", [])?; - tx.execute("DELETE FROM glyph_layer_anchors", [])?; - tx.execute("DELETE FROM glyph_components", [])?; - tx.execute("DELETE FROM glyph_layers", [])?; - tx.execute("DELETE FROM glyph_unicodes", [])?; - tx.execute("DELETE FROM glyphs", [])?; - tx.execute("DELETE FROM source_locations", [])?; - tx.execute("DELETE FROM source_metric_values", [])?; - tx.execute("DELETE FROM source_lib", [])?; - tx.execute("DELETE FROM sources", [])?; - tx.execute("DELETE FROM metric_definitions", [])?; - tx.execute("DELETE FROM axis_mappings", [])?; - tx.execute("DELETE FROM named_instances", [])?; - tx.execute("DELETE FROM axes", [])?; + for (order_index, axis) in font.axes().iter().enumerate() { + insert_axis(tx, axis, order_index as i64, false)?; + } + replace_axis_mappings(tx, font.axis_mappings())?; + replace_named_instances(tx, font.named_instances())?; + replace_metric_definitions(tx, font.metric_definitions())?; - upsert_font_info(&tx, font)?; - replace_feature_text(&tx, font.features().fea_source())?; - replace_font_guidelines(&tx, font.guidelines())?; - replace_lib_data(&tx, "font_lib", "key", None, font.lib())?; + for (order_index, source) in font.sources().iter().enumerate() { + upsert_source( + tx, + &source.id(), + SourceRow { + name: Some(source.name()), + filename: source.filename(), + color: source.color(), + kind: SourceKind::from(source.role()), + layer_name: source.layer_name(), + italic_angle: source.italic_angle(), + line_gap: source.line_gap(), + underline_position: source.underline_position(), + underline_thickness: source.underline_thickness(), + order_index: order_index as i64, + }, + )?; + replace_source_metric_values(tx, source.id(), source.metric_values().iter())?; replace_lib_data( - &tx, - "fontinfo_remainder", - "key", - None, - font.fontinfo_remainder(), + tx, + "source_lib", + "source_id", + Some(&source.id().to_string()), + source.lib(), )?; - replace_font_binaries(&tx, "data", font.data_files())?; - replace_font_binaries(&tx, "image", font.images())?; - replace_kerning(&tx, font.kerning())?; - for (order_index, axis) in font.axes().iter().enumerate() { - insert_axis(&tx, axis, order_index as i64, false)?; + for (axis_id, value) in source.location().iter() { + // Location entries on undefined axes have no row to reference. + if font.axes().iter().any(|axis| axis.id() == *axis_id) { + upsert_source_location(tx, &source.id(), axis_id, *value)?; + } } - replace_axis_mappings(&tx, font.axis_mappings())?; - replace_named_instances(&tx, font.named_instances())?; - replace_metric_definitions(&tx, font.metric_definitions())?; + } - for (order_index, source) in font.sources().iter().enumerate() { - upsert_source( - &tx, - &source.id(), - SourceRow { - name: Some(source.name()), - filename: source.filename(), - color: source.color(), - kind: SourceKind::from(source.role()), - layer_name: source.layer_name(), - italic_angle: source.italic_angle(), - line_gap: source.line_gap(), - underline_position: source.underline_position(), - underline_thickness: source.underline_thickness(), - order_index: order_index as i64, - }, - )?; - replace_source_metric_values(&tx, source.id(), source.metric_values().iter())?; - replace_lib_data( - &tx, - "source_lib", - "source_id", - Some(&source.id().to_string()), - source.lib(), - )?; + Ok(()) +} - for (axis_id, value) in source.location().iter() { - // Location entries on undefined axes have no row to reference. - if font.axes().iter().any(|axis| axis.id() == *axis_id) { - upsert_source_location(&tx, &source.id(), axis_id, *value)?; - } +pub(crate) fn write_glyph_in_tx( + tx: &Transaction<'_>, + glyph: &font::Glyph, + order_index: i64, +) -> Result<(), StoreError> { + write_glyph_directory_in_tx(tx, glyph, order_index, WriteMode::Upsert)?; + + for layer in glyph.layers().values().map(|layer| layer.as_ref()) { + write_layer_in_tx(tx, &glyph.id(), layer)?; + } + Ok(()) +} + +pub(crate) fn write_glyph_directory_in_tx( + tx: &Transaction<'_>, + glyph: &font::Glyph, + order_index: i64, + mode: WriteMode, +) -> Result<(), StoreError> { + match mode { + WriteMode::Insert => { + tx.prepare_cached("INSERT INTO glyphs (id, name, order_index) VALUES (?1, ?2, ?3)")? + .execute(params![ + glyph.id().to_string(), + glyph.glyph_name().as_str(), + order_index + ])?; + for (unicode_order, unicode) in glyph.unicodes().iter().enumerate() { + tx.prepare_cached( + "INSERT INTO glyph_unicodes (glyph_id, unicode, order_index) VALUES (?1, ?2, ?3)", + )? + .execute(params![ + glyph.id().to_string(), + i64::from(*unicode), + unicode_order as i64 + ])?; } + for (key, value) in glyph.lib().iter() { + tx.prepare_cached( + "INSERT INTO glyph_lib (glyph_id, key, value_json) VALUES (?1, ?2, ?3)", + )? + .execute(params![ + glyph.id().to_string(), + key, + lib_value_json(value)? + ])?; + } + Ok(()) } - - for (order_index, glyph) in font.glyphs().enumerate() { - upsert_glyph(&tx, &glyph.id(), glyph.glyph_name(), order_index as i64)?; - replace_glyph_unicodes(&tx, &glyph.id(), glyph.unicodes())?; + WriteMode::Upsert => { + upsert_glyph(tx, &glyph.id(), glyph.glyph_name(), order_index)?; + replace_glyph_unicodes(tx, &glyph.id(), glyph.unicodes())?; replace_lib_data( - &tx, + tx, "glyph_lib", "glyph_id", Some(&glyph.id().to_string()), glyph.lib(), - )?; - - for layer in glyph.layers().values().map(|layer| layer.as_ref()) { - upsert_layer( - &tx, - &layer.id(), - &glyph.id(), - &layer.source_id(), - Some(glyph.glyph_name()), - layer.width(), - layer.height(), - )?; - replace_full_layer_state(&tx, layer)?; - } + ) } - - tx.commit()?; - Ok(()) } } @@ -251,15 +335,18 @@ fn apply_change(tx: &Transaction<'_>, change: &font::FontChange) -> Result<(), S Ok(()) } font::FontChange::GlyphIdentityChanged(change) => { - upsert_glyph(tx, &change.glyph_id, &change.to_name, 0)?; + let rows_changed = tx.execute( + "UPDATE glyphs SET name = ?1 WHERE id = ?2", + params![change.to_name.as_str(), change.glyph_id.to_string()], + )?; + require_changed(rows_changed, "glyph", change.glyph_id.to_string())?; replace_glyph_unicodes(tx, &change.glyph_id, &change.to_unicodes) } - font::FontChange::GlyphLayerCreated(change) => upsert_layer( + font::FontChange::GlyphLayerCreated(change) => create_empty_layer_in_tx( tx, - &change.layer_id, &change.glyph_id, - &change.source_id, - change.name.as_ref(), + change.layer_id.clone(), + change.source_id.clone(), change.width, change.height, ), @@ -272,92 +359,167 @@ fn apply_change(tx: &Transaction<'_>, change: &font::FontChange) -> Result<(), S Ok(()) } font::FontChange::LayerMetricsChanged(change) => { - let rows_changed = tx.execute( - " - UPDATE glyph_layers - SET width = ?2, height = ?3 - WHERE id = ?1 - ", - params![layer_row_id(&change.layer_id), change.width, change.height,], - )?; - require_changed(rows_changed, "glyph layer", layer_row_id(&change.layer_id))?; - Ok(()) + update_packed_layer(tx, &change.layer_id, |layer| { + layer.set_width(change.width); + layer.set_height(change.height); + Ok(()) + }) } font::FontChange::ContourAdded(change) => { - require_layer_exists(tx, &change.layer_id)?; - replace_contour(tx, &change.layer_id, &change.contour) + update_packed_layer(tx, &change.layer_id, |layer| { + layer.add_contour(contour_from_value(&change.contour)); + Ok(()) + }) } font::FontChange::ContourOpenClosedChanged(change) => { - require_layer_exists(tx, &change.layer_id)?; - let rows_changed = tx.execute( - " - UPDATE glyph_layer_contours - SET closed = ?2 - WHERE id = ?1 - ", - params![change.contour_id.to_string(), change.closed], - )?; - require_changed(rows_changed, "contour", change.contour_id.to_string())?; - Ok(()) + update_packed_layer(tx, &change.layer_id, |layer| { + let contour = layer + .contour_mut(change.contour_id.clone()) + .ok_or_else(|| StoreError::MissingEntity { + kind: "contour", + id: change.contour_id.to_string(), + })?; + if change.closed { + contour.close(); + } else { + contour.open(); + } + Ok(()) + }) } font::FontChange::PointsAdded(change) => { - require_layer_exists(tx, &change.layer_id)?; - replace_contour(tx, &change.layer_id, &change.contour) + update_packed_layer(tx, &change.layer_id, |layer| { + layer.add_contour(contour_from_value(&change.contour)); + Ok(()) + }) } font::FontChange::PointsDeleted(change) => { - require_layer_exists(tx, &change.layer_id)?; - replace_contour(tx, &change.layer_id, &change.contour) + update_packed_layer(tx, &change.layer_id, |layer| { + layer.add_contour(contour_from_value(&change.contour)); + Ok(()) + }) } font::FontChange::PointSmoothChanged(change) => { - require_layer_exists(tx, &change.layer_id)?; - let rows_changed = tx.execute( - " - UPDATE glyph_layer_points - SET smooth = ?2 - WHERE id = ?1 - ", - params![change.point_id.to_string(), change.smooth], - )?; - require_changed(rows_changed, "point", change.point_id.to_string())?; - Ok(()) + update_packed_layer(tx, &change.layer_id, |layer| { + let point = layer + .contours_iter_mut() + .flat_map(|contour| contour.points_mut()) + .find(|point| point.id() == change.point_id) + .ok_or_else(|| StoreError::MissingEntity { + kind: "point", + id: change.point_id.to_string(), + })?; + point.set_smooth(change.smooth); + Ok(()) + }) } font::FontChange::PointPositionsChanged(change) => { - require_layer_exists(tx, &change.layer_id)?; - for point in &change.points { - let rows_changed = tx.execute( - " - UPDATE glyph_layer_points - SET x = ?2, y = ?3 - WHERE id = ?1 - ", - params![point.point_id.to_string(), point.x, point.y], - )?; - require_changed(rows_changed, "point", point.point_id.to_string())?; - } - Ok(()) + update_packed_layer(tx, &change.layer_id, |layer| { + for position in &change.points { + let point = layer + .contours_iter_mut() + .flat_map(|contour| contour.points_mut()) + .find(|point| point.id() == position.point_id) + .ok_or_else(|| StoreError::MissingEntity { + kind: "point", + id: position.point_id.to_string(), + })?; + point.set_position(position.x, position.y); + } + Ok(()) + }) } font::FontChange::AnchorPositionsChanged(change) => { - require_layer_exists(tx, &change.layer_id)?; - for anchor in &change.anchors { - let rows_changed = tx.execute( - " - UPDATE glyph_layer_anchors - SET x = ?2, y = ?3 - WHERE id = ?1 - ", - params![anchor.anchor_id.to_string(), anchor.x, anchor.y], - )?; - require_changed(rows_changed, "anchor", anchor.anchor_id.to_string())?; - } - Ok(()) + update_packed_layer(tx, &change.layer_id, |layer| { + for position in &change.anchors { + if !layer.set_anchor_position( + position.anchor_id.clone(), + position.x, + position.y, + ) { + return Err(StoreError::MissingEntity { + kind: "anchor", + id: position.anchor_id.to_string(), + }); + } + } + Ok(()) + }) } font::FontChange::LayerGeometryReplaced(change) => { - require_layer_exists(tx, &change.layer_id)?; - replace_layer_geometry(tx, &change.layer_id, &change.layer) + update_packed_layer(tx, &change.layer_id, |layer| { + layer.set_width(change.layer.width); + layer.set_height(change.layer.height); + layer.clear_contours(); + for contour in &change.layer.contours { + layer.add_contour(contour_from_value(contour)); + } + layer.clear_anchors(); + let mut anchors = change.layer.anchors.iter().collect::>(); + anchors.sort_by_key(|anchor| anchor.order_index); + for anchor in anchors { + layer.add_anchor(font::Anchor::with_id( + anchor.id.clone(), + anchor.name.clone(), + anchor.x, + anchor.y, + )); + } + Ok(()) + }) } } } +fn post_font_supersedes_incremental_layer_write(change: &font::FontChange) -> bool { + matches!( + change, + font::FontChange::LayerMetricsChanged(_) + | font::FontChange::ContourAdded(_) + | font::FontChange::ContourOpenClosedChanged(_) + | font::FontChange::PointsAdded(_) + | font::FontChange::PointsDeleted(_) + | font::FontChange::PointSmoothChanged(_) + | font::FontChange::PointPositionsChanged(_) + | font::FontChange::AnchorPositionsChanged(_) + | font::FontChange::LayerGeometryReplaced(_) + ) +} + +fn update_packed_layer( + tx: &Transaction<'_>, + layer_id: &font::LayerId, + update: impl FnOnce(&mut font::GlyphLayer) -> Result<(), StoreError>, +) -> Result<(), StoreError> { + let mut layer = crate::layer::load_glyph_layer_from_conn(tx, layer_id)?.ok_or_else(|| { + StoreError::MissingEntity { + kind: "glyph layer", + id: layer_id.to_string(), + } + })?; + update(&mut layer)?; + rewrite_layer_in_tx(tx, &layer) +} + +fn contour_from_value(value: &font::ContourValue) -> font::Contour { + let mut contour = font::Contour::with_id(value.id.clone()); + let mut points = value.points.iter().collect::>(); + points.sort_by_key(|point| point.order_index); + for point in points { + contour.push_point(font::Point::new( + point.id.clone(), + point.x, + point.y, + point.point_type, + point.smooth, + )); + } + if value.closed { + contour.close(); + } + contour +} + fn upsert_axis_with_order( tx: &Transaction<'_>, axis: &font::Axis, @@ -575,7 +737,7 @@ fn upsert_glyph( name: &font::GlyphName, order_index: i64, ) -> Result<(), StoreError> { - tx.execute( + tx.prepare_cached( " INSERT INTO glyphs (id, name, order_index) VALUES (?1, ?2, ?3) @@ -583,8 +745,8 @@ fn upsert_glyph( name = excluded.name, order_index = excluded.order_index ", - params![glyph_id.to_string(), name.as_str(), order_index], - )?; + )? + .execute(params![glyph_id.to_string(), name.as_str(), order_index])?; Ok(()) } @@ -593,122 +755,23 @@ fn replace_glyph_unicodes( glyph_id: &font::GlyphId, unicodes: &[u32], ) -> Result<(), StoreError> { - tx.execute( - "DELETE FROM glyph_unicodes WHERE glyph_id = ?1", - [glyph_id.to_string()], - )?; + tx.prepare_cached("DELETE FROM glyph_unicodes WHERE glyph_id = ?1")? + .execute([glyph_id.to_string()])?; for (order_index, unicode) in unicodes.iter().enumerate() { - tx.execute( + tx.prepare_cached( " INSERT INTO glyph_unicodes (glyph_id, unicode, order_index) VALUES (?1, ?2, ?3) ", - params![glyph_id.to_string(), *unicode as i64, order_index as i64], - )?; - } - - Ok(()) -} - -fn upsert_layer( - tx: &Transaction<'_>, - layer_id: &font::LayerId, - glyph_id: &font::GlyphId, - source_id: &font::SourceId, - name: Option<&font::GlyphName>, - width: f64, - height: Option, -) -> Result<(), StoreError> { - tx.execute( - " - INSERT INTO glyph_layers (id, glyph_id, source_id, name, width, height) - VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ON CONFLICT(id) DO UPDATE SET - glyph_id = excluded.glyph_id, - source_id = excluded.source_id, - name = excluded.name - ", - params![ - layer_row_id(layer_id), + )? + .execute(params![ glyph_id.to_string(), - source_id.to_string(), - name.map(font::GlyphName::as_str), - width, - height, - ], - )?; - Ok(()) -} - -fn replace_layer_geometry( - tx: &Transaction<'_>, - layer_id: &font::LayerId, - layer: &font::GlyphLayerValue, -) -> Result<(), StoreError> { - let rows_changed = tx.execute( - " - UPDATE glyph_layers - SET width = ?2, height = ?3 - WHERE id = ?1 - ", - params![layer_row_id(layer_id), layer.width, layer.height], - )?; - require_changed(rows_changed, "glyph layer", layer_row_id(layer_id))?; - tx.execute( - " - DELETE FROM glyph_layer_contours - WHERE layer_id = ?1 - ", - [layer_row_id(layer_id)], - )?; - - for (order_index, contour) in layer.contours.iter().enumerate() { - insert_contour(tx, layer_id, order_index, contour)?; - } - - tx.execute( - " - DELETE FROM glyph_layer_anchors - WHERE layer_id = ?1 - ", - [layer_row_id(layer_id)], - )?; - - for anchor in &layer.anchors { - insert_anchor(tx, layer_id, anchor)?; - } - - Ok(()) -} - -fn replace_full_layer_state( - tx: &Transaction<'_>, - layer: &font::GlyphLayer, -) -> Result<(), StoreError> { - replace_layer_geometry(tx, &layer.id(), &font::GlyphLayerValue::from(layer))?; - - tx.execute( - " - DELETE FROM glyph_components - WHERE layer_id = ?1 - ", - [layer_row_id(&layer.id())], - )?; - - for (order_index, component) in layer.components_iter().enumerate() { - insert_component(tx, &layer.id(), component, order_index)?; + *unicode as i64, + order_index as i64 + ])?; } - replace_layer_guidelines(tx, &layer.id(), layer.guidelines())?; - replace_lib_data( - tx, - "glyph_layer_lib", - "layer_id", - Some(&layer.id().to_string()), - layer.lib(), - )?; - Ok(()) } @@ -846,135 +909,26 @@ fn replace_font_guidelines( ) -> Result<(), StoreError> { tx.execute("DELETE FROM font_guidelines", [])?; for (order_index, guideline) in guidelines.iter().enumerate() { - insert_guideline(tx, "font_guidelines", None, guideline, order_index)?; - } - Ok(()) -} - -fn replace_layer_guidelines( - tx: &Transaction<'_>, - layer_id: &font::LayerId, - guidelines: &[font::Guideline], -) -> Result<(), StoreError> { - tx.execute( - " - DELETE FROM glyph_layer_guidelines - WHERE layer_id = ?1 - ", - [layer_row_id(layer_id)], - )?; - for (order_index, guideline) in guidelines.iter().enumerate() { - insert_guideline( - tx, - "glyph_layer_guidelines", - Some(&layer_row_id(layer_id)), - guideline, - order_index, + tx.execute( + " + INSERT INTO font_guidelines ( + id, x, y, angle, name, color, order_index + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ", + params![ + guideline.id().to_string(), + guideline.x(), + guideline.y(), + guideline.angle(), + guideline.name(), + guideline.color(), + order_index as i64, + ], )?; } Ok(()) } - -fn insert_guideline( - tx: &Transaction<'_>, - table: &'static str, - layer_id: Option<&str>, - guideline: &font::Guideline, - order_index: usize, -) -> Result<(), StoreError> { - match layer_id { - Some(layer_id) => { - debug_assert_eq!(table, "glyph_layer_guidelines"); - tx.execute( - " - INSERT INTO glyph_layer_guidelines ( - id, layer_id, x, y, angle, name, color, order_index - ) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) - ", - params![ - guideline.id().to_string(), - layer_id, - guideline.x(), - guideline.y(), - guideline.angle(), - guideline.name(), - guideline.color(), - order_index as i64, - ], - )?; - } - None => { - debug_assert_eq!(table, "font_guidelines"); - tx.execute( - " - INSERT INTO font_guidelines ( - id, x, y, angle, name, color, order_index - ) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) - ", - params![ - guideline.id().to_string(), - guideline.x(), - guideline.y(), - guideline.angle(), - guideline.name(), - guideline.color(), - order_index as i64, - ], - )?; - } - } - Ok(()) -} - -fn insert_component( - tx: &Transaction<'_>, - layer_id: &font::LayerId, - component: &font::Component, - order_index: usize, -) -> Result<(), StoreError> { - let transform = component.transform(); - tx.execute( - " - INSERT INTO glyph_components ( - id, - layer_id, - base_glyph_id, - base_glyph_name, - translate_x, - translate_y, - rotation, - scale_x, - scale_y, - skew_x, - skew_y, - t_center_x, - t_center_y, - order_index - ) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14) - ", - params![ - component.id().to_string(), - layer_row_id(layer_id), - component.base_glyph_id().to_string(), - component.base_glyph_name().as_str(), - transform.translate_x, - transform.translate_y, - transform.rotation, - transform.scale_x, - transform.scale_y, - transform.skew_x, - transform.skew_y, - transform.t_center_x, - transform.t_center_y, - order_index as i64, - ], - )?; - Ok(()) -} - fn replace_kerning(tx: &Transaction<'_>, kerning: &font::KerningData) -> Result<(), StoreError> { tx.execute("DELETE FROM kerning_pairs", [])?; tx.execute("DELETE FROM kerning_group_members", [])?; @@ -1056,12 +1010,16 @@ fn replace_lib_data( match owner_id { Some(owner_id) => { let delete_sql = format!("DELETE FROM {table} WHERE {owner_column} = ?1"); - tx.execute(&delete_sql, [owner_id])?; + tx.prepare_cached(&delete_sql)?.execute([owner_id])?; let insert_sql = format!( "INSERT INTO {table} ({owner_column}, key, value_json) VALUES (?1, ?2, ?3)" ); for (key, value) in lib.iter() { - tx.execute(&insert_sql, params![owner_id, key, lib_value_json(value)?])?; + tx.prepare_cached(&insert_sql)?.execute(params![ + owner_id, + key, + lib_value_json(value)? + ])?; } } None => { @@ -1134,134 +1092,6 @@ fn typed_json(kind: &'static str, value: serde_json::Value) -> serde_json::Value }) } -fn replace_contour( - tx: &Transaction<'_>, - layer_id: &font::LayerId, - contour: &font::ContourValue, -) -> Result<(), StoreError> { - tx.execute( - " - INSERT INTO glyph_layer_contours (id, layer_id, closed, order_index) - VALUES (?1, ?2, ?3, COALESCE( - (SELECT order_index FROM glyph_layer_contours WHERE id = ?1), - (SELECT COUNT(*) FROM glyph_layer_contours WHERE layer_id = ?2) - )) - ON CONFLICT(id) DO UPDATE SET - closed = excluded.closed, - layer_id = excluded.layer_id - ", - params![ - contour.id.to_string(), - layer_row_id(layer_id), - contour.closed, - ], - )?; - tx.execute( - "DELETE FROM glyph_layer_points WHERE contour_id = ?1", - [contour.id.to_string()], - )?; - - for point in &contour.points { - insert_point(tx, &contour.id, point)?; - } - - Ok(()) -} - -fn insert_contour( - tx: &Transaction<'_>, - layer_id: &font::LayerId, - order_index: usize, - contour: &font::ContourValue, -) -> Result<(), StoreError> { - tx.execute( - " - INSERT INTO glyph_layer_contours (id, layer_id, closed, order_index) - VALUES (?1, ?2, ?3, ?4) - ", - params![ - contour.id.to_string(), - layer_row_id(layer_id), - contour.closed, - order_index as i64, - ], - )?; - - for point in &contour.points { - insert_point(tx, &contour.id, point)?; - } - - Ok(()) -} - -fn insert_point( - tx: &Transaction<'_>, - contour_id: &font::ContourId, - point: &font::PointValue, -) -> Result<(), StoreError> { - tx.execute( - " - INSERT INTO glyph_layer_points ( - id, - contour_id, - order_index, - x, - y, - point_type, - smooth - ) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) - ", - params![ - point.id.to_string(), - contour_id.to_string(), - point.order_index as i64, - point.x, - point.y, - point_type_name(point.point_type), - point.smooth, - ], - )?; - Ok(()) -} - -fn insert_anchor( - tx: &Transaction<'_>, - layer_id: &font::LayerId, - anchor: &font::AnchorValue, -) -> Result<(), StoreError> { - tx.execute( - " - INSERT INTO glyph_layer_anchors ( - id, - layer_id, - name, - x, - y, - order_index - ) - VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ", - params![ - anchor.id.to_string(), - layer_row_id(layer_id), - anchor.name, - anchor.x, - anchor.y, - anchor.order_index as i64, - ], - )?; - Ok(()) -} - -fn point_type_name(point_type: font::PointType) -> &'static str { - match point_type { - font::PointType::OnCurve => "onCurve", - font::PointType::OffCurve => "offCurve", - font::PointType::QCurve => "qCurve", - } -} - fn require_changed(rows_changed: usize, kind: &'static str, id: String) -> Result<(), StoreError> { if rows_changed == 0 { Err(StoreError::MissingEntity { kind, id }) @@ -1270,22 +1100,6 @@ fn require_changed(rows_changed: usize, kind: &'static str, id: String) -> Resul } } -fn require_layer_exists(tx: &Transaction<'_>, layer_id: &font::LayerId) -> Result<(), StoreError> { - let exists: bool = tx.query_row( - "SELECT EXISTS(SELECT 1 FROM glyph_layers WHERE id = ?1)", - [layer_row_id(layer_id)], - |row| row.get(0), - )?; - if exists { - Ok(()) - } else { - Err(StoreError::MissingEntity { - kind: "glyph layer", - id: layer_row_id(layer_id), - }) - } -} - fn layer_row_id(layer_id: &font::LayerId) -> String { layer_id.to_string() } diff --git a/crates/shift-store/src/component.rs b/crates/shift-store/src/component.rs deleted file mode 100644 index 81351cfb..00000000 --- a/crates/shift-store/src/component.rs +++ /dev/null @@ -1,153 +0,0 @@ -use crate::{ComponentId, GlyphId, LayerId, ShiftStore, StoreError}; - -pub struct NewGlyphComponent { - pub id: ComponentId, - pub layer_id: LayerId, - pub base_glyph_id: GlyphId, - pub base_glyph_name: String, - pub transform: shift_font::DecomposedTransform, - pub order_index: i64, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct GlyphComponentRecord { - pub id: ComponentId, - pub layer_id: LayerId, - pub base_glyph_id: GlyphId, - pub base_glyph_name: String, - pub transform: shift_font::DecomposedTransform, - pub order_index: i64, -} - -impl ShiftStore { - pub fn create_glyph_component( - &mut self, - component: NewGlyphComponent, - ) -> Result<(), StoreError> { - self.conn.execute( - " - INSERT INTO glyph_components ( - id, - layer_id, - base_glyph_id, - base_glyph_name, - translate_x, - translate_y, - rotation, - scale_x, - scale_y, - skew_x, - skew_y, - t_center_x, - t_center_y, - order_index - ) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14) - ", - rusqlite::params![ - component.id.as_str(), - component.layer_id.as_str(), - component.base_glyph_id.as_str(), - component.base_glyph_name, - component.transform.translate_x, - component.transform.translate_y, - component.transform.rotation, - component.transform.scale_x, - component.transform.scale_y, - component.transform.skew_x, - component.transform.skew_y, - component.transform.t_center_x, - component.transform.t_center_y, - component.order_index, - ], - )?; - - Ok(()) - } - - pub fn get_glyph_component( - &self, - id: &ComponentId, - ) -> Result, StoreError> { - let mut stmt = self.conn.prepare( - " - SELECT - id, - layer_id, - base_glyph_id, - base_glyph_name, - translate_x, - translate_y, - rotation, - scale_x, - scale_y, - skew_x, - skew_y, - t_center_x, - t_center_y, - order_index - FROM glyph_components - WHERE id = ?1 - ", - )?; - - match stmt.query_row([id.as_str()], map_glyph_component_row) { - Ok(component) => Ok(Some(component)), - Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), - Err(err) => Err(err.into()), - } - } - - pub fn list_glyph_components_for_layer( - &self, - layer_id: &LayerId, - ) -> Result, StoreError> { - let mut stmt = self.conn.prepare( - " - SELECT - id, - layer_id, - base_glyph_id, - base_glyph_name, - translate_x, - translate_y, - rotation, - scale_x, - scale_y, - skew_x, - skew_y, - t_center_x, - t_center_y, - order_index - FROM glyph_components - WHERE layer_id = ?1 - ORDER BY order_index, id - ", - )?; - - let rows = stmt.query_map([layer_id.as_str()], map_glyph_component_row)?; - rows.collect::, _>>() - .map_err(StoreError::from) - } -} - -fn map_glyph_component_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(GlyphComponentRecord { - id: ComponentId::new(row.get::<_, String>(0)?), - layer_id: LayerId::new(row.get::<_, String>(1)?), - base_glyph_id: GlyphId::new(row.get::<_, String>(2)?), - base_glyph_name: row.get(3)?, - transform: shift_font::DecomposedTransform { - translate_x: row.get(4)?, - translate_y: row.get(5)?, - rotation: row.get(6)?, - scale_x: row.get(7)?, - scale_y: row.get(8)?, - skew_x: row.get(9)?, - skew_y: row.get(10)?, - t_center_x: row.get(11)?, - t_center_y: row.get(12)?, - }, - order_index: row.get(13)?, - }) -} diff --git a/crates/shift-store/src/connection.rs b/crates/shift-store/src/connection.rs index 56ed20f7..c4e6a214 100644 --- a/crates/shift-store/src/connection.rs +++ b/crates/shift-store/src/connection.rs @@ -4,23 +4,73 @@ use crate::{ShiftStore, StoreError, schema}; impl ShiftStore { pub fn open(path: impl AsRef) -> Result { + let path = path.as_ref(); let conn = rusqlite::Connection::open(path)?; configure_connection(&conn)?; schema::ensure_current(&conn)?; - Ok(Self { conn }) + Ok(Self { + conn, + path: Some(path.to_path_buf()), + }) + } + + /// Opens a disposable import destination with rollback-capable in-memory + /// journaling. The foreign source remains authoritative until + /// [`Self::finish_import`] makes the completed store durable. + pub fn open_for_import(path: impl AsRef) -> Result { + let path = path.as_ref(); + let conn = rusqlite::Connection::open(path)?; + configure_import_connection(&conn)?; + schema::ensure_current(&conn)?; + let has_workspace: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM workspace_state WHERE id = 1)", + [], + |row| row.get(0), + )?; + if has_workspace { + return Err(StoreError::ImportDestinationNotEmpty(path.to_path_buf())); + } + Ok(Self { + conn, + path: Some(path.to_path_buf()), + }) + } + + /// Flushes a completed staged import and restores the normal edit-time + /// WAL posture before the workspace publishes the closed database. + pub fn finish_import(&self) -> Result<(), StoreError> { + self.sync_store_file()?; + + let journal_mode: String = self + .conn + .query_row("PRAGMA journal_mode=WAL", [], |row| row.get(0))?; + debug_assert_eq!(journal_mode, "wal"); + self.conn.pragma_update(None, "synchronous", "NORMAL")?; + self.sync_store_file()?; + Ok(()) + } + + fn sync_store_file(&self) -> Result<(), StoreError> { + if let Some(path) = &self.path { + std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path)? + .sync_all()?; + } + Ok(()) } pub fn open_memory_for_test() -> Result { let conn = rusqlite::Connection::open_in_memory()?; configure_connection(&conn)?; schema::ensure_current(&conn)?; - Ok(Self { conn }) + Ok(Self { conn, path: None }) } } fn configure_connection(conn: &rusqlite::Connection) -> Result<(), StoreError> { - conn.pragma_update(None, "foreign_keys", "ON")?; - conn.busy_timeout(std::time::Duration::from_millis(5_000))?; + configure_common(conn)?; // WAL + NORMAL is the durability/latency posture for a store that // commits per user action. In-memory test connections report "memory" @@ -29,6 +79,21 @@ fn configure_connection(conn: &rusqlite::Connection) -> Result<(), StoreError> { let journal_mode: String = conn.query_row("PRAGMA journal_mode=WAL", [], |row| row.get(0))?; debug_assert!(matches!(journal_mode.as_str(), "wal" | "memory")); conn.pragma_update(None, "synchronous", "NORMAL")?; + Ok(()) +} +fn configure_import_connection(conn: &rusqlite::Connection) -> Result<(), StoreError> { + configure_common(conn)?; + let journal_mode: String = + conn.query_row("PRAGMA journal_mode=MEMORY", [], |row| row.get(0))?; + debug_assert_eq!(journal_mode, "memory"); + conn.pragma_update(None, "synchronous", "OFF")?; + Ok(()) +} + +fn configure_common(conn: &rusqlite::Connection) -> Result<(), StoreError> { + conn.set_prepared_statement_cache_capacity(64); + conn.pragma_update(None, "foreign_keys", "ON")?; + conn.busy_timeout(std::time::Duration::from_millis(5_000))?; Ok(()) } diff --git a/crates/shift-store/src/error.rs b/crates/shift-store/src/error.rs index 535aa97c..97714a62 100644 --- a/crates/shift-store/src/error.rs +++ b/crates/shift-store/src/error.rs @@ -1,5 +1,8 @@ #[derive(Debug, thiserror::Error)] pub enum StoreError { + #[error("store file-system error: {0}")] + Io(#[from] std::io::Error), + #[error("sqlite error: {0}")] Sqlite(#[from] rusqlite::Error), @@ -9,6 +12,42 @@ pub enum StoreError { #[error(transparent)] Font(#[from] shift_font::error::CoreError), + #[error("glyph layer encoding failed: {0}")] + LayerEncoding(String), + + #[error("glyph layer decoding failed: {0}")] + LayerDecoding(String), + + #[error("invalid glyph layer payload: {0}")] + InvalidLayerPayload(String), + + #[error("unsupported glyph layer payload format: {0}")] + UnsupportedLayerFormat(String), + + #[error("unsupported glyph layer compression: {0}")] + UnsupportedLayerCompression(String), + + #[error("glyph layer compression failed: {0}")] + LayerCompression(String), + + #[error("glyph layer decompression failed: {0}")] + LayerDecompression(String), + + #[error("glyph layer payload has {bytes} bytes; limit is {limit}")] + LayerPayloadTooLarge { bytes: u64, limit: u64 }, + + #[error("glyph layer read batch has {layers} layers; limit is {limit}")] + LayerReadBatchTooLarge { layers: usize, limit: usize }, + + #[error("glyph layer read batch has {bytes} decoded bytes; limit is {limit}")] + LayerReadBatchDecodedTooLarge { bytes: u64, limit: u64 }, + + #[error("glyph layer {layer_id} payload disagrees with its directory: {detail}")] + LayerDirectoryMismatch { layer_id: String, detail: String }, + + #[error("glyph layer {layer_id} has a stale component/reference index")] + StaleLayerReferenceIndex { layer_id: String }, + #[error("unknown source kind: {0}")] UnknownSourceKind(String), @@ -21,6 +60,9 @@ pub enum StoreError { #[error("invalid workspace state: {0}")] InvalidWorkspaceState(String), + #[error("import destination already contains published workspace state: {0}")] + ImportDestinationNotEmpty(std::path::PathBuf), + #[error("missing {kind}: {id}")] MissingEntity { kind: &'static str, id: String }, diff --git a/crates/shift-store/src/font_state.rs b/crates/shift-store/src/font_state.rs index 736507f0..e9aed2bf 100644 --- a/crates/shift-store/src/font_state.rs +++ b/crates/shift-store/src/font_state.rs @@ -1,5 +1,4 @@ -use std::collections::HashMap; -use std::str::FromStr; +use std::collections::{BTreeMap, HashMap}; use rusqlite::params; use shift_font as font; @@ -7,7 +6,19 @@ use shift_font as font; use crate::{FontInfo, ShiftStore, StoreError, source::SourceKind}; impl ShiftStore { + /// Materializes the complete font. Use only for export/save boundaries; + /// interactive open should use [`Self::load_font_directory`]. pub fn load_font_state(&self) -> Result { + self.load_font(true) + } + + /// Loads eager font metadata and the glyph/layer directory without + /// reading any layer payload BLOB. + pub fn load_font_directory(&self) -> Result { + self.load_font(false) + } + + fn load_font(&self, include_layer_payloads: bool) -> Result { let mut font = font::Font::empty(); if let Some(info) = self.get_font_info()? { @@ -43,7 +54,7 @@ impl ShiftStore { font.set_default_source_id(default_source_id); } - for glyph in load_glyphs(&self.conn)? { + for glyph in load_glyphs(&self.conn, include_layer_payloads)? { font.insert_glyph(glyph).map_err(StoreError::from)?; } @@ -297,257 +308,125 @@ fn load_source_location( Ok(location) } -fn load_glyphs(conn: &rusqlite::Connection) -> Result, StoreError> { - let mut stmt = conn.prepare( - " - SELECT id, name - FROM glyphs - ORDER BY order_index, id - ", - )?; - - let rows = stmt.query_map([], |row| { - Ok(( - font::GlyphId::from_raw(row.get::<_, String>(0)?), - row.get::<_, Option>(1)?.unwrap_or_default(), - )) - })?; - - let mut glyphs = Vec::new(); - for row in rows { - let (glyph_id, name) = row?; - let mut glyph = font::Glyph::with_id(glyph_id.clone(), name); - glyph.set_unicodes(load_glyph_unicodes(conn, &glyph_id)?); - *glyph.lib_mut() = - load_lib_data(conn, "glyph_lib", Some(("glyph_id", &glyph_id.to_string())))?; +fn load_glyphs( + conn: &rusqlite::Connection, + include_layer_payloads: bool, +) -> Result, StoreError> { + let glyph_rows = { + let mut stmt = conn.prepare("SELECT id, name FROM glyphs ORDER BY order_index, id")?; + stmt.query_map([], |row| { + Ok(( + font::GlyphId::from_raw(row.get::<_, String>(0)?), + row.get::<_, Option>(1)?.unwrap_or_default(), + )) + })? + .collect::, _>>()? + }; - for layer in load_layers_for_glyph(conn, &glyph_id)? { - glyph.set_layer(layer); + let mut unicodes: HashMap> = HashMap::new(); + { + let mut stmt = conn.prepare( + "SELECT glyph_id, unicode, order_index FROM glyph_unicodes ORDER BY glyph_id, order_index", + )?; + let rows = stmt.query_map([], |row| { + Ok(( + font::GlyphId::from_raw(row.get::<_, String>(0)?), + row.get::<_, i64>(1)? as u32, + row.get::<_, i64>(2)?, + )) + })?; + for row in rows { + let (glyph_id, unicode, order_index) = row?; + unicodes + .entry(glyph_id) + .or_default() + .push((order_index, unicode)); } - - glyphs.push(glyph); } - Ok(glyphs) -} - -fn load_glyph_unicodes( - conn: &rusqlite::Connection, - glyph_id: &font::GlyphId, -) -> Result, StoreError> { - let mut stmt = conn.prepare( - " - SELECT unicode - FROM glyph_unicodes - WHERE glyph_id = ?1 - ORDER BY order_index - ", - )?; - let rows = stmt.query_map([glyph_id.to_string()], |row| row.get::<_, i64>(0))?; - rows.map(|row| row.map(|value| value as u32)) - .collect::, _>>() - .map_err(StoreError::from) -} - -fn load_layers_for_glyph( - conn: &rusqlite::Connection, - glyph_id: &font::GlyphId, -) -> Result, StoreError> { - let mut stmt = conn.prepare( - " - SELECT id, source_id, width, height - FROM glyph_layers - WHERE glyph_id = ?1 - ORDER BY id - ", - )?; - - let rows = stmt.query_map([glyph_id.to_string()], |row| { - Ok(( - font::LayerId::from_raw(row.get::<_, String>(0)?), - font::SourceId::from_raw(row.get::<_, String>(1)?), - row.get::<_, f64>(2)?, - row.get::<_, Option>(3)?, - )) - })?; - - let mut layers = Vec::new(); - for row in rows { - let (layer_id, source_id, width, height) = row?; - let mut layer = font::GlyphLayer::with_width(layer_id.clone(), source_id, width); - layer.set_height(height); - - for contour in load_contours_for_layer(conn, &layer_id)? { - layer.add_contour(contour); - } - for component in load_components_for_layer(conn, &layer_id)? { - layer.add_component(component); - } - for anchor in load_anchors_for_layer(conn, &layer_id)? { - layer.add_anchor(anchor); - } - for guideline in load_layer_guidelines(conn, &layer_id)? { - layer.add_guideline(guideline); + let mut libs: HashMap> = HashMap::new(); + { + let mut stmt = conn.prepare("SELECT glyph_id, key, value_json FROM glyph_lib")?; + let rows = stmt.query_map([], |row| { + Ok(( + font::GlyphId::from_raw(row.get::<_, String>(0)?), + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + })?; + for row in rows { + let (glyph_id, key, value_json) = row?; + libs.entry(glyph_id) + .or_default() + .insert(key, lib_value_from_json(&value_json)?); } - *layer.lib_mut() = load_lib_data( - conn, - "glyph_layer_lib", - Some(("layer_id", &layer_id.to_string())), - )?; - - layers.push(layer); } - Ok(layers) -} - -fn load_contours_for_layer( - conn: &rusqlite::Connection, - layer_id: &font::LayerId, -) -> Result, StoreError> { - let mut stmt = conn.prepare( - " - SELECT id, closed - FROM glyph_layer_contours - WHERE layer_id = ?1 - ORDER BY order_index - ", - )?; - - let rows = stmt.query_map([layer_id.to_string()], |row| { - Ok(( - font::ContourId::from_raw(row.get::<_, String>(0)?), - row.get::<_, bool>(1)?, - )) - })?; - - let mut contours = Vec::new(); - for row in rows { - let (contour_id, closed) = row?; - let mut contour = font::Contour::with_id(contour_id.clone()); - for point in load_points_for_contour(conn, &contour_id)? { - contour.push_point(point); - } - if closed { - contour.close(); + let layer_rows = { + let mut stmt = conn.prepare( + " + SELECT id, glyph_id, source_id, width, height + FROM glyph_layers + ORDER BY glyph_id, source_id, id + ", + )?; + stmt.query_map([], |row| { + Ok(( + font::LayerId::from_raw(row.get::<_, String>(0)?), + font::GlyphId::from_raw(row.get::<_, String>(1)?), + font::SourceId::from_raw(row.get::<_, String>(2)?), + row.get::<_, f64>(3)?, + row.get::<_, Option>(4)?, + )) + })? + .collect::, _>>()? + }; + let mut loaded_layers = HashMap::new(); + if include_layer_payloads { + let layer_ids = layer_rows + .iter() + .map(|(layer_id, _, _, _, _)| layer_id.clone()) + .collect::>(); + for layer in crate::layer::load_glyph_layers_from_conn(conn, &layer_ids)? { + loaded_layers.insert(layer.id(), layer); } - contours.push(contour); } - Ok(contours) -} - -fn load_points_for_contour( - conn: &rusqlite::Connection, - contour_id: &font::ContourId, -) -> Result, StoreError> { - let mut stmt = conn.prepare( - " - SELECT id, x, y, point_type, smooth - FROM glyph_layer_points - WHERE contour_id = ?1 - ORDER BY order_index - ", - )?; - - let rows = stmt.query_map([contour_id.to_string()], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, f64>(1)?, - row.get::<_, f64>(2)?, - row.get::<_, String>(3)?, - row.get::<_, bool>(4)?, - )) - })?; - - let mut points = Vec::new(); - for row in rows { - let (id, x, y, point_type, smooth) = row?; - let point_type = - font::PointType::from_str(&point_type).map_err(StoreError::InvalidPointType)?; - points.push(font::Point::new( - font::PointId::from_raw(id), - x, - y, - point_type, - smooth, - )); + let mut layers: HashMap> = HashMap::new(); + for (layer_id, glyph_id, source_id, width, height) in layer_rows { + let layer = if include_layer_payloads { + loaded_layers + .remove(&layer_id) + .ok_or_else(|| StoreError::MissingEntity { + kind: "glyph layer payload", + id: layer_id.to_string(), + })? + } else { + let mut layer = font::GlyphLayer::with_width(layer_id, source_id, width); + layer.set_height(height); + layer + }; + layers.entry(glyph_id).or_default().push(layer); } - Ok(points) -} - -fn load_components_for_layer( - conn: &rusqlite::Connection, - layer_id: &font::LayerId, -) -> Result, StoreError> { - let mut stmt = conn.prepare( - " - SELECT - id, - base_glyph_id, - base_glyph_name, - translate_x, - translate_y, - rotation, - scale_x, - scale_y, - skew_x, - skew_y, - t_center_x, - t_center_y - FROM glyph_components - WHERE layer_id = ?1 - ORDER BY order_index, id - ", - )?; - - let rows = stmt.query_map([layer_id.to_string()], |row| { - Ok(font::Component::with_id( - font::ComponentId::from_raw(row.get::<_, String>(0)?), - font::GlyphId::from_raw(row.get::<_, String>(1)?), - row.get::<_, String>(2)?, - font::DecomposedTransform { - translate_x: row.get(3)?, - translate_y: row.get(4)?, - rotation: row.get(5)?, - scale_x: row.get(6)?, - scale_y: row.get(7)?, - skew_x: row.get(8)?, - skew_y: row.get(9)?, - t_center_x: row.get(10)?, - t_center_y: row.get(11)?, - }, - )) - })?; - - rows.collect::, _>>() - .map_err(StoreError::from) -} - -fn load_anchors_for_layer( - conn: &rusqlite::Connection, - layer_id: &font::LayerId, -) -> Result, StoreError> { - let mut stmt = conn.prepare( - " - SELECT id, name, x, y - FROM glyph_layer_anchors - WHERE layer_id = ?1 - ORDER BY order_index - ", - )?; - - let rows = stmt.query_map([layer_id.to_string()], |row| { - Ok(font::Anchor::with_id( - font::AnchorId::from_raw(row.get::<_, String>(0)?), - row.get::<_, Option>(1)?, - row.get(2)?, - row.get(3)?, - )) - })?; - rows.collect::, _>>() - .map_err(StoreError::from) + let mut glyphs = Vec::with_capacity(glyph_rows.len()); + for (glyph_id, name) in glyph_rows { + let mut glyph = font::Glyph::with_id(glyph_id.clone(), name); + let mut glyph_unicodes = unicodes.remove(&glyph_id).unwrap_or_default(); + glyph_unicodes.sort_by_key(|(order_index, _)| *order_index); + glyph.set_unicodes( + glyph_unicodes + .into_iter() + .map(|(_, unicode)| unicode) + .collect(), + ); + *glyph.lib_mut() = font::LibData::from_map(libs.remove(&glyph_id).unwrap_or_default()); + for layer in layers.remove(&glyph_id).unwrap_or_default() { + glyph.set_layer(layer); + } + glyphs.push(glyph); + } + Ok(glyphs) } fn load_font_guidelines(conn: &rusqlite::Connection) -> Result, StoreError> { @@ -564,24 +443,6 @@ fn load_font_guidelines(conn: &rusqlite::Connection) -> Result Result, StoreError> { - let mut stmt = conn.prepare( - " - SELECT id, x, y, angle, name, color - FROM glyph_layer_guidelines - WHERE layer_id = ?1 - ORDER BY order_index - ", - )?; - let rows = stmt.query_map([layer_id.to_string()], map_guideline_row)?; - - rows.collect::, _>>() - .map_err(StoreError::from) -} - fn map_guideline_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { Ok(font::Guideline::with_id( font::GuidelineId::from_raw(row.get::<_, String>(0)?), @@ -778,7 +639,7 @@ fn lib_value_from_value(value: serde_json::Value) -> Result values .into_iter() .map(|(key, value)| lib_value_from_value(value).map(|value| (key, value))) - .collect::, _>>() + .collect::, _>>() .map(font::LibValue::Dict), _ => Err(StoreError::InvalidLibValue("expected dict".to_string())), }, diff --git a/crates/shift-store/src/import_writer.rs b/crates/shift-store/src/import_writer.rs new file mode 100644 index 00000000..9a6e35d3 --- /dev/null +++ b/crates/shift-store/src/import_writer.rs @@ -0,0 +1,213 @@ +use std::time::Instant; + +use rayon::prelude::*; +use rusqlite::Transaction; +use shift_font as font; + +use crate::{ + GlyphWriteBatch, LayerBatchTiming, ShiftStore, StoreError, + change_set::{replace_font_header_in_tx, write_glyph_directory_in_tx}, + layer::{StoredLayerPayload, compress_layer, encode_layer, store_stored_layer_in_tx}, + schema::{defer_import_indexes, restore_import_indexes}, + types::{EncodedGlyphLayers, PackedGlyph}, + write_mode::WriteMode, +}; + +/// Bounded transactional sink for format-specific importers. +/// +/// A foreign reader can write one glyph batch at a time without building a +/// complete [`font::Font`]. The transaction commits only when [`Self::finish`] +/// is called; dropping the writer rolls the entire stream back. +pub struct FontImportWriter<'store> { + tx: Transaction<'store>, + next_glyph_order: i64, +} + +impl ShiftStore { + /// Starts an atomic replacement using a glyph-free font header. + pub fn begin_import( + &mut self, + font_header: &font::Font, + ) -> Result, StoreError> { + let tx = self.conn.transaction()?; + replace_font_header_in_tx(&tx, font_header)?; + defer_import_indexes(&tx)?; + Ok(FontImportWriter { + tx, + next_glyph_order: 0, + }) + } +} + +/// Encodes and compresses an owned bounded glyph batch without touching SQLite. +/// The returned batch retains glyph directory facts for the single writer. +pub fn encode_glyph_batch(glyphs: Vec) -> Result { + let started = Instant::now(); + let encoded_layers = encode_glyph_layers(&glyphs)?; + let pack_elapsed = started.elapsed(); + let started = Instant::now(); + let layers = compress_glyph_layers(encoded_layers)?; + let compression_elapsed = started.elapsed(); + let glyphs = glyphs + .into_iter() + .zip(layers) + .map(|(glyph, layers)| PackedGlyph { glyph, layers }) + .collect(); + + Ok(GlyphWriteBatch { + glyphs, + pack_elapsed, + compression_elapsed, + }) +} + +impl FontImportWriter<'_> { + /// Writes one complete glyph and all of its currently authored layers. + pub fn write_glyph(&mut self, glyph: &font::Glyph) -> Result<(), StoreError> { + self.write_glyph_batch(std::slice::from_ref(glyph))?; + Ok(()) + } + + /// Encodes a bounded glyph batch in parallel, then writes it in stable order. + pub fn write_glyph_batch( + &mut self, + glyphs: &[font::Glyph], + ) -> Result { + let started = Instant::now(); + let encoded_layers = encode_glyph_layers(glyphs)?; + let pack_elapsed = started.elapsed(); + let started = Instant::now(); + let layers = compress_glyph_layers(encoded_layers)?; + let compression_elapsed = started.elapsed(); + + let started = Instant::now(); + for (glyph, layers) in glyphs.iter().zip(&layers) { + self.write_packed_glyph(glyph, layers)?; + } + + Ok(LayerBatchTiming { + pack_elapsed, + compression_elapsed, + sqlite_elapsed: started.elapsed(), + }) + } + + /// Writes a prepared batch in stable order through the owned transaction. + pub fn write_packed_glyph_batch( + &mut self, + batch: GlyphWriteBatch, + ) -> Result { + let started = Instant::now(); + for glyph in &batch.glyphs { + self.write_packed_glyph(&glyph.glyph, &glyph.layers)?; + } + Ok(started.elapsed()) + } + + fn write_packed_glyph( + &mut self, + glyph: &font::Glyph, + layers: &[(font::LayerId, StoredLayerPayload)], + ) -> Result<(), StoreError> { + write_glyph_directory_in_tx(&self.tx, glyph, self.next_glyph_order, WriteMode::Insert)?; + for (layer_id, stored) in layers { + let layer = glyph + .layers() + .get(layer_id) + .ok_or_else(|| StoreError::MissingEntity { + kind: "glyph layer", + id: layer_id.to_string(), + })?; + store_stored_layer_in_tx(&self.tx, &glyph.id(), layer, stored, WriteMode::Insert)?; + } + self.next_glyph_order += 1; + Ok(()) + } + + /// Atomically publishes every glyph and layer written by this stream. + pub fn finish(self) -> Result<(), StoreError> { + restore_import_indexes(&self.tx)?; + self.tx.commit()?; + Ok(()) + } +} + +fn encode_glyph_layers(glyphs: &[font::Glyph]) -> Result { + glyphs + .par_iter() + .map(|glyph| { + glyph + .layers() + .values() + .map(|layer| encode_layer(layer).map(|encoded| (layer.id(), encoded))) + .collect::, _>>() + }) + .collect() +} + +fn compress_glyph_layers( + layers: Vec)>>, +) -> Result>, StoreError> { + layers + .into_par_iter() + .map(|layers| { + layers + .into_iter() + .map(|(layer_id, encoded)| { + compress_layer(&encoded).map(|stored| (layer_id, stored)) + }) + .collect::, _>>() + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + const DEFERRED_INDEXES: [&str; 4] = [ + "glyphs_name_idx", + "glyph_layers_glyph_id_idx", + "glyph_layers_source_id_idx", + "glyph_components_base_glyph_id_idx", + ]; + + #[test] + fn stream_indexes_are_transactionally_deferred_and_restored() { + let font = font::test_support::sample_font(); + let mut store = ShiftStore::open_memory_for_test().unwrap(); + for name in DEFERRED_INDEXES { + assert!(index_exists(&store.conn, name)); + } + assert!(!index_exists(&store.conn, "glyph_unicodes_glyph_id_idx")); + assert!(!index_exists(&store.conn, "glyph_components_layer_id_idx")); + + { + let writer = store.begin_import(&font).unwrap(); + for name in DEFERRED_INDEXES { + assert!(!index_exists(&writer.tx, name)); + } + } + for name in DEFERRED_INDEXES { + assert!(index_exists(&store.conn, name)); + } + + let mut writer = store.begin_import(&font).unwrap(); + for glyph in font.glyphs() { + writer.write_glyph(glyph).unwrap(); + } + writer.finish().unwrap(); + for name in DEFERRED_INDEXES { + assert!(index_exists(&store.conn, name)); + } + } + + fn index_exists(conn: &rusqlite::Connection, name: &str) -> bool { + conn.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type = 'index' AND name = ?1)", + [name], + |row| row.get(0), + ) + .unwrap() + } +} diff --git a/crates/shift-store/src/layer.rs b/crates/shift-store/src/layer.rs deleted file mode 100644 index 6c3b983a..00000000 --- a/crates/shift-store/src/layer.rs +++ /dev/null @@ -1,99 +0,0 @@ -use crate::{GlyphId, LayerId, ShiftStore, SourceId, StoreError}; - -pub struct NewGlyphLayer { - pub id: LayerId, - pub glyph_id: GlyphId, - pub source_id: SourceId, - pub name: Option, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct GlyphLayerRecord { - pub id: LayerId, - pub glyph_id: GlyphId, - pub source_id: SourceId, - pub name: Option, - pub width: f64, - pub height: Option, -} - -impl ShiftStore { - pub fn create_glyph_layer(&mut self, layer: NewGlyphLayer) -> Result<(), StoreError> { - self.conn.execute( - " - INSERT INTO glyph_layers ( - id, - glyph_id, - source_id, - name - ) - VALUES (?1, ?2, ?3, ?4) - ", - rusqlite::params![ - layer.id.as_str(), - layer.glyph_id.as_str(), - layer.source_id.as_str(), - layer.name, - ], - )?; - - Ok(()) - } - - pub fn get_glyph_layer(&self, id: &LayerId) -> Result, StoreError> { - let mut stmt = self.conn.prepare( - " - SELECT - id, - glyph_id, - source_id, - name, - width, - height - FROM glyph_layers - WHERE id = ?1 - ", - )?; - - match stmt.query_row([id.as_str()], map_glyph_layer_row) { - Ok(layer) => Ok(Some(layer)), - Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), - Err(err) => Err(err.into()), - } - } - - pub fn list_glyph_layers_for_glyph( - &self, - glyph_id: &GlyphId, - ) -> Result, StoreError> { - let mut stmt = self.conn.prepare( - " - SELECT - id, - glyph_id, - source_id, - name, - width, - height - FROM glyph_layers - WHERE glyph_id = ?1 - ORDER BY source_id, id - ", - )?; - - let rows = stmt.query_map([glyph_id.as_str()], map_glyph_layer_row)?; - rows.collect::, _>>() - .map_err(StoreError::from) - } -} - -fn map_glyph_layer_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(GlyphLayerRecord { - id: LayerId::new(row.get::<_, String>(0)?), - glyph_id: GlyphId::new(row.get::<_, String>(1)?), - source_id: SourceId::new(row.get::<_, String>(2)?), - name: row.get(3)?, - width: row.get(4)?, - height: row.get(5)?, - }) -} diff --git a/crates/shift-store/src/layer/directory.rs b/crates/shift-store/src/layer/directory.rs new file mode 100644 index 00000000..f7501728 --- /dev/null +++ b/crates/shift-store/src/layer/directory.rs @@ -0,0 +1,191 @@ +use shift_font as font; + +use super::{ + GLYPH_LAYER_FORMAT, MAX_LAYER_PAYLOAD_BYTES, + payload::{LayerPayloadCompression, StoredLayerPayload}, +}; +use crate::{ShiftStore, StoreError}; + +const MAX_LAYER_PAYLOAD_BYTES_SQL: i64 = MAX_LAYER_PAYLOAD_BYTES as i64; + +/// Relational facts needed to locate a layer without reading its payload. +#[derive(Clone, Debug, PartialEq)] +pub struct GlyphLayerDirectoryEntry { + pub layer_id: font::LayerId, + pub glyph_id: font::GlyphId, + pub source_id: font::SourceId, + pub name: Option, + pub width: f64, + pub height: Option, + pub stored_byte_length: u64, + pub decoded_byte_length: u64, +} + +impl ShiftStore { + pub fn list_glyph_layer_directory(&self) -> Result, StoreError> { + let mut stmt = self.conn.prepare( + " + SELECT l.id, l.glyph_id, l.source_id, g.name, l.width, l.height, + p.stored_byte_length, p.decoded_byte_length + FROM glyph_layers AS l + JOIN glyphs AS g ON g.id = l.glyph_id + JOIN glyph_layer_payloads AS p ON p.layer_id = l.id + ORDER BY l.glyph_id, l.source_id, l.id + ", + )?; + let rows = stmt.query_map([], map_directory_row)?; + rows.collect::, _>>().map_err(Into::into) + } +} + +pub(super) struct DirectoryFacts { + pub(super) layer_id: String, + pub(super) source_id: String, + pub(super) width: f64, + pub(super) height: Option, + inner_format: String, + compression: String, + stored_byte_length: i64, + decoded_byte_length: i64, + decoded_blake3: Vec, + pub(super) actual_stored_byte_length: i64, +} + +impl DirectoryFacts { + pub(super) fn validate(&self) -> Result { + if self.inner_format != GLYPH_LAYER_FORMAT { + return Err(StoreError::UnsupportedLayerFormat( + self.inner_format.clone(), + )); + } + LayerPayloadCompression::try_from(self.compression.as_str())?; + if self.stored_byte_length != self.actual_stored_byte_length { + return Err(StoreError::LayerDirectoryMismatch { + layer_id: self.layer_id.clone(), + detail: format!( + "declared stored byte length {} != actual {}", + self.stored_byte_length, self.actual_stored_byte_length + ), + }); + } + for byte_length in [self.stored_byte_length, self.decoded_byte_length] { + if !(0..=MAX_LAYER_PAYLOAD_BYTES_SQL).contains(&byte_length) { + return Err(StoreError::LayerPayloadTooLarge { + bytes: u64::try_from(byte_length).unwrap_or(u64::MAX), + limit: MAX_LAYER_PAYLOAD_BYTES as u64, + }); + } + } + if self.stored_byte_length > self.decoded_byte_length { + return Err(StoreError::LayerDirectoryMismatch { + layer_id: self.layer_id.clone(), + detail: format!( + "stored byte length {} exceeds decoded byte length {}", + self.stored_byte_length, self.decoded_byte_length + ), + }); + } + if self.decoded_blake3.len() != 32 { + return Err(StoreError::LayerDirectoryMismatch { + layer_id: self.layer_id.clone(), + detail: format!( + "decoded BLAKE3 has {} bytes instead of 32", + self.decoded_blake3.len() + ), + }); + } + Ok(self.decoded_byte_length as u64) + } + + pub(super) fn stored_layer(&self, payload: Vec) -> Result { + let decoded_blake3: [u8; 32] = self.decoded_blake3.as_slice().try_into().map_err(|_| { + StoreError::LayerDirectoryMismatch { + layer_id: self.layer_id.clone(), + detail: "decoded BLAKE3 is not 32 bytes".to_string(), + } + })?; + Ok(StoredLayerPayload { + compression: self.compression.as_str().try_into()?, + stored_byte_length: self.stored_byte_length as u64, + decoded_byte_length: self.decoded_byte_length as u64, + decoded_blake3, + bytes: payload, + }) + } +} + +pub(super) fn map_directory_facts_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(DirectoryFacts { + layer_id: row.get(0)?, + source_id: row.get(1)?, + width: row.get(2)?, + height: row.get(3)?, + inner_format: row.get(4)?, + compression: row.get(5)?, + stored_byte_length: row.get(6)?, + decoded_byte_length: row.get(7)?, + decoded_blake3: row.get(8)?, + actual_stored_byte_length: row.get(9)?, + }) +} + +pub(super) fn validate_directory_facts( + expected_id: &font::LayerId, + expected_source_id: &str, + expected_width: f64, + expected_height: Option, + layer: &font::GlyphLayer, +) -> Result<(), StoreError> { + let detail = if layer.id() != *expected_id { + Some(format!("payload id is {}", layer.id())) + } else if layer.source_id().as_str() != expected_source_id { + Some(format!("payload source id is {}", layer.source_id())) + } else if layer.width().to_bits() != expected_width.to_bits() { + Some("payload width differs from directory width".to_string()) + } else if !same_optional_f64(layer.height(), expected_height) { + Some("payload height differs from directory height".to_string()) + } else { + None + }; + if let Some(detail) = detail { + return Err(StoreError::LayerDirectoryMismatch { + layer_id: expected_id.to_string(), + detail, + }); + } + Ok(()) +} + +fn map_directory_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let stored_byte_length = directory_byte_length(row, 6)?; + let decoded_byte_length = directory_byte_length(row, 7)?; + Ok(GlyphLayerDirectoryEntry { + layer_id: font::LayerId::from_raw(row.get::<_, String>(0)?), + glyph_id: font::GlyphId::from_raw(row.get::<_, String>(1)?), + source_id: font::SourceId::from_raw(row.get::<_, String>(2)?), + name: row.get::<_, Option>(3)?.map(font::GlyphName::from), + width: row.get(4)?, + height: row.get(5)?, + stored_byte_length, + decoded_byte_length, + }) +} + +fn directory_byte_length(row: &rusqlite::Row<'_>, index: usize) -> rusqlite::Result { + let byte_length = row.get::<_, i64>(index)?; + u64::try_from(byte_length).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + index, + rusqlite::types::Type::Integer, + Box::new(error), + ) + }) +} + +fn same_optional_f64(left: Option, right: Option) -> bool { + match (left, right) { + (Some(left), Some(right)) => left.to_bits() == right.to_bits(), + (None, None) => true, + _ => false, + } +} diff --git a/crates/shift-store/src/layer/format.rs b/crates/shift-store/src/layer/format.rs new file mode 100644 index 00000000..7772db0a --- /dev/null +++ b/crates/shift-store/src/layer/format.rs @@ -0,0 +1,328 @@ +use std::{collections::HashSet, io::Cursor}; + +use serde::Deserialize; +use shift_font::{self as font, GlyphEntityId, LibValue}; + +use super::check_layer_length; +use crate::StoreError; + +pub const GLYPH_LAYER_FORMAT: &str = "shift.glyph-layer.v1"; +const MAX_NESTING_DEPTH: usize = 64; +const MAX_LIB_VALUES: usize = 1_000_000; + +pub(crate) fn encode_layer(layer: &font::GlyphLayer) -> Result, StoreError> { + validate_layer(layer).map_err(StoreError::InvalidLayerPayload)?; + let bytes = + rmp_serde::to_vec(layer).map_err(|error| StoreError::LayerEncoding(error.to_string()))?; + check_layer_length(bytes.len() as u64)?; + Ok(bytes) +} + +pub(crate) fn decode_layer(bytes: &[u8]) -> Result { + check_layer_length(bytes.len() as u64)?; + let mut decoder = rmp_serde::Deserializer::new(Cursor::new(bytes)); + decoder.set_max_depth(MAX_NESTING_DEPTH); + let layer = font::GlyphLayer::deserialize(&mut decoder) + .map_err(|error| StoreError::LayerDecoding(error.to_string()))?; + let consumed = decoder.position() as usize; + if consumed != bytes.len() { + return Err(StoreError::LayerDecoding(format!( + "payload has {} trailing bytes", + bytes.len() - consumed + ))); + } + validate_layer(&layer).map_err(StoreError::InvalidLayerPayload)?; + Ok(layer) +} + +fn validate_layer(layer: &font::GlyphLayer) -> Result<(), String> { + validate_finite(layer.width(), "width")?; + if let Some(height) = layer.height() { + validate_finite(height, "height")?; + } + + let mut entity_ids = HashSet::new(); + for contour in layer.contours_iter() { + record_entity(&mut entity_ids, contour.id().into())?; + for point in contour.points() { + record_entity(&mut entity_ids, point.id().into())?; + validate_finite(point.x(), "point x")?; + validate_finite(point.y(), "point y")?; + } + } + + for component in layer.components_iter() { + record_entity(&mut entity_ids, component.id().into())?; + let transform = component.transform(); + for (value, field) in [ + (transform.translate_x, "component translate x"), + (transform.translate_y, "component translate y"), + (transform.rotation, "component rotation"), + (transform.scale_x, "component scale x"), + (transform.scale_y, "component scale y"), + (transform.skew_x, "component skew x"), + (transform.skew_y, "component skew y"), + (transform.t_center_x, "component center x"), + (transform.t_center_y, "component center y"), + ] { + validate_finite(value, field)?; + } + } + + for anchor in layer.anchors_iter() { + record_entity(&mut entity_ids, anchor.id().into())?; + validate_finite(anchor.x(), "anchor x")?; + validate_finite(anchor.y(), "anchor y")?; + } + + for guideline in layer.guidelines() { + record_entity(&mut entity_ids, guideline.id().into())?; + for (value, field) in [ + (guideline.x(), "guideline x"), + (guideline.y(), "guideline y"), + (guideline.angle(), "guideline angle"), + ] { + if let Some(value) = value { + validate_finite(value, field)?; + } + } + } + + let mut lib_values = 0; + for value in layer.lib().iter().map(|(_, value)| value) { + validate_lib_value(value, 0, &mut lib_values)?; + } + Ok(()) +} + +fn record_entity( + entity_ids: &mut HashSet, + entity_id: GlyphEntityId, +) -> Result<(), String> { + if entity_ids.insert(entity_id.clone()) { + return Ok(()); + } + + let (kind, value) = match &entity_id { + GlyphEntityId::Contour(id) => ("contour", id.as_str()), + GlyphEntityId::Point(id) => ("point", id.as_str()), + GlyphEntityId::Component(id) => ("component", id.as_str()), + GlyphEntityId::Anchor(id) => ("anchor", id.as_str()), + GlyphEntityId::Guideline(id) => ("guideline", id.as_str()), + }; + Err(format!("duplicate {kind} ID {value:?}")) +} + +fn validate_lib_value(value: &LibValue, depth: usize, count: &mut usize) -> Result<(), String> { + if depth >= MAX_NESTING_DEPTH { + return Err("layer lib is nested too deeply".to_string()); + } + *count += 1; + if *count > MAX_LIB_VALUES { + return Err("layer lib has too many values".to_string()); + } + + match value { + LibValue::Float(value) => validate_finite(*value, "layer lib float"), + LibValue::Array(values) => { + for value in values { + validate_lib_value(value, depth + 1, count)?; + } + Ok(()) + } + LibValue::Dict(values) => { + for value in values.values() { + validate_lib_value(value, depth + 1, count)?; + } + Ok(()) + } + _ => Ok(()), + } +} + +fn validate_finite(value: f64, field: &str) -> Result<(), String> { + if value.is_finite() { + return Ok(()); + } + + Err(format!("non-finite {field}")) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + + #[test] + fn complete_layer_roundtrips_through_deterministic_bytes() { + let font = font::test_support::sample_font(); + let layer = font + .glyph_by_name("A") + .unwrap() + .layers() + .values() + .find(|layer| !layer.lib().is_empty()) + .unwrap(); + + let encoded = encode_layer(layer).unwrap(); + let decoded = decode_layer(&encoded).unwrap(); + + assert_eq!(&decoded, layer.as_ref()); + assert_eq!(encode_layer(&decoded).unwrap(), encoded); + } + + #[test] + fn map_insertion_order_does_not_change_bytes() { + let font = font::test_support::sample_font(); + let mut layer = font + .glyph_by_name("A") + .unwrap() + .layers() + .values() + .next() + .unwrap() + .as_ref() + .clone(); + let mut nested = BTreeMap::new(); + for index in (0..100).rev() { + layer.lib_mut().set( + format!("root-{index:03}"), + LibValue::Integer(i64::from(index)), + ); + nested.insert( + format!("nested-{index:03}"), + LibValue::Integer(i64::from(index)), + ); + } + layer.lib_mut().set("nested".into(), LibValue::Dict(nested)); + + let first = encode_layer(&layer).unwrap(); + let decoded = decode_layer(&first).unwrap(); + + assert_eq!(encode_layer(&decoded).unwrap(), first); + } + + #[test] + fn truncations_and_trailing_bytes_are_rejected() { + let font = font::test_support::sample_font(); + let layer = font + .glyph_by_name("A") + .unwrap() + .layers() + .values() + .next() + .unwrap(); + let complete = encode_layer(layer).unwrap(); + for length in 0..complete.len() { + assert!( + decode_layer(&complete[..length]).is_err(), + "accepted truncation at {length}" + ); + } + + let mut trailing = complete; + trailing.push(0); + assert!(decode_layer(&trailing).is_err()); + } + + #[test] + fn hostile_declared_collection_length_is_rejected() { + let mut bytes = vec![0x99, 0xa7]; + bytes.extend_from_slice(b"layer_x"); + bytes.push(0xa8); + bytes.extend_from_slice(b"source_x"); + bytes.extend_from_slice(&[0xcb, 0, 0, 0, 0, 0, 0, 0, 0]); + bytes.push(0xc0); + bytes.extend_from_slice(&[0xdd, 0xff, 0xff, 0xff, 0xff]); + + assert!(decode_layer(&bytes).is_err()); + } + + #[test] + fn excessive_nesting_is_rejected_before_encode_and_during_decode() { + let font = font::test_support::sample_font(); + let mut layer = font + .glyph_by_name("A") + .unwrap() + .layers() + .values() + .next() + .unwrap() + .as_ref() + .clone(); + let mut value = LibValue::Boolean(true); + for _ in 0..MAX_NESTING_DEPTH + 1 { + value = LibValue::Array(vec![value]); + } + layer.lib_mut().set("nested".into(), value); + + assert!(encode_layer(&layer).is_err()); + let encoded = rmp_serde::to_vec(&layer).unwrap(); + assert!(decode_layer(&encoded).is_err()); + } + + #[test] + fn invalid_ids_and_non_finite_values_are_rejected() { + let invalid_id = rmp_serde::to_vec(&( + "wrong", + "source_test", + 0.0_f64, + None::, + Vec::::new(), + Vec::::new(), + Vec::::new(), + Vec::::new(), + font::LibData::new(), + )) + .unwrap(); + assert!(decode_layer(&invalid_id).is_err()); + + let non_finite = rmp_serde::to_vec(&( + "layer_test", + "source_test", + f64::NAN, + None::, + Vec::::new(), + Vec::::new(), + Vec::::new(), + Vec::::new(), + font::LibData::new(), + )) + .unwrap(); + assert!(matches!( + decode_layer(&non_finite), + Err(StoreError::InvalidLayerPayload(_)) + )); + } + + #[test] + fn duplicate_point_identity_is_rejected() { + let point_id = font::PointId::from_raw("duplicate"); + let mut contour = font::Contour::new(); + contour.push_point(font::Point::new( + point_id.clone(), + 0.0, + 0.0, + font::PointType::OnCurve, + false, + )); + contour.push_point(font::Point::new( + point_id, + 1.0, + 1.0, + font::PointType::OnCurve, + false, + )); + let mut layer = font::GlyphLayer::new( + font::LayerId::from_raw("test"), + font::SourceId::from_raw("test"), + ); + layer.add_contour(contour); + + assert!(matches!( + encode_layer(&layer), + Err(StoreError::InvalidLayerPayload(_)) + )); + } +} diff --git a/crates/shift-store/src/layer/load.rs b/crates/shift-store/src/layer/load.rs new file mode 100644 index 00000000..f18f5f01 --- /dev/null +++ b/crates/shift-store/src/layer/load.rs @@ -0,0 +1,356 @@ +use std::collections::{HashMap, HashSet}; + +use rayon::prelude::*; +use rusqlite::{Connection, OptionalExtension, params}; +use shift_font as font; + +use super::{ + MAX_LAYER_PAYLOAD_BYTES, + directory::{map_directory_facts_row, validate_directory_facts}, + format::decode_layer, + payload::decompress_layer, + references::{validate_component_index, validate_component_rows}, +}; +use crate::{ShiftStore, StoreError}; + +pub const MAX_LAYER_READ_BATCH_COUNT: usize = 512; +pub const MAX_LAYER_READ_BATCH_DECODED_BYTES: u64 = 256 * 1024 * 1024; +const MAX_LAYER_PAYLOAD_BYTES_SQL: i64 = MAX_LAYER_PAYLOAD_BYTES as i64; + +impl ShiftStore { + /// Fetches, bounds-checks, decodes, and cross-checks one canonical layer. + pub fn load_glyph_layer( + &self, + layer_id: &font::LayerId, + ) -> Result, StoreError> { + load_glyph_layer_from_conn(&self.conn, layer_id) + } + + /// Fetches and cross-validates requested layers through count- and + /// decoded-byte-bounded batches. + pub fn load_glyph_layers( + &self, + layer_ids: &[font::LayerId], + ) -> Result, StoreError> { + load_glyph_layers_from_conn(&self.conn, layer_ids) + } +} + +pub(crate) fn load_glyph_layer_from_conn( + conn: &Connection, + layer_id: &font::LayerId, +) -> Result, StoreError> { + let mut directory_stmt = conn.prepare_cached( + " + SELECT l.id, l.source_id, l.width, l.height, + p.inner_format, p.compression, + p.stored_byte_length, p.decoded_byte_length, p.decoded_blake3, + length(p.payload) + FROM glyph_layers AS l + JOIN glyph_layer_payloads AS p ON p.layer_id = l.id + WHERE l.id = ?1 + ", + )?; + let facts = directory_stmt + .query_row([layer_id.to_string()], map_directory_facts_row) + .optional()?; + drop(directory_stmt); + let Some(facts) = facts else { + return Ok(None); + }; + facts.validate()?; + + // The length gate is deliberately a separate query so SQLite cannot + // allocate an unbounded Vec before the store applies its limit. + let mut payload_stmt = conn.prepare_cached( + " + SELECT payload + FROM glyph_layer_payloads + WHERE layer_id = ?1 + AND length(payload) = ?2 + AND length(payload) <= ?3 + ", + )?; + let payload: Vec = payload_stmt + .query_row( + params![ + layer_id.to_string(), + facts.actual_stored_byte_length, + MAX_LAYER_PAYLOAD_BYTES_SQL + ], + |row| row.get(0), + ) + .optional()? + .ok_or_else(|| StoreError::LayerDirectoryMismatch { + layer_id: layer_id.to_string(), + detail: "payload changed during bounded read".to_string(), + })?; + let decoded = decompress_layer(facts.layer_id.as_str(), facts.stored_layer(payload)?)?; + let layer = decode_layer(&decoded)?; + validate_directory_facts( + layer_id, + &facts.source_id, + facts.width, + facts.height, + &layer, + )?; + validate_component_index(conn, &layer)?; + Ok(Some(layer)) +} + +pub(crate) fn load_glyph_layers_from_conn( + conn: &Connection, + layer_ids: &[font::LayerId], +) -> Result, StoreError> { + let mut keys = layer_ids + .iter() + .map(ToString::to_string) + .collect::>(); + keys.sort(); + keys.dedup(); + if keys.is_empty() { + return Ok(Vec::new()); + } + + let mut layers = Vec::with_capacity(keys.len()); + for count_batch in keys.chunks(MAX_LAYER_READ_BATCH_COUNT) { + let placeholders = (0..count_batch.len()) + .map(|_| "?") + .collect::>() + .join(","); + let sql = format!( + "SELECT l.id, p.decoded_byte_length + FROM glyph_layers AS l + JOIN glyph_layer_payloads AS p ON p.layer_id = l.id + WHERE l.id IN ({placeholders}) + ORDER BY l.id" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(rusqlite::params_from_iter(count_batch.iter()), |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + let mut decoded_lengths = HashMap::with_capacity(count_batch.len()); + for row in rows { + let (layer_id, decoded_byte_length) = row?; + if !(0..=MAX_LAYER_PAYLOAD_BYTES_SQL).contains(&decoded_byte_length) { + return Err(StoreError::LayerPayloadTooLarge { + bytes: u64::try_from(decoded_byte_length).unwrap_or(u64::MAX), + limit: MAX_LAYER_PAYLOAD_BYTES as u64, + }); + } + decoded_lengths.insert(layer_id, decoded_byte_length as u64); + } + drop(stmt); + + let mut batch = Vec::new(); + let mut batch_decoded_bytes = 0_u64; + for key in count_batch { + let decoded_byte_length = + decoded_lengths + .get(key) + .copied() + .ok_or_else(|| StoreError::MissingEntity { + kind: "glyph layer", + id: key.clone(), + })?; + if !batch.is_empty() + && batch_decoded_bytes.saturating_add(decoded_byte_length) + > MAX_LAYER_READ_BATCH_DECODED_BYTES + { + layers.extend(load_glyph_layer_batch_from_conn(conn, &batch)?); + batch.clear(); + batch_decoded_bytes = 0; + } + batch.push(font::LayerId::from_raw(key.clone())); + batch_decoded_bytes = batch_decoded_bytes.saturating_add(decoded_byte_length); + } + if !batch.is_empty() { + layers.extend(load_glyph_layer_batch_from_conn(conn, &batch)?); + } + } + + Ok(layers) +} + +pub(super) fn load_glyph_layer_batch_from_conn( + conn: &Connection, + layer_ids: &[font::LayerId], +) -> Result, StoreError> { + let mut keys = layer_ids + .iter() + .map(ToString::to_string) + .collect::>(); + keys.sort(); + keys.dedup(); + if keys.is_empty() { + return Ok(Vec::new()); + } + if keys.len() > MAX_LAYER_READ_BATCH_COUNT { + return Err(StoreError::LayerReadBatchTooLarge { + layers: keys.len(), + limit: MAX_LAYER_READ_BATCH_COUNT, + }); + } + + let placeholders = (0..keys.len()).map(|_| "?").collect::>().join(","); + let directory_sql = format!( + " + SELECT l.id, l.source_id, l.width, l.height, + p.inner_format, p.compression, + p.stored_byte_length, p.decoded_byte_length, p.decoded_blake3, + length(p.payload) + FROM glyph_layers AS l + JOIN glyph_layer_payloads AS p ON p.layer_id = l.id + WHERE l.id IN ({placeholders}) + ORDER BY l.id + " + ); + let mut directory_stmt = conn.prepare(&directory_sql)?; + let directory_rows = directory_stmt.query_map( + rusqlite::params_from_iter(keys.iter()), + map_directory_facts_row, + )?; + let mut facts = HashMap::with_capacity(keys.len()); + let mut batch_decoded_bytes = 0_u64; + for row in directory_rows { + let facts_row = row?; + let decoded_byte_length = facts_row.validate()?; + batch_decoded_bytes = batch_decoded_bytes.saturating_add(decoded_byte_length); + facts.insert(facts_row.layer_id.clone(), facts_row); + } + drop(directory_stmt); + + for key in &keys { + if !facts.contains_key(key) { + return Err(StoreError::MissingEntity { + kind: "glyph layer", + id: key.clone(), + }); + } + } + if batch_decoded_bytes > MAX_LAYER_READ_BATCH_DECODED_BYTES { + return Err(StoreError::LayerReadBatchDecodedTooLarge { + bytes: batch_decoded_bytes, + limit: MAX_LAYER_READ_BATCH_DECODED_BYTES, + }); + } + + let payload_sql = format!( + " + SELECT layer_id, payload, length(payload) + FROM glyph_layer_payloads + WHERE layer_id IN ({placeholders}) + AND length(payload) <= ? + ORDER BY layer_id + " + ); + let mut payload_params = keys + .iter() + .cloned() + .map(rusqlite::types::Value::Text) + .collect::>(); + payload_params.push(rusqlite::types::Value::Integer(MAX_LAYER_PAYLOAD_BYTES_SQL)); + let mut payload_stmt = conn.prepare(&payload_sql)?; + let payload_rows = + payload_stmt.query_map(rusqlite::params_from_iter(payload_params), |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Vec>(1)?, + row.get::<_, i64>(2)?, + )) + })?; + let mut payloads = Vec::with_capacity(keys.len()); + for row in payload_rows { + let (layer_id, payload, actual_len) = row?; + let facts = facts + .get(&layer_id) + .ok_or_else(|| StoreError::LayerDirectoryMismatch { + layer_id: layer_id.clone(), + detail: "payload has no directory row".to_string(), + })?; + if actual_len != facts.actual_stored_byte_length + || payload.len() as i64 != facts.actual_stored_byte_length + { + return Err(StoreError::LayerDirectoryMismatch { + layer_id, + detail: "payload changed during bounded batch read".to_string(), + }); + } + payloads.push((layer_id, payload)); + } + drop(payload_stmt); + + let decoded = payloads + .into_par_iter() + .map(|(layer_id, payload)| { + let facts = facts + .get(&layer_id) + .ok_or_else(|| StoreError::LayerDirectoryMismatch { + layer_id: layer_id.clone(), + detail: "payload has no directory row".to_string(), + })?; + let decoded = decompress_layer(layer_id.as_str(), facts.stored_layer(payload)?)?; + let layer = decode_layer(&decoded)?; + let expected_id = font::LayerId::from_raw(layer_id.clone()); + validate_directory_facts( + &expected_id, + &facts.source_id, + facts.width, + facts.height, + &layer, + )?; + Ok((layer_id, layer)) + }) + .collect::, StoreError>>()?; + let loaded = decoded + .iter() + .map(|(layer_id, _)| layer_id.clone()) + .collect::>(); + let layers = decoded + .into_iter() + .map(|(_, layer)| layer) + .collect::>(); + + for key in &keys { + if !loaded.contains(key) { + return Err(StoreError::LayerDirectoryMismatch { + layer_id: key.clone(), + detail: "payload changed during bounded batch read".to_string(), + }); + } + } + + let component_sql = format!( + " + SELECT layer_id, id, base_glyph_id, order_index + FROM glyph_components + WHERE layer_id IN ({placeholders}) + ORDER BY layer_id, order_index, id + " + ); + let mut component_stmt = conn.prepare(&component_sql)?; + let component_rows = + component_stmt.query_map(rusqlite::params_from_iter(keys.iter()), |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + )) + })?; + let mut components: HashMap> = HashMap::new(); + for row in component_rows { + let (layer_id, component_id, base_glyph_id, order_index) = row?; + components + .entry(layer_id) + .or_default() + .push((component_id, base_glyph_id, order_index)); + } + drop(component_stmt); + + for layer in &layers { + let indexed = components.remove(layer.id().as_str()).unwrap_or_default(); + validate_component_rows(layer, &indexed)?; + } + Ok(layers) +} diff --git a/crates/shift-store/src/layer/mod.rs b/crates/shift-store/src/layer/mod.rs new file mode 100644 index 00000000..7f29803e --- /dev/null +++ b/crates/shift-store/src/layer/mod.rs @@ -0,0 +1,36 @@ +mod directory; +mod format; +mod load; +mod payload; +mod references; +mod write; + +#[cfg(test)] +mod tests; + +#[cfg(test)] +use load::load_glyph_layer_batch_from_conn; +#[cfg(test)] +use payload::{LayerPayloadCompression, decompress_layer}; + +pub use directory::GlyphLayerDirectoryEntry; +pub use format::GLYPH_LAYER_FORMAT; +pub use load::{MAX_LAYER_READ_BATCH_COUNT, MAX_LAYER_READ_BATCH_DECODED_BYTES}; + +pub(crate) use format::encode_layer; +pub(crate) use load::{load_glyph_layer_from_conn, load_glyph_layers_from_conn}; +pub(crate) use payload::{StoredLayerPayload, compress_layer}; +pub(crate) use write::{ + create_empty_layer_in_tx, rewrite_layer_in_tx, store_stored_layer_in_tx, write_layer_in_tx, +}; + +pub(crate) const MAX_LAYER_PAYLOAD_BYTES: usize = 256 * 1024 * 1024; + +fn check_layer_length(bytes: u64) -> Result<(), crate::StoreError> { + let limit = MAX_LAYER_PAYLOAD_BYTES as u64; + if bytes > limit { + return Err(crate::StoreError::LayerPayloadTooLarge { bytes, limit }); + } + + Ok(()) +} diff --git a/crates/shift-store/src/layer/payload.rs b/crates/shift-store/src/layer/payload.rs new file mode 100644 index 00000000..c1799b96 --- /dev/null +++ b/crates/shift-store/src/layer/payload.rs @@ -0,0 +1,216 @@ +use std::cell::RefCell; + +use super::{MAX_LAYER_PAYLOAD_BYTES, check_layer_length}; +use crate::StoreError; + +const ZSTD_COMPRESSION_LEVEL: i32 = 1; + +thread_local! { + static ZSTD_COMPRESSOR: RefCell>> = const { RefCell::new(None) }; + static ZSTD_DECOMPRESSOR: RefCell>> = const { RefCell::new(None) }; +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum LayerPayloadCompression { + None, + ZstandardV1, +} + +impl LayerPayloadCompression { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::ZstandardV1 => "zstd.v1", + } + } +} + +impl TryFrom<&str> for LayerPayloadCompression { + type Error = StoreError; + + fn try_from(value: &str) -> Result { + match value { + "none" => Ok(Self::None), + "zstd.v1" => Ok(Self::ZstandardV1), + value => Err(StoreError::UnsupportedLayerCompression(value.to_owned())), + } + } +} + +pub(crate) struct StoredLayerPayload { + pub(crate) compression: LayerPayloadCompression, + pub(crate) bytes: Vec, + pub(crate) stored_byte_length: u64, + pub(crate) decoded_byte_length: u64, + pub(crate) decoded_blake3: [u8; 32], +} + +pub(crate) fn compress_layer(decoded: &[u8]) -> Result { + check_layer_length(decoded.len() as u64)?; + let compressed = ZSTD_COMPRESSOR.with(|slot| { + let mut slot = slot.borrow_mut(); + if slot.is_none() { + *slot = Some( + zstd::bulk::Compressor::new(ZSTD_COMPRESSION_LEVEL) + .map_err(|error| StoreError::LayerCompression(error.to_string()))?, + ); + } + slot.as_mut() + .expect("compressor was initialized") + .compress(decoded) + .map_err(|error| StoreError::LayerCompression(error.to_string())) + })?; + let decoded_byte_length = decoded.len() as u64; + let decoded_blake3 = *blake3::hash(decoded).as_bytes(); + + if compressed.len() < decoded.len() { + return Ok(StoredLayerPayload { + compression: LayerPayloadCompression::ZstandardV1, + stored_byte_length: compressed.len() as u64, + decoded_byte_length, + decoded_blake3, + bytes: compressed, + }); + } + + Ok(StoredLayerPayload { + compression: LayerPayloadCompression::None, + stored_byte_length: decoded_byte_length, + decoded_byte_length, + decoded_blake3, + bytes: decoded.to_vec(), + }) +} + +pub(crate) fn decompress_layer( + layer_id: &str, + stored: StoredLayerPayload, +) -> Result, StoreError> { + check_layer_length(stored.stored_byte_length)?; + check_layer_length(stored.decoded_byte_length)?; + if stored.bytes.len() as u64 != stored.stored_byte_length { + return Err(directory_mismatch( + layer_id, + format!( + "stored byte length {} != actual {}", + stored.stored_byte_length, + stored.bytes.len() + ), + )); + } + + let decoded = match stored.compression { + LayerPayloadCompression::None => stored.bytes, + LayerPayloadCompression::ZstandardV1 => { + let frame_length = + zstd::zstd_safe::find_frame_compressed_size(&stored.bytes).map_err(|code| { + StoreError::LayerDecompression(zstd::zstd_safe::get_error_name(code).to_owned()) + })?; + if frame_length != stored.bytes.len() { + return Err(directory_mismatch( + layer_id, + format!( + "zstd frame has {frame_length} bytes, stored payload has {}", + stored.bytes.len() + ), + )); + } + let decoded_capacity = usize::try_from(stored.decoded_byte_length).map_err(|_| { + StoreError::LayerPayloadTooLarge { + bytes: stored.decoded_byte_length, + limit: MAX_LAYER_PAYLOAD_BYTES as u64, + } + })?; + ZSTD_DECOMPRESSOR.with(|slot| { + let mut slot = slot.borrow_mut(); + if slot.is_none() { + *slot = Some( + zstd::bulk::Decompressor::new() + .map_err(|error| StoreError::LayerDecompression(error.to_string()))?, + ); + } + slot.as_mut() + .expect("decompressor was initialized") + .decompress(&stored.bytes, decoded_capacity) + .map_err(|error| StoreError::LayerDecompression(error.to_string())) + })? + } + }; + + if decoded.len() as u64 != stored.decoded_byte_length { + return Err(directory_mismatch( + layer_id, + format!( + "decoded byte length {} != actual {}", + stored.decoded_byte_length, + decoded.len() + ), + )); + } + if blake3::hash(&decoded).as_bytes() != &stored.decoded_blake3 { + return Err(directory_mismatch(layer_id, "decoded BLAKE3 mismatch")); + } + Ok(decoded) +} + +fn directory_mismatch(layer_id: &str, detail: impl Into) -> StoreError { + StoreError::LayerDirectoryMismatch { + layer_id: layer_id.to_owned(), + detail: detail.into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn incompressible_bytes_fall_back_to_none() { + let mut value = 0x1234_5678_u32; + let decoded = (0..64) + .map(|_| { + value = value.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (value >> 24) as u8 + }) + .collect::>(); + + let stored = compress_layer(&decoded).unwrap(); + + assert_eq!(stored.compression, LayerPayloadCompression::None); + assert_eq!(decompress_layer("layer_test", stored).unwrap(), decoded); + } + + #[test] + fn compressed_bytes_reject_truncation_trailing_data_and_wrong_hash() { + let decoded = vec![0; 4096]; + let mut truncated = compress_layer(&decoded).unwrap(); + assert_eq!(truncated.compression, LayerPayloadCompression::ZstandardV1); + truncated.bytes.pop(); + truncated.stored_byte_length -= 1; + assert!(decompress_layer("layer_test", truncated).is_err()); + + let mut trailing = compress_layer(&decoded).unwrap(); + trailing.bytes.push(0); + trailing.stored_byte_length += 1; + assert!(decompress_layer("layer_test", trailing).is_err()); + + let mut wrong_hash = compress_layer(&decoded).unwrap(); + wrong_hash.decoded_blake3[0] ^= 1; + assert!(decompress_layer("layer_test", wrong_hash).is_err()); + } + + #[test] + fn unsupported_compression_and_oversized_decode_are_rejected() { + assert!(matches!( + LayerPayloadCompression::try_from("zstd.v2"), + Err(StoreError::UnsupportedLayerCompression(_)) + )); + + let mut stored = compress_layer(&vec![0; 4096]).unwrap(); + stored.decoded_byte_length = MAX_LAYER_PAYLOAD_BYTES as u64 + 1; + assert!(matches!( + decompress_layer("layer_test", stored), + Err(StoreError::LayerPayloadTooLarge { .. }) + )); + } +} diff --git a/crates/shift-store/src/layer/references.rs b/crates/shift-store/src/layer/references.rs new file mode 100644 index 00000000..11fe0a9c --- /dev/null +++ b/crates/shift-store/src/layer/references.rs @@ -0,0 +1,154 @@ +use std::collections::{HashMap, HashSet, VecDeque}; + +use rusqlite::Connection; +use shift_font as font; + +use crate::{ShiftStore, StoreError}; + +impl ShiftStore { + /// Returns every direct glyph-component edge from one ordered relational + /// scan. Glyphs without components are absent from the map. + pub fn glyph_component_references( + &self, + ) -> Result>, StoreError> { + let mut stmt = self.conn.prepare( + " + SELECT DISTINCT l.glyph_id, c.base_glyph_id + FROM glyph_components AS c + JOIN glyph_layers AS l ON l.id = c.layer_id + ORDER BY l.glyph_id, c.base_glyph_id + ", + )?; + let rows = stmt.query_map([], |row| { + Ok(( + font::GlyphId::from_raw(row.get::<_, String>(0)?), + font::GlyphId::from_raw(row.get::<_, String>(1)?), + )) + })?; + let mut references = HashMap::new(); + for row in rows { + let (glyph_id, base_glyph_id) = row?; + references + .entry(glyph_id) + .or_insert_with(Vec::new) + .push(base_glyph_id); + } + Ok(references) + } + + /// Returns referenced glyph identities entirely from the relational index. + pub fn referenced_glyph_ids_for_glyph( + &self, + glyph_id: &font::GlyphId, + ) -> Result, StoreError> { + let mut stmt = self.conn.prepare( + " + SELECT DISTINCT c.base_glyph_id + FROM glyph_components AS c + JOIN glyph_layers AS l ON l.id = c.layer_id + WHERE l.glyph_id = ?1 + ORDER BY c.base_glyph_id + ", + )?; + let rows = stmt.query_map([glyph_id.to_string()], |row| { + Ok(font::GlyphId::from_raw(row.get::<_, String>(0)?)) + })?; + rows.collect::, _>>().map_err(Into::into) + } + + /// Computes the transitive component closure without scanning payloads. + pub fn referenced_glyph_closure( + &self, + roots: impl IntoIterator, + ) -> Result, StoreError> { + let mut pending: VecDeque<_> = roots.into_iter().collect(); + let mut seen = HashSet::new(); + let mut result = Vec::new(); + while let Some(glyph_id) = pending.pop_front() { + if !seen.insert(glyph_id.clone()) { + continue; + } + pending.extend(self.referenced_glyph_ids_for_glyph(&glyph_id)?); + result.push(glyph_id); + } + Ok(result) + } + + /// Reverse-reference lookup used for dependent invalidation. + pub fn dependent_glyph_ids_for_layers( + &self, + layer_ids: &[font::LayerId], + ) -> Result, StoreError> { + if layer_ids.is_empty() { + return Ok(Vec::new()); + } + let mut result = HashSet::new(); + let mut stmt = self.conn.prepare( + " + SELECT DISTINCT owner.glyph_id + FROM glyph_components AS c + JOIN glyph_layers AS target ON target.glyph_id = c.base_glyph_id + JOIN glyph_layers AS owner ON owner.id = c.layer_id + WHERE target.id = ?1 + ", + )?; + for layer_id in layer_ids { + let rows = stmt.query_map([layer_id.to_string()], |row| { + Ok(font::GlyphId::from_raw(row.get::<_, String>(0)?)) + })?; + for row in rows { + result.insert(row?); + } + } + let mut result: Vec<_> = result.into_iter().collect(); + result.sort_by(|left, right| left.as_str().cmp(right.as_str())); + Ok(result) + } +} + +pub(super) fn validate_component_index( + conn: &Connection, + layer: &font::GlyphLayer, +) -> Result<(), StoreError> { + let mut stmt = conn.prepare_cached( + " + SELECT id, base_glyph_id, order_index + FROM glyph_components + WHERE layer_id = ?1 + ORDER BY order_index, id + ", + )?; + let indexed = stmt + .query_map([layer.id().to_string()], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + })? + .collect::, _>>()?; + validate_component_rows(layer, &indexed) +} + +pub(super) fn validate_component_rows( + layer: &font::GlyphLayer, + indexed: &[(String, String, i64)], +) -> Result<(), StoreError> { + let authored: Vec<_> = layer + .components_iter() + .enumerate() + .map(|(order_index, component)| { + ( + component.id().to_string(), + component.base_glyph_id().to_string(), + order_index as i64, + ) + }) + .collect(); + if indexed != authored { + return Err(StoreError::StaleLayerReferenceIndex { + layer_id: layer.id().to_string(), + }); + } + Ok(()) +} diff --git a/crates/shift-store/src/layer/tests.rs b/crates/shift-store/src/layer/tests.rs new file mode 100644 index 00000000..a56e8e6d --- /dev/null +++ b/crates/shift-store/src/layer/tests.rs @@ -0,0 +1,669 @@ +use std::time::{Duration, Instant}; + +use rusqlite::params; +use shift_font as font; + +use super::*; +use crate::{ShiftStore, StoreError, WorkspaceState, write_mode::WriteMode}; + +fn populated_store() -> (ShiftStore, font::Font) { + let mut store = ShiftStore::open_memory_for_test().unwrap(); + let font = font::test_support::sample_font(); + store.replace_font_state(&font).unwrap(); + store + .set_workspace_state(WorkspaceState::untitled(None)) + .unwrap(); + (store, font) +} + +#[test] +fn sqlite_payload_preserves_canonical_bytes() { + let font = font::test_support::sample_font(); + let layer = font + .glyph_by_name("A") + .unwrap() + .layers() + .values() + .next() + .unwrap(); + let encoded = encode_layer(layer).unwrap(); + let glyph_id = font::GlyphId::from_raw("storage_golden"); + let glyph_name = font::GlyphName::from("storageGolden"); + let mut store = ShiftStore::open_memory_for_test().unwrap(); + store + .conn + .execute( + "INSERT INTO sources (id, name, kind, order_index) VALUES (?1, 'Golden', 'master', 0)", + [layer.source_id().to_string()], + ) + .unwrap(); + store + .conn + .execute( + "INSERT INTO glyphs (id, name, order_index) VALUES (?1, ?2, 0)", + params![glyph_id.to_string(), glyph_name.as_str()], + ) + .unwrap(); + + let tx = store.conn.transaction().unwrap(); + write_layer_in_tx(&tx, &glyph_id, layer).unwrap(); + tx.commit().unwrap(); + + let stored = store + .conn + .query_row( + "SELECT compression, payload, stored_byte_length, + decoded_byte_length, decoded_blake3 + FROM glyph_layer_payloads WHERE layer_id = ?1", + [layer.id().to_string()], + |row| { + Ok(StoredLayerPayload { + compression: match row.get::<_, String>(0)?.as_str() { + "zstd.v1" => LayerPayloadCompression::ZstandardV1, + _ => LayerPayloadCompression::None, + }, + bytes: row.get(1)?, + stored_byte_length: row.get::<_, i64>(2)? as u64, + decoded_byte_length: row.get::<_, i64>(3)? as u64, + decoded_blake3: row.get::<_, Vec>(4)?.try_into().unwrap(), + }) + }, + ) + .unwrap(); + assert_eq!(stored.compression, LayerPayloadCompression::ZstandardV1); + assert!(stored.stored_byte_length < stored.decoded_byte_length); + assert_eq!( + decompress_layer(layer.id().as_str(), stored).unwrap(), + encoded + ); + let decoded = store.load_glyph_layer(&layer.id()).unwrap().unwrap(); + assert_eq!(encode_layer(&decoded).unwrap(), encoded); +} + +#[test] +fn bounded_batch_load_matches_complete_font_layers() { + let (store, font) = populated_store(); + let mut layer_ids = font + .glyphs() + .flat_map(|glyph| glyph.layers().keys().cloned()) + .collect::>(); + layer_ids.push(layer_ids[0].clone()); + + let layers = store.load_glyph_layers(&layer_ids).unwrap(); + + assert_eq!(layers.len(), layer_ids.len() - 1); + for layer in layers { + assert_eq!(font.layer(layer.id()).unwrap(), &layer); + } +} + +#[test] +fn requested_layers_load_across_multiple_count_bounded_batches() { + let mut font = font::Font::new(); + let source_id = font.default_source_id().unwrap(); + let mut layer_ids = Vec::new(); + for index in 0..=MAX_LAYER_READ_BATCH_COUNT { + let layer = font::GlyphLayer::with_width( + font::LayerId::from_raw(format!("layer_{index:04}")), + source_id.clone(), + index as f64, + ); + layer_ids.push(layer.id()); + let mut glyph = font::Glyph::new(format!("glyph_{index:04}")); + glyph.set_layer(layer); + font.insert_glyph(glyph).unwrap(); + } + let mut store = ShiftStore::open_memory_for_test().unwrap(); + store.replace_font_state(&font).unwrap(); + + let layers = store.load_glyph_layers(&layer_ids).unwrap(); + + assert_eq!(layers.len(), layer_ids.len()); + assert_eq!(layers.first().unwrap().id(), layer_ids[0]); + assert_eq!(layers.last().unwrap().id(), layer_ids[512]); +} + +#[test] +fn one_internal_batch_rejects_excessive_layer_counts_before_sql() { + let store = ShiftStore::open_memory_for_test().unwrap(); + let layer_ids = (0..=MAX_LAYER_READ_BATCH_COUNT) + .map(|index| font::LayerId::from_raw(format!("layer_{index}"))) + .collect::>(); + + assert!(matches!( + load_glyph_layer_batch_from_conn(&store.conn, &layer_ids), + Err(StoreError::LayerReadBatchTooLarge { .. }) + )); +} + +#[test] +fn batch_bounds_declared_decoded_bytes_before_fetching_payloads() { + let (store, font) = populated_store(); + let layer_ids = font + .glyphs() + .flat_map(|glyph| glyph.layers().keys().cloned()) + .take(2) + .collect::>(); + let decoded_byte_length = MAX_LAYER_READ_BATCH_DECODED_BYTES / 2 + 1; + for layer_id in &layer_ids { + store + .conn + .execute( + "UPDATE glyph_layer_payloads + SET decoded_byte_length = ?1 + WHERE layer_id = ?2", + params![decoded_byte_length as i64, layer_id.to_string()], + ) + .unwrap(); + } + + assert!(matches!( + load_glyph_layer_batch_from_conn(&store.conn, &layer_ids), + Err(StoreError::LayerReadBatchDecodedTooLarge { .. }) + )); +} + +#[test] +fn mixed_compressed_and_uncompressed_layers_load_in_stable_order() { + let (store, font) = populated_store(); + let mut layer_ids = font + .glyphs() + .flat_map(|glyph| glyph.layers().keys().cloned()) + .take(2) + .collect::>(); + layer_ids.sort_by(|left, right| left.as_str().cmp(right.as_str())); + let uncompressed_layer = font.layer(layer_ids[0].clone()).unwrap(); + let encoded = encode_layer(uncompressed_layer).unwrap(); + let decoded_blake3 = blake3::hash(&encoded); + store + .conn + .execute( + "UPDATE glyph_layer_payloads + SET compression = 'none', payload = ?1, + stored_byte_length = ?2, decoded_byte_length = ?2, + decoded_blake3 = ?3 + WHERE layer_id = ?4", + params![ + encoded.as_slice(), + encoded.len() as i64, + decoded_blake3.as_bytes().as_slice(), + layer_ids[0].to_string(), + ], + ) + .unwrap(); + + let loaded = store.load_glyph_layers(&layer_ids).unwrap(); + + assert_eq!(loaded.len(), 2); + assert_eq!(loaded[0].id(), layer_ids[0]); + assert_eq!(loaded[1].id(), layer_ids[1]); + for layer in loaded { + assert_eq!(font.layer(layer.id()).unwrap(), &layer); + } +} + +#[test] +fn declared_stored_length_mismatch_is_rejected_before_decode() { + let (store, font) = populated_store(); + let layer_id = font + .glyphs() + .next() + .unwrap() + .layers() + .keys() + .next() + .unwrap(); + store + .conn + .pragma_update(None, "ignore_check_constraints", "ON") + .unwrap(); + store + .conn + .execute( + "UPDATE glyph_layer_payloads + SET stored_byte_length = stored_byte_length + 1 + WHERE layer_id = ?1", + [layer_id.to_string()], + ) + .unwrap(); + store + .conn + .pragma_update(None, "ignore_check_constraints", "OFF") + .unwrap(); + + assert!(matches!( + store.load_glyph_layer(layer_id), + Err(StoreError::LayerDirectoryMismatch { detail, .. }) + if detail.contains("declared stored byte length") + )); + assert!(matches!( + store.load_glyph_layers(std::slice::from_ref(layer_id)), + Err(StoreError::LayerDirectoryMismatch { detail, .. }) + if detail.contains("declared stored byte length") + )); +} + +#[test] +fn layer_rewrite_transitions_from_uncompressed_to_zstd_consistently() { + let (mut store, font) = populated_store(); + let layer = font + .glyphs() + .next() + .unwrap() + .layers() + .values() + .next() + .unwrap(); + let mut compressible_layer = layer.as_ref().clone(); + compressible_layer.lib_mut().set( + "com.shift.compression-fixture".to_string(), + font::LibValue::Data(vec![0; 8 * 1024]), + ); + let encoded = encode_layer(&compressible_layer).unwrap(); + let uncompressed = StoredLayerPayload { + compression: LayerPayloadCompression::None, + bytes: encoded.clone(), + stored_byte_length: encoded.len() as u64, + decoded_byte_length: encoded.len() as u64, + decoded_blake3: *blake3::hash(&encoded).as_bytes(), + }; + let glyph_id = font.glyph_id_by_layer(layer.id()).unwrap(); + let tx = store.conn.transaction().unwrap(); + store_stored_layer_in_tx( + &tx, + &glyph_id, + &compressible_layer, + &uncompressed, + WriteMode::Upsert, + ) + .unwrap(); + tx.commit().unwrap(); + assert_eq!( + store + .conn + .query_row( + "SELECT compression FROM glyph_layer_payloads WHERE layer_id = ?1", + [layer.id().to_string()], + |row| row.get::<_, String>(0), + ) + .unwrap(), + "none" + ); + + store.replace_glyph_layer(&compressible_layer).unwrap(); + + let (compression, stored, decoded, hash_length, payload_length) = store + .conn + .query_row( + "SELECT compression, stored_byte_length, decoded_byte_length, + length(decoded_blake3), length(payload) + FROM glyph_layer_payloads WHERE layer_id = ?1", + [layer.id().to_string()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + )) + }, + ) + .unwrap(); + assert_eq!(compression, "zstd.v1"); + assert_eq!(stored, payload_length); + assert!(stored < decoded); + assert_eq!(hash_length, 32); + assert_eq!( + store.load_glyph_layer(&layer.id()).unwrap().unwrap(), + compressible_layer + ); +} + +#[test] +fn false_decoded_length_and_hash_are_rejected() { + let (store, font) = populated_store(); + let layer_id = font + .glyphs() + .next() + .unwrap() + .layers() + .keys() + .next() + .unwrap(); + store + .conn + .execute( + "UPDATE glyph_layer_payloads + SET decoded_byte_length = decoded_byte_length + 1 + WHERE layer_id = ?1", + [layer_id.to_string()], + ) + .unwrap(); + assert!(matches!( + store.load_glyph_layer(layer_id), + Err(StoreError::LayerDirectoryMismatch { detail, .. }) + if detail.contains("decoded byte length") + )); + + let encoded = encode_layer(font.layer(layer_id.clone()).unwrap()).unwrap(); + store + .conn + .execute( + "UPDATE glyph_layer_payloads + SET decoded_byte_length = ?1, decoded_blake3 = zeroblob(32) + WHERE layer_id = ?2", + params![encoded.len() as i64, layer_id.to_string()], + ) + .unwrap(); + assert!(matches!( + store.load_glyph_layer(layer_id), + Err(StoreError::LayerDirectoryMismatch { detail, .. }) + if detail.contains("decoded BLAKE3 mismatch") + )); +} + +#[test] +fn directory_open_never_reads_malformed_payloads() { + let (store, font) = populated_store(); + let layer_id = font + .glyphs() + .next() + .unwrap() + .layers() + .keys() + .next() + .unwrap(); + store + .conn + .execute( + "UPDATE glyph_layer_payloads + SET payload = X'00', stored_byte_length = 1 + WHERE layer_id = ?1", + [layer_id.to_string()], + ) + .unwrap(); + + let directory = store.load_font_directory().unwrap(); + assert_eq!(directory.glyph_count(), font.glyph_count()); + assert!(directory.layer(layer_id.clone()).unwrap().is_empty()); + assert!(store.load_glyph_layer(layer_id).is_err()); + assert!( + store + .load_glyph_layers(std::slice::from_ref(layer_id)) + .is_err() + ); +} + +#[test] +fn directory_retains_layer_when_payload_row_is_missing() { + let (store, font) = populated_store(); + let layer_id = font + .glyphs() + .next() + .unwrap() + .layers() + .keys() + .next() + .unwrap(); + store + .conn + .execute( + "DELETE FROM glyph_layer_payloads WHERE layer_id = ?1", + [layer_id.to_string()], + ) + .unwrap(); + + let directory = store.load_font_directory().unwrap(); + + assert!(directory.layer(layer_id.clone()).is_some()); + assert!(store.load_font_state().is_err()); +} + +#[test] +fn stale_reference_index_is_rejected() { + let (store, font) = populated_store(); + let layer = font + .glyphs() + .flat_map(|glyph| glyph.layers().values()) + .find(|layer| !layer.components().is_empty()) + .unwrap(); + store + .conn + .execute( + "DELETE FROM glyph_components WHERE layer_id = ?1", + [layer.id().to_string()], + ) + .unwrap(); + + assert!(matches!( + store.load_glyph_layer(&layer.id()), + Err(StoreError::StaleLayerReferenceIndex { .. }) + )); + assert!(matches!( + store.load_glyph_layers(&[layer.id()]), + Err(StoreError::StaleLayerReferenceIndex { .. }) + )); +} + +#[test] +fn index_constraint_failure_rolls_back_payload_directory_and_revision() { + let (mut store, font) = populated_store(); + let indexed_layer = font + .glyphs() + .flat_map(|glyph| glyph.layers().values()) + .find(|layer| !layer.components().is_empty()) + .unwrap(); + let duplicate = indexed_layer.components_iter().next().unwrap(); + let mut target = font + .glyphs() + .flat_map(|glyph| glyph.layers().values()) + .find(|layer| layer.id() != indexed_layer.id() && layer.components().is_empty()) + .unwrap() + .as_ref() + .clone(); + target.add_component(font::Component::with_id( + duplicate.id(), + duplicate.base_glyph_id(), + duplicate.base_glyph_name().clone(), + *duplicate.transform(), + )); + + let before_payload: Vec = store + .conn + .query_row( + "SELECT payload FROM glyph_layer_payloads WHERE layer_id = ?1", + [target.id().to_string()], + |row| row.get(0), + ) + .unwrap(); + let before_revision = store.workspace_state().unwrap().unwrap().revision; + + assert!(store.replace_glyph_layer(&target).is_err()); + + let after_payload: Vec = store + .conn + .query_row( + "SELECT payload FROM glyph_layer_payloads WHERE layer_id = ?1", + [target.id().to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(after_payload, before_payload); + assert_eq!( + store.workspace_state().unwrap().unwrap().revision, + before_revision + ); +} + +#[test] +fn replacement_updates_payload_directory_references_and_revision_atomically() { + let (mut store, font) = populated_store(); + let mut layer = font + .glyphs() + .flat_map(|glyph| glyph.layers().values()) + .find(|layer| !layer.components().is_empty()) + .unwrap() + .as_ref() + .clone(); + let owner = font.glyph_id_by_layer(layer.id()).unwrap(); + layer.set_width(layer.width() + 23.0); + layer.clear_components(); + + store.replace_glyph_layer(&layer).unwrap(); + + assert_eq!( + store.load_glyph_layer(&layer.id()).unwrap(), + Some(layer.clone()) + ); + let directory = store + .list_glyph_layer_directory() + .unwrap() + .into_iter() + .find(|entry| entry.layer_id == layer.id()) + .unwrap(); + assert_eq!(directory.width.to_bits(), layer.width().to_bits()); + assert!( + store + .referenced_glyph_ids_for_glyph(&owner) + .unwrap() + .is_empty() + ); + assert_eq!(store.workspace_state().unwrap().unwrap().revision, 1); +} + +#[test] +fn post_font_change_set_rewrites_touched_layer_without_decoding_old_payload() { + let (mut store, mut post_font) = populated_store(); + let mut layer = post_font + .glyphs() + .flat_map(|glyph| glyph.layers().values()) + .next() + .unwrap() + .as_ref() + .clone(); + layer.set_width(layer.width() + 41.0); + post_font.replace_glyph_layers(vec![layer.clone()]).unwrap(); + store + .conn + .execute( + "UPDATE glyph_layer_payloads + SET payload = X'00', stored_byte_length = 1 + WHERE layer_id = ?1", + [layer.id().to_string()], + ) + .unwrap(); + let changes = font::FontChangeSet::new(vec![font::FontChange::layer_metrics_changed(&layer)]); + + store + .apply_change_set_with_font(&changes, &post_font) + .unwrap(); + + assert_eq!(store.load_glyph_layer(&layer.id()).unwrap(), Some(layer)); +} + +#[test] +fn abandoned_transaction_preserves_last_committed_layer() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("working.sqlite"); + let font = font::test_support::sample_font(); + let layer = font + .glyphs() + .flat_map(|glyph| glyph.layers().values()) + .next() + .unwrap() + .as_ref() + .clone(); + let mut store = ShiftStore::open(&path).unwrap(); + store.replace_font_state(&font).unwrap(); + let mut uncommitted = layer.clone(); + uncommitted.set_width(layer.width() + 999.0); + { + let tx = store.conn.transaction().unwrap(); + rewrite_layer_in_tx(&tx, &uncommitted).unwrap(); + // Dropping an open transaction models recovery from a process + // that exits before COMMIT reaches the WAL. + } + drop(store); + + let reopened = ShiftStore::open(&path).unwrap(); + assert_eq!(reopened.load_glyph_layer(&layer.id()).unwrap(), Some(layer)); +} + +#[test] +fn abandoned_font_stream_rolls_back_header_and_glyphs() { + let (mut store, font) = populated_store(); + let before = store.list_glyph_layer_directory().unwrap(); + { + let mut writer = store.begin_import(&font).unwrap(); + let glyph = font.glyphs().next().unwrap(); + writer + .write_glyph_batch(std::slice::from_ref(glyph)) + .unwrap(); + } + + assert_eq!(store.list_glyph_layer_directory().unwrap(), before); +} + +#[test] +fn cjk_scale_directory_open_is_payload_independent() { + const GLYPH_COUNT: usize = 65_536; + let mut store = ShiftStore::open_memory_for_test().unwrap(); + let source_id = font::SourceId::from_raw("cjk").to_string(); + store + .conn + .execute( + "INSERT INTO sources (id, name, kind, order_index) VALUES (?1, 'CJK', 'master', 0)", + [&source_id], + ) + .unwrap(); + let tx = store.conn.transaction().unwrap(); + { + let mut glyph_stmt = tx + .prepare("INSERT INTO glyphs (id, name, order_index) VALUES (?1, ?2, ?3)") + .unwrap(); + let mut layer_stmt = tx + .prepare("INSERT INTO glyph_layers (id, glyph_id, source_id, width, height) VALUES (?1, ?2, ?3, 1000, NULL)") + .unwrap(); + let mut payload_stmt = tx + .prepare( + "INSERT INTO glyph_layer_payloads ( + layer_id, inner_format, compression, payload, + stored_byte_length, decoded_byte_length, decoded_blake3 + ) VALUES (?1, ?2, 'none', X'00', 1, 1, zeroblob(32))", + ) + .unwrap(); + for index in 0..GLYPH_COUNT { + let glyph_id = font::GlyphId::from_raw(format!("cjk{index}")).to_string(); + let layer_id = font::LayerId::from_raw(format!("cjk{index}")).to_string(); + let name = format!("uni{index:05X}"); + glyph_stmt + .execute(params![glyph_id, name, index as i64]) + .unwrap(); + layer_stmt + .execute(params![layer_id, glyph_id, source_id]) + .unwrap(); + payload_stmt + .execute(params![layer_id, GLYPH_LAYER_FORMAT]) + .unwrap(); + } + } + tx.commit().unwrap(); + + let started = Instant::now(); + let directory = store.load_font_directory().unwrap(); + let elapsed = started.elapsed(); + + assert_eq!(directory.glyph_count(), GLYPH_COUNT); + assert_eq!( + directory + .glyphs() + .map(|glyph| glyph.layers().len()) + .sum::(), + GLYPH_COUNT + ); + assert!( + elapsed < Duration::from_secs(15), + "directory open took {elapsed:?}" + ); +} diff --git a/crates/shift-store/src/layer/write.rs b/crates/shift-store/src/layer/write.rs new file mode 100644 index 00000000..2e42e6de --- /dev/null +++ b/crates/shift-store/src/layer/write.rs @@ -0,0 +1,178 @@ +use rusqlite::{Connection, OptionalExtension, Transaction, params}; +use shift_font as font; + +use super::{ + GLYPH_LAYER_FORMAT, MAX_LAYER_PAYLOAD_BYTES, + format::encode_layer, + payload::{StoredLayerPayload, compress_layer}, +}; +use crate::{ + ShiftStore, StoreError, workspace_state::mark_workspace_dirty_in_tx, write_mode::WriteMode, +}; + +impl ShiftStore { + /// Replaces one independently-addressed layer and its earned reference + /// rows in one transaction. The workspace revision advances only after + /// encoding and index derivation have succeeded. + pub fn replace_glyph_layer(&mut self, layer: &font::GlyphLayer) -> Result<(), StoreError> { + let owner = + layer_owner(&self.conn, &layer.id())?.ok_or_else(|| StoreError::MissingEntity { + kind: "glyph layer", + id: layer.id().to_string(), + })?; + let tx = self.conn.transaction()?; + write_layer_in_tx(&tx, &owner, layer)?; + mark_workspace_dirty_in_tx(&tx)?; + tx.commit()?; + Ok(()) + } +} + +pub(crate) fn write_layer_in_tx( + tx: &Transaction<'_>, + glyph_id: &font::GlyphId, + layer: &font::GlyphLayer, +) -> Result<(), StoreError> { + let encoded = encode_layer(layer)?; + let stored = compress_layer(&encoded)?; + store_stored_layer_in_tx(tx, glyph_id, layer, &stored, WriteMode::Upsert) +} + +pub(crate) fn store_stored_layer_in_tx( + tx: &Transaction<'_>, + glyph_id: &font::GlyphId, + layer: &font::GlyphLayer, + stored: &StoredLayerPayload, + mode: WriteMode, +) -> Result<(), StoreError> { + let stored_byte_length = encoded_len(stored.stored_byte_length)?; + let decoded_byte_length = encoded_len(stored.decoded_byte_length)?; + let layer_sql = match mode { + WriteMode::Insert => { + "INSERT INTO glyph_layers (id, glyph_id, source_id, width, height) VALUES (?1, ?2, ?3, ?4, ?5)" + } + WriteMode::Upsert => { + " + INSERT INTO glyph_layers (id, glyph_id, source_id, width, height) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(id) DO UPDATE SET + glyph_id = excluded.glyph_id, + source_id = excluded.source_id, + width = excluded.width, + height = excluded.height + " + } + }; + tx.prepare_cached(layer_sql)?.execute(params![ + layer.id().to_string(), + glyph_id.to_string(), + layer.source_id().to_string(), + layer.width(), + layer.height(), + ])?; + + let payload_sql = match mode { + WriteMode::Insert => { + "INSERT INTO glyph_layer_payloads ( + layer_id, inner_format, compression, payload, + stored_byte_length, decoded_byte_length, decoded_blake3 + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)" + } + WriteMode::Upsert => { + " + INSERT INTO glyph_layer_payloads ( + layer_id, inner_format, compression, payload, + stored_byte_length, decoded_byte_length, decoded_blake3 + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(layer_id) DO UPDATE SET + inner_format = excluded.inner_format, + compression = excluded.compression, + payload = excluded.payload, + stored_byte_length = excluded.stored_byte_length, + decoded_byte_length = excluded.decoded_byte_length, + decoded_blake3 = excluded.decoded_blake3 + " + } + }; + tx.prepare_cached(payload_sql)?.execute(params![ + layer.id().to_string(), + GLYPH_LAYER_FORMAT, + stored.compression.as_str(), + stored.bytes.as_slice(), + stored_byte_length, + decoded_byte_length, + stored.decoded_blake3.as_slice(), + ])?; + write_component_index(tx, layer, mode) +} + +fn encoded_len(bytes: u64) -> Result { + i64::try_from(bytes).map_err(|_| StoreError::LayerPayloadTooLarge { + bytes, + limit: MAX_LAYER_PAYLOAD_BYTES as u64, + }) +} + +pub(crate) fn rewrite_layer_in_tx( + tx: &Transaction<'_>, + layer: &font::GlyphLayer, +) -> Result<(), StoreError> { + let owner = layer_owner(tx, &layer.id())?.ok_or_else(|| StoreError::MissingEntity { + kind: "glyph layer", + id: layer.id().to_string(), + })?; + write_layer_in_tx(tx, &owner, layer) +} + +pub(crate) fn create_empty_layer_in_tx( + tx: &Transaction<'_>, + glyph_id: &font::GlyphId, + layer_id: font::LayerId, + source_id: font::SourceId, + width: f64, + height: Option, +) -> Result<(), StoreError> { + let mut layer = font::GlyphLayer::with_width(layer_id, source_id, width); + layer.set_height(height); + write_layer_in_tx(tx, glyph_id, &layer) +} + +fn write_component_index( + tx: &Transaction<'_>, + layer: &font::GlyphLayer, + mode: WriteMode, +) -> Result<(), StoreError> { + if mode == WriteMode::Upsert { + tx.prepare_cached("DELETE FROM glyph_components WHERE layer_id = ?1")? + .execute([layer.id().to_string()])?; + } + for (order_index, component) in layer.components_iter().enumerate() { + tx.prepare_cached( + " + INSERT INTO glyph_components ( + id, layer_id, base_glyph_id, order_index + ) VALUES (?1, ?2, ?3, ?4) + ", + )? + .execute(params![ + component.id().to_string(), + layer.id().to_string(), + component.base_glyph_id().to_string(), + order_index as i64, + ])?; + } + Ok(()) +} + +fn layer_owner( + conn: &Connection, + layer_id: &font::LayerId, +) -> Result, StoreError> { + conn.query_row( + "SELECT glyph_id FROM glyph_layers WHERE id = ?1", + [layer_id.to_string()], + |row| Ok(font::GlyphId::from_raw(row.get::<_, String>(0)?)), + ) + .optional() + .map_err(Into::into) +} diff --git a/crates/shift-store/src/lib.rs b/crates/shift-store/src/lib.rs index 967c075e..5fb6a706 100644 --- a/crates/shift-store/src/lib.rs +++ b/crates/shift-store/src/lib.rs @@ -1,27 +1,29 @@ mod change_set; -mod component; mod connection; mod error; mod font; mod font_state; mod glyph; +mod import_writer; mod layer; -mod outline; mod schema; mod source; mod store; mod types; mod workspace_state; +mod write_mode; -pub use component::{GlyphComponentRecord, NewGlyphComponent}; pub use error::StoreError; pub use font::FontInfo; pub use glyph::{GlyphRecord, NewGlyph}; -pub use layer::{GlyphLayerRecord, NewGlyphLayer}; -pub use outline::{AnchorRecord, ContourRecord, PointRecord}; +pub use import_writer::{FontImportWriter, encode_glyph_batch}; +pub use layer::{ + GLYPH_LAYER_FORMAT, GlyphLayerDirectoryEntry, MAX_LAYER_READ_BATCH_COUNT, + MAX_LAYER_READ_BATCH_DECODED_BYTES, +}; pub use source::{AxisRecord, NewAxis, NewSource, SourceAxisLocation, SourceKind, SourceRecord}; pub use store::ShiftStore; -pub use types::{AxisId, ComponentId, GlyphId, LayerId, RevisionId, SourceId}; +pub use types::{AxisId, GlyphId, GlyphWriteBatch, LayerBatchTiming, RevisionId, SourceId}; pub use workspace_state::{ Evidence, FileIdentity, SourceIdentitySnapshot, WorkspaceSourceKind, WorkspaceState, }; diff --git a/crates/shift-store/src/outline.rs b/crates/shift-store/src/outline.rs deleted file mode 100644 index f2a59267..00000000 --- a/crates/shift-store/src/outline.rs +++ /dev/null @@ -1,118 +0,0 @@ -use crate::{LayerId, ShiftStore, StoreError}; - -#[derive(Clone, Debug, PartialEq)] -pub struct ContourRecord { - pub id: String, - pub layer_id: LayerId, - pub closed: bool, - pub order_index: i64, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct PointRecord { - pub id: String, - pub contour_id: String, - pub order_index: i64, - pub x: f64, - pub y: f64, - pub point_type: String, - pub smooth: bool, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct AnchorRecord { - pub id: String, - pub layer_id: LayerId, - pub name: Option, - pub x: f64, - pub y: f64, - pub order_index: i64, -} - -impl ShiftStore { - pub fn list_contours_for_layer( - &self, - layer_id: &LayerId, - ) -> Result, StoreError> { - let mut stmt = self.conn.prepare( - " - SELECT id, layer_id, closed, order_index - FROM glyph_layer_contours - WHERE layer_id = ?1 - ORDER BY order_index - ", - )?; - - let rows = stmt.query_map([layer_id.as_str()], map_contour_row)?; - rows.collect::, _>>() - .map_err(StoreError::from) - } - - pub fn list_points_for_contour( - &self, - contour_id: &str, - ) -> Result, StoreError> { - let mut stmt = self.conn.prepare( - " - SELECT id, contour_id, order_index, x, y, point_type, smooth - FROM glyph_layer_points - WHERE contour_id = ?1 - ORDER BY order_index - ", - )?; - - let rows = stmt.query_map([contour_id], map_point_row)?; - rows.collect::, _>>() - .map_err(StoreError::from) - } - - pub fn list_anchors_for_layer( - &self, - layer_id: &LayerId, - ) -> Result, StoreError> { - let mut stmt = self.conn.prepare( - " - SELECT id, layer_id, name, x, y, order_index - FROM glyph_layer_anchors - WHERE layer_id = ?1 - ORDER BY order_index - ", - )?; - - let rows = stmt.query_map([layer_id.as_str()], map_anchor_row)?; - rows.collect::, _>>() - .map_err(StoreError::from) - } -} - -fn map_contour_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(ContourRecord { - id: row.get(0)?, - layer_id: LayerId::new(row.get::<_, String>(1)?), - closed: row.get(2)?, - order_index: row.get(3)?, - }) -} - -fn map_point_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(PointRecord { - id: row.get(0)?, - contour_id: row.get(1)?, - order_index: row.get(2)?, - x: row.get(3)?, - y: row.get(4)?, - point_type: row.get(5)?, - smooth: row.get(6)?, - }) -} - -fn map_anchor_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(AnchorRecord { - id: row.get(0)?, - layer_id: LayerId::new(row.get::<_, String>(1)?), - name: row.get(2)?, - x: row.get(3)?, - y: row.get(4)?, - order_index: row.get(5)?, - }) -} diff --git a/crates/shift-store/src/schema.rs b/crates/shift-store/src/schema.rs index d9a7d50d..df8949d1 100644 --- a/crates/shift-store/src/schema.rs +++ b/crates/shift-store/src/schema.rs @@ -1,5 +1,21 @@ +use rusqlite::Transaction; + use crate::StoreError; +const DEFER_IMPORT_INDEXES: &str = r#" +DROP INDEX IF EXISTS glyphs_name_idx; +DROP INDEX IF EXISTS glyph_layers_glyph_id_idx; +DROP INDEX IF EXISTS glyph_layers_source_id_idx; +DROP INDEX IF EXISTS glyph_components_base_glyph_id_idx; +"#; + +const RESTORE_IMPORT_INDEXES: &str = r#" +CREATE INDEX glyphs_name_idx ON glyphs(name); +CREATE INDEX glyph_layers_glyph_id_idx ON glyph_layers(glyph_id); +CREATE INDEX glyph_layers_source_id_idx ON glyph_layers(source_id); +CREATE INDEX glyph_components_base_glyph_id_idx ON glyph_components(base_glyph_id); +"#; + pub(crate) const SCHEMA_V1: &str = r#" CREATE TABLE IF NOT EXISTS font_info ( id INTEGER PRIMARY KEY CHECK (id = 1), @@ -92,14 +108,10 @@ CREATE TABLE IF NOT EXISTS glyph_unicodes ( FOREIGN KEY (glyph_id) REFERENCES glyphs(id) ON DELETE CASCADE ); -CREATE INDEX IF NOT EXISTS glyph_unicodes_glyph_id_idx -ON glyph_unicodes(glyph_id); - CREATE TABLE IF NOT EXISTS glyph_layers ( id TEXT PRIMARY KEY, glyph_id TEXT NOT NULL, source_id TEXT NOT NULL, - name TEXT, width REAL NOT NULL DEFAULT 0, height REAL, FOREIGN KEY (glyph_id) REFERENCES glyphs(id) ON DELETE CASCADE, @@ -112,51 +124,23 @@ ON glyph_layers(glyph_id); CREATE INDEX IF NOT EXISTS glyph_layers_source_id_idx ON glyph_layers(source_id); -CREATE TABLE IF NOT EXISTS glyph_layer_contours ( - id TEXT PRIMARY KEY, - layer_id TEXT NOT NULL, - closed INTEGER NOT NULL DEFAULT 0 CHECK (closed IN (0, 1)), - order_index INTEGER NOT NULL, +CREATE TABLE IF NOT EXISTS glyph_layer_payloads ( + layer_id TEXT PRIMARY KEY, + inner_format TEXT NOT NULL, + compression TEXT NOT NULL CHECK (compression IN ('none', 'zstd.v1')), + payload BLOB NOT NULL, + stored_byte_length INTEGER NOT NULL + CHECK (stored_byte_length >= 0 AND stored_byte_length = length(payload)), + decoded_byte_length INTEGER NOT NULL + CHECK (decoded_byte_length >= 0 AND stored_byte_length <= decoded_byte_length), + decoded_blake3 BLOB NOT NULL CHECK (length(decoded_blake3) = 32), FOREIGN KEY (layer_id) REFERENCES glyph_layers(id) ON DELETE CASCADE ); -CREATE UNIQUE INDEX IF NOT EXISTS glyph_layer_contours_layer_order_unique -ON glyph_layer_contours(layer_id, order_index); - -CREATE INDEX IF NOT EXISTS glyph_layer_contours_layer_id_idx -ON glyph_layer_contours(layer_id); - -CREATE TABLE IF NOT EXISTS glyph_layer_points ( - id TEXT PRIMARY KEY, - contour_id TEXT NOT NULL, - order_index INTEGER NOT NULL, - x REAL NOT NULL, - y REAL NOT NULL, - point_type TEXT NOT NULL, - smooth INTEGER NOT NULL DEFAULT 0 CHECK (smooth IN (0, 1)), - FOREIGN KEY (contour_id) REFERENCES glyph_layer_contours(id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX IF NOT EXISTS glyph_layer_points_contour_order_unique -ON glyph_layer_points(contour_id, order_index); - -CREATE INDEX IF NOT EXISTS glyph_layer_points_contour_id_idx -ON glyph_layer_points(contour_id); - CREATE TABLE IF NOT EXISTS glyph_components ( id TEXT PRIMARY KEY, layer_id TEXT NOT NULL, base_glyph_id TEXT NOT NULL, - base_glyph_name TEXT NOT NULL, - translate_x REAL NOT NULL DEFAULT 0, - translate_y REAL NOT NULL DEFAULT 0, - rotation REAL NOT NULL DEFAULT 0, - scale_x REAL NOT NULL DEFAULT 1, - scale_y REAL NOT NULL DEFAULT 1, - skew_x REAL NOT NULL DEFAULT 0, - skew_y REAL NOT NULL DEFAULT 0, - t_center_x REAL NOT NULL DEFAULT 0, - t_center_y REAL NOT NULL DEFAULT 0, order_index INTEGER NOT NULL, FOREIGN KEY (layer_id) REFERENCES glyph_layers(id) ON DELETE CASCADE ); @@ -164,28 +148,9 @@ CREATE TABLE IF NOT EXISTS glyph_components ( CREATE UNIQUE INDEX IF NOT EXISTS glyph_components_layer_order_unique ON glyph_components(layer_id, order_index); -CREATE INDEX IF NOT EXISTS glyph_components_layer_id_idx -ON glyph_components(layer_id); - CREATE INDEX IF NOT EXISTS glyph_components_base_glyph_id_idx ON glyph_components(base_glyph_id); -CREATE TABLE IF NOT EXISTS glyph_layer_anchors ( - id TEXT PRIMARY KEY, - layer_id TEXT NOT NULL, - name TEXT, - x REAL NOT NULL, - y REAL NOT NULL, - order_index INTEGER NOT NULL, - FOREIGN KEY (layer_id) REFERENCES glyph_layers(id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX IF NOT EXISTS glyph_layer_anchors_layer_order_unique -ON glyph_layer_anchors(layer_id, order_index); - -CREATE INDEX IF NOT EXISTS glyph_layer_anchors_layer_id_idx -ON glyph_layer_anchors(layer_id); - CREATE TABLE IF NOT EXISTS font_guidelines ( id TEXT PRIMARY KEY, x REAL, @@ -196,21 +161,6 @@ CREATE TABLE IF NOT EXISTS font_guidelines ( order_index INTEGER NOT NULL ); -CREATE TABLE IF NOT EXISTS glyph_layer_guidelines ( - id TEXT PRIMARY KEY, - layer_id TEXT NOT NULL, - x REAL, - y REAL, - angle REAL, - name TEXT, - color TEXT, - order_index INTEGER NOT NULL, - FOREIGN KEY (layer_id) REFERENCES glyph_layers(id) ON DELETE CASCADE -); - -CREATE INDEX IF NOT EXISTS glyph_layer_guidelines_layer_id_idx -ON glyph_layer_guidelines(layer_id); - CREATE TABLE IF NOT EXISTS source_locations ( source_id TEXT NOT NULL, axis_id TEXT NOT NULL, @@ -285,14 +235,6 @@ CREATE TABLE IF NOT EXISTS glyph_lib ( FOREIGN KEY (glyph_id) REFERENCES glyphs(id) ON DELETE CASCADE ); -CREATE TABLE IF NOT EXISTS glyph_layer_lib ( - layer_id TEXT NOT NULL, - key TEXT NOT NULL, - value_json TEXT NOT NULL, - PRIMARY KEY (layer_id, key), - FOREIGN KEY (layer_id) REFERENCES glyph_layers(id) ON DELETE CASCADE -); - CREATE TABLE IF NOT EXISTS font_binaries ( kind TEXT NOT NULL CHECK (kind IN ('data', 'image')), path TEXT NOT NULL, @@ -322,6 +264,16 @@ CREATE TABLE IF NOT EXISTS workspace_state ( pub(crate) const SCHEMA_VERSION: i64 = 1; +pub(crate) fn defer_import_indexes(tx: &Transaction<'_>) -> Result<(), StoreError> { + tx.execute_batch(DEFER_IMPORT_INDEXES)?; + Ok(()) +} + +pub(crate) fn restore_import_indexes(tx: &Transaction<'_>) -> Result<(), StoreError> { + tx.execute_batch(RESTORE_IMPORT_INDEXES)?; + Ok(()) +} + /// Creates the baseline schema and stamps `user_version`. /// /// Pre-release policy: the app has not shipped, so schema changes edit the diff --git a/crates/shift-store/src/store.rs b/crates/shift-store/src/store.rs index 398cc350..42fe434f 100644 --- a/crates/shift-store/src/store.rs +++ b/crates/shift-store/src/store.rs @@ -1,3 +1,4 @@ pub struct ShiftStore { pub(crate) conn: rusqlite::Connection, + pub(crate) path: Option, } diff --git a/crates/shift-store/src/types.rs b/crates/shift-store/src/types.rs index b9791716..76945170 100644 --- a/crates/shift-store/src/types.rs +++ b/crates/shift-store/src/types.rs @@ -1,16 +1,39 @@ -#[derive(Clone, Debug, Eq, PartialEq, Hash)] -pub struct ComponentId(String); +use std::time::Duration; -impl ComponentId { - pub fn new(id: impl Into) -> Self { - Self(id.into()) +use shift_font::{Glyph, LayerId}; + +use crate::layer::StoredLayerPayload; + +pub(crate) type EncodedGlyphLayers = Vec)>>; + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct LayerBatchTiming { + pub pack_elapsed: Duration, + pub compression_elapsed: Duration, + pub sqlite_elapsed: Duration, +} + +pub struct GlyphWriteBatch { + pub(crate) glyphs: Vec, + pub(crate) pack_elapsed: Duration, + pub(crate) compression_elapsed: Duration, +} + +impl GlyphWriteBatch { + pub fn pack_elapsed(&self) -> Duration { + self.pack_elapsed } - pub fn as_str(&self) -> &str { - &self.0 + pub fn compression_elapsed(&self) -> Duration { + self.compression_elapsed } } +pub(crate) struct PackedGlyph { + pub(crate) glyph: Glyph, + pub(crate) layers: Vec<(LayerId, StoredLayerPayload)>, +} + #[derive(Clone, Debug, Eq, PartialEq, Hash)] pub struct AxisId(String); @@ -50,19 +73,6 @@ impl GlyphId { } } -#[derive(Clone, Debug, Eq, PartialEq, Hash)] -pub struct LayerId(String); - -impl LayerId { - pub fn new(id: impl Into) -> Self { - Self(id.into()) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] pub struct RevisionId(i64); diff --git a/crates/shift-store/src/write_mode.rs b/crates/shift-store/src/write_mode.rs new file mode 100644 index 00000000..729e042f --- /dev/null +++ b/crates/shift-store/src/write_mode.rs @@ -0,0 +1,5 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum WriteMode { + Insert, + Upsert, +} diff --git a/crates/shift-store/tests/store_test.rs b/crates/shift-store/tests/store_test.rs index 2c67983f..52d0e0d8 100644 --- a/crates/shift-store/tests/store_test.rs +++ b/crates/shift-store/tests/store_test.rs @@ -1,8 +1,7 @@ use shift_font::{FontMetadata, test_support::sample_font}; use shift_store::{ - AxisId, ComponentId, Evidence, FileIdentity, FontInfo, GlyphId, LayerId, NewAxis, NewGlyph, - NewGlyphComponent, NewGlyphLayer, NewSource, ShiftStore, SourceId, SourceIdentitySnapshot, - SourceKind, WorkspaceState, + AxisId, Evidence, FileIdentity, FontInfo, GlyphId, NewAxis, NewGlyph, NewSource, ShiftStore, + SourceId, SourceIdentitySnapshot, SourceKind, WorkspaceState, }; #[test] @@ -283,27 +282,34 @@ fn creates_and_reads_glyph_layer() { let source_id = create_regular_source(&mut store); let layer_id = create_default_glyph_layer(&mut store, &glyph_id, &source_id); - let layer = store - .get_glyph_layer(&layer_id) - .expect("glyph layer query should succeed") + let directory = store + .list_glyph_layer_directory() + .expect("glyph layer directory query should succeed"); + let layer = directory + .iter() + .find(|entry| entry.layer_id == layer_id) .expect("glyph layer should exist"); - assert_eq!(layer.id, layer_id); - assert_eq!(layer.glyph_id, glyph_id); - assert_eq!(layer.source_id, source_id); - assert_eq!(layer.name.as_deref(), Some("Regular")); + assert_eq!(layer.glyph_id.as_str(), glyph_id.as_str()); + assert_eq!(layer.source_id.as_str(), source_id.as_str()); + assert_eq!(layer.name.as_ref().map(|name| name.as_str()), Some("A")); } #[test] fn glyph_layer_requires_existing_glyph_and_source() { let mut store = ShiftStore::open_memory_for_test().expect("memory store should open"); - let result = store.create_glyph_layer(NewGlyphLayer { - id: LayerId::new("layer-A-regular"), - glyph_id: GlyphId::new("glyph-missing"), - source_id: SourceId::new("source-missing"), - name: Some("Regular".to_string()), - }); + let layer = shift_font::GlyphLayer::with_width( + shift_font::LayerId::from_raw("layer-A-regular"), + shift_font::SourceId::from_raw("source-missing"), + 0.0, + ); + let result = store.apply_change_set(&shift_font::FontChangeSet::new(vec![ + shift_font::FontChange::glyph_layer_created( + shift_font::GlyphId::from_raw("glyph-missing"), + &layer, + ), + ])); assert!(result.is_err()); } @@ -316,13 +322,15 @@ fn lists_glyph_layers_for_glyph() { let layer_id = create_default_glyph_layer(&mut store, &glyph_id, &source_id); let layers = store - .list_glyph_layers_for_glyph(&glyph_id) - .expect("glyph layers query should succeed"); + .list_glyph_layer_directory() + .expect("glyph layers query should succeed") + .into_iter() + .filter(|entry| entry.glyph_id.as_str() == glyph_id.as_str()) + .collect::>(); assert_eq!(layers.len(), 1); - assert_eq!(layers[0].id, layer_id); - assert_eq!(layers[0].glyph_id, glyph_id); - assert_eq!(layers[0].source_id, source_id); + assert_eq!(layers[0].layer_id, layer_id); + assert_eq!(layers[0].source_id.as_str(), source_id.as_str()); } #[test] @@ -334,29 +342,35 @@ fn creates_and_reads_glyph_component() { let layer_id = create_default_glyph_layer(&mut store, &glyph_id, &source_id); let component_id = create_default_component(&mut store, &layer_id, &base_glyph_id); - let component = store - .get_glyph_component(&component_id) - .expect("glyph component query should succeed") + let layer = store + .load_glyph_layer(&layer_id) + .expect("glyph layer query should succeed") + .expect("glyph layer should exist"); + let component = layer + .components_iter() + .next() .expect("glyph component should exist"); - assert_eq!(component.id, component_id); - assert_eq!(component.layer_id, layer_id); - assert_eq!(component.base_glyph_id, base_glyph_id); - assert_eq!(component.order_index, 0); + assert_eq!(component.id(), component_id); + assert_eq!(component.base_glyph_id().as_str(), base_glyph_id.as_str()); } #[test] -fn glyph_component_requires_existing_layer_and_base_glyph() { +fn glyph_component_replacement_requires_existing_layer() { let mut store = ShiftStore::open_memory_for_test().expect("memory store should open"); + let mut layer = shift_font::GlyphLayer::with_width( + shift_font::LayerId::from_raw("layer-missing"), + shift_font::SourceId::from_raw("source-missing"), + 0.0, + ); + layer.add_component(shift_font::Component::with_id( + shift_font::ComponentId::from_raw("component-A-missing"), + shift_font::GlyphId::from_raw("glyph-missing"), + "Missing", + shift_font::DecomposedTransform::identity(), + )); - let result = store.create_glyph_component(NewGlyphComponent { - id: ComponentId::new("component-A-B"), - layer_id: LayerId::new("layer-missing"), - base_glyph_id: GlyphId::new("glyph-missing"), - base_glyph_name: "Missing".to_string(), - transform: shift_font::DecomposedTransform::identity(), - order_index: 0, - }); + let result = store.replace_glyph_layer(&layer); assert!(result.is_err()); } @@ -370,15 +384,16 @@ fn lists_glyph_components_for_layer() { let layer_id = create_default_glyph_layer(&mut store, &glyph_id, &source_id); let component_id = create_default_component(&mut store, &layer_id, &base_glyph_id); - let components = store - .list_glyph_components_for_layer(&layer_id) - .expect("glyph components query should succeed"); + let layer = store + .load_glyph_layer(&layer_id) + .expect("glyph layer query should succeed") + .expect("glyph layer should exist"); + let components = layer.components(); + let (_, component) = components.iter().next().expect("component should exist"); assert_eq!(components.len(), 1); - assert_eq!(components[0].id, component_id); - assert_eq!(components[0].layer_id, layer_id); - assert_eq!(components[0].base_glyph_id, base_glyph_id); - assert_eq!(components[0].order_index, 0); + assert_eq!(component.id(), component_id); + assert_eq!(component.base_glyph_id().as_str(), base_glyph_id.as_str()); } #[test] @@ -386,10 +401,15 @@ fn applies_glyph_identity_change_set() { let mut store = ShiftStore::open_memory_for_test().expect("memory store should open"); let glyph = shift_font::Glyph::with_unicode("A", 65); let glyph_id = glyph.id(); + let source_id = shift_font::SourceId::new(); + let layer = + shift_font::GlyphLayer::with_width(shift_font::LayerId::new(), source_id.clone(), 500.0); + create_regular_source_with_id(&mut store, source_id); store .apply_change_set(&shift_font::FontChangeSet::new(vec![ shift_font::FontChange::GlyphCreated(shift_font::GlyphCreated::from(&glyph)), + shift_font::FontChange::glyph_layer_created(glyph.id(), &layer), shift_font::FontChange::GlyphIdentityChanged(shift_font::GlyphIdentityChanged { glyph_id: glyph_id.clone(), from_name: shift_font::GlyphName::from("A"), @@ -408,8 +428,46 @@ fn applies_glyph_identity_change_set() { .list_glyph_unicodes(&stored.id) .expect("unicode query should succeed"); + let directory = store.list_glyph_layer_directory().unwrap(); + assert_eq!(stored.name.as_deref(), Some("A.alt")); assert_eq!(unicodes, vec![0x00c1]); + assert_eq!(directory.len(), 1); + assert_eq!(directory[0].name.as_deref(), Some("A.alt")); +} + +#[test] +fn glyph_rename_preserves_authored_order() { + let mut font = shift_font::Font::new(); + let first = shift_font::Glyph::new("first"); + let second = shift_font::Glyph::new("second"); + let second_id = second.id(); + font.insert_glyph(first).unwrap(); + font.insert_glyph(second).unwrap(); + let mut store = ShiftStore::open_memory_for_test().unwrap(); + store.replace_font_state(&font).unwrap(); + + store + .apply_change_set(&shift_font::FontChangeSet::new(vec![ + shift_font::FontChange::GlyphIdentityChanged(shift_font::GlyphIdentityChanged { + glyph_id: second_id, + from_name: shift_font::GlyphName::from("second"), + to_name: shift_font::GlyphName::from("second.alt"), + from_unicodes: vec![], + to_unicodes: vec![], + }), + ])) + .unwrap(); + + assert_eq!( + store + .load_font_directory() + .unwrap() + .glyphs() + .map(|glyph| glyph.name().to_string()) + .collect::>(), + vec!["first", "second.alt"] + ); } #[test] @@ -425,11 +483,7 @@ fn applies_glyph_delete_change_set_and_cascades_layers() { store .apply_change_set(&shift_font::FontChangeSet::new(vec![ shift_font::FontChange::glyph_created(&glyph), - shift_font::FontChange::glyph_layer_created( - glyph.id(), - Some(glyph.glyph_name().clone()), - &layer, - ), + shift_font::FontChange::glyph_layer_created(glyph.id(), &layer), shift_font::FontChange::glyph_deleted(glyph.id()), ])) .expect("change set should apply"); @@ -438,8 +492,11 @@ fn applies_glyph_delete_change_set_and_cascades_layers() { .get_glyph(&GlyphId::new(glyph_id.to_string())) .expect("glyph query should succeed"); let layers = store - .list_glyph_layers_for_glyph(&GlyphId::new(glyph_id.to_string())) - .expect("layer query should succeed"); + .list_glyph_layer_directory() + .expect("layer query should succeed") + .into_iter() + .filter(|entry| entry.glyph_id == glyph_id) + .collect::>(); let unicodes = store .list_glyph_unicodes(&GlyphId::new(glyph_id.to_string())) .expect("unicode query should succeed"); @@ -458,11 +515,7 @@ fn applies_layer_metrics_and_contour_point_changes() { store .apply_change_set(&shift_font::FontChangeSet::new(vec![ shift_font::FontChange::glyph_created(&glyph), - shift_font::FontChange::glyph_layer_created( - glyph.id(), - Some(glyph.glyph_name().clone()), - &layer, - ), + shift_font::FontChange::glyph_layer_created(glyph.id(), &layer), shift_font::FontChange::LayerMetricsChanged(shift_font::LayerMetricsChanged { layer_id: layer.id(), width: 720.0, @@ -484,23 +537,20 @@ fn applies_layer_metrics_and_contour_point_changes() { .expect("change set should apply"); let layer = store - .get_glyph_layer(&LayerId::new(layer.id().to_string())) + .load_glyph_layer(&layer.id()) .expect("layer query should succeed") .expect("layer should exist"); - let contours = store - .list_contours_for_layer(&layer.id) - .expect("contour query should succeed"); - let points = store - .list_points_for_contour(&contours[0].id) - .expect("point query should succeed"); - - assert_eq!(layer.width, 720.0); + let contours = layer.contours(); + let (_, contour) = contours.iter().next().expect("contour should exist"); + let points = contour.points(); + + assert_eq!(layer.width(), 720.0); assert_eq!(contours.len(), 1); - assert!(!contours[0].closed); + assert!(!contour.is_closed()); assert_eq!(points.len(), 1); - assert_eq!(points[0].id, point_id.to_string()); - assert_eq!((points[0].x, points[0].y), (40.0, 50.0)); - assert_eq!(points[0].point_type, "onCurve"); + assert_eq!(points[0].id(), point_id); + assert_eq!((points[0].x(), points[0].y()), (40.0, 50.0)); + assert_eq!(points[0].point_type(), shift_font::PointType::OnCurve); } #[test] @@ -514,11 +564,7 @@ fn applies_layer_geometry_replacement() { store .apply_change_set(&shift_font::FontChangeSet::new(vec![ shift_font::FontChange::glyph_created(&glyph), - shift_font::FontChange::glyph_layer_created( - glyph.id(), - Some(glyph.glyph_name().clone()), - &layer, - ), + shift_font::FontChange::glyph_layer_created(glyph.id(), &layer), shift_font::FontChange::ContourAdded(shift_font::ContourAdded { layer_id: layer.id(), contour: first_contour, @@ -530,16 +576,17 @@ fn applies_layer_geometry_replacement() { ])) .expect("change set should apply"); - let contours = store - .list_contours_for_layer(&LayerId::new(layer.id().to_string())) - .expect("contour query should succeed"); - let points = store - .list_points_for_contour(&contours[0].id) - .expect("point query should succeed"); + let layer = store + .load_glyph_layer(&layer.id()) + .expect("layer query should succeed") + .expect("layer should exist"); + let contours = layer.contours(); + let (_, contour) = contours.iter().next().expect("contour should exist"); + let points = contour.points(); assert_eq!(contours.len(), 1); assert_eq!(points.len(), 1); - assert_eq!((points[0].x, points[0].y), (10.0, 20.0)); + assert_eq!((points[0].x(), points[0].y()), (10.0, 20.0)); } #[test] @@ -552,15 +599,16 @@ fn layer_geometry_replacement_round_trips_anchors() { .apply_change_set(&anchored_layer_change_set(&glyph, &layer)) .expect("change set should apply"); - let anchors = store - .list_anchors_for_layer(&LayerId::new(layer.id().to_string())) - .expect("anchor query should succeed"); + let layer = store + .load_glyph_layer(&layer.id()) + .expect("anchor query should succeed") + .expect("layer should exist"); + let anchors = layer.anchors(); assert_eq!(anchors.len(), 1); - assert_eq!(anchors[0].id, anchor_id.to_string()); - assert_eq!(anchors[0].name.as_deref(), Some("top")); - assert_eq!((anchors[0].x, anchors[0].y), (250.0, 700.0)); - assert_eq!(anchors[0].order_index, 0); + assert_eq!(anchors[0].id(), anchor_id); + assert_eq!(anchors[0].name(), Some("top")); + assert_eq!((anchors[0].x(), anchors[0].y()), (250.0, 700.0)); } #[test] @@ -585,10 +633,12 @@ fn applies_anchor_position_changes() { ])) .expect("anchor positions should apply"); - let anchors = store - .list_anchors_for_layer(&LayerId::new(layer.id().to_string())) - .expect("anchor query should succeed"); - assert_eq!((anchors[0].x, anchors[0].y), (300.0, 650.0)); + let layer = store + .load_glyph_layer(&layer.id()) + .expect("anchor query should succeed") + .expect("layer should exist"); + let anchors = layer.anchors(); + assert_eq!((anchors[0].x(), anchors[0].y()), (300.0, 650.0)); } #[test] @@ -634,14 +684,16 @@ fn reopen_preserves_layer_anchors() { } let store = ShiftStore::open(&path).expect("reopen"); - let anchors = store - .list_anchors_for_layer(&LayerId::new(layer.id().to_string())) - .expect("anchor query should succeed"); + let layer = store + .load_glyph_layer(&layer.id()) + .expect("anchor query should succeed") + .expect("layer should exist"); + let anchors = layer.anchors(); assert_eq!(anchors.len(), 1); - assert_eq!(anchors[0].id, anchor_id.to_string()); - assert_eq!(anchors[0].name.as_deref(), Some("top")); - assert_eq!((anchors[0].x, anchors[0].y), (250.0, 700.0)); + assert_eq!(anchors[0].id(), anchor_id); + assert_eq!(anchors[0].name(), Some("top")); + assert_eq!((anchors[0].x(), anchors[0].y()), (250.0, 700.0)); std::fs::remove_dir_all(path.parent().unwrap()).ok(); } @@ -655,11 +707,7 @@ fn rejects_incremental_change_for_missing_point_row() { let result = store.apply_change_set(&shift_font::FontChangeSet::new(vec![ shift_font::FontChange::glyph_created(&glyph), - shift_font::FontChange::glyph_layer_created( - glyph.id(), - Some(glyph.glyph_name().clone()), - &layer, - ), + shift_font::FontChange::glyph_layer_created(glyph.id(), &layer), shift_font::FontChange::PointPositionsChanged(shift_font::PointPositionsChanged { layer_id: layer.id(), points: vec![shift_font::PointPosition { @@ -700,7 +748,7 @@ fn rejects_layer_edit_for_missing_layer_row() { } fn create_glyph_a(store: &mut ShiftStore) -> GlyphId { - let glyph_id = GlyphId::new("glyph-A"); + let glyph_id = GlyphId::new(shift_font::GlyphId::from_raw("A").to_string()); store .create_glyph(NewGlyph { @@ -769,7 +817,7 @@ fn empty_font_info() -> FontInfo { } fn create_glyph_b(store: &mut ShiftStore) -> GlyphId { - let glyph_id = GlyphId::new("glyph-B"); + let glyph_id = GlyphId::new(shift_font::GlyphId::from_raw("B").to_string()); store .create_glyph(NewGlyph { @@ -785,37 +833,44 @@ fn create_default_glyph_layer( store: &mut ShiftStore, glyph_id: &GlyphId, source_id: &SourceId, -) -> LayerId { - let layer_id = LayerId::new("layer-A-regular"); +) -> shift_font::LayerId { + let layer = shift_font::GlyphLayer::with_width( + shift_font::LayerId::from_raw("A-regular"), + shift_font::SourceId::from_raw(source_id.as_str()), + 0.0, + ); store - .create_glyph_layer(NewGlyphLayer { - id: layer_id.clone(), - glyph_id: glyph_id.clone(), - source_id: source_id.clone(), - name: Some("Regular".to_string()), - }) + .apply_change_set(&shift_font::FontChangeSet::new(vec![ + shift_font::FontChange::glyph_layer_created( + shift_font::GlyphId::from_raw(glyph_id.as_str()), + &layer, + ), + ])) .expect("glyph layer should be created"); - layer_id + layer.id() } fn create_default_component( store: &mut ShiftStore, - layer_id: &LayerId, + layer_id: &shift_font::LayerId, base_glyph_id: &GlyphId, -) -> ComponentId { - let component_id = ComponentId::new("component-A-B"); +) -> shift_font::ComponentId { + let component_id = shift_font::ComponentId::from_raw("A-B"); + let mut layer = store + .load_glyph_layer(layer_id) + .expect("glyph layer query should succeed") + .expect("glyph layer should exist"); + layer.add_component(shift_font::Component::with_id( + component_id.clone(), + shift_font::GlyphId::from_raw(base_glyph_id.as_str()), + "B", + shift_font::DecomposedTransform::identity(), + )); store - .create_glyph_component(NewGlyphComponent { - id: component_id.clone(), - layer_id: layer_id.clone(), - base_glyph_id: base_glyph_id.clone(), - base_glyph_name: "B".to_string(), - transform: shift_font::DecomposedTransform::identity(), - order_index: 0, - }) + .replace_glyph_layer(&layer) .expect("glyph component should be created"); component_id @@ -840,7 +895,7 @@ fn create_weight_axis(store: &mut ShiftStore) -> AxisId { } fn create_regular_source(store: &mut ShiftStore) -> SourceId { - let source_id = SourceId::new("source-regular"); + let source_id = SourceId::new(shift_font::SourceId::from_raw("regular").to_string()); store .create_source(NewSource { @@ -913,11 +968,7 @@ fn anchored_layer_change_set( ) -> shift_font::FontChangeSet { shift_font::FontChangeSet::new(vec![ shift_font::FontChange::glyph_created(glyph), - shift_font::FontChange::glyph_layer_created( - glyph.id(), - Some(glyph.glyph_name().clone()), - layer, - ), + shift_font::FontChange::glyph_layer_created(glyph.id(), layer), shift_font::FontChange::layer_geometry_replaced(layer), ]) } @@ -953,6 +1004,53 @@ fn file_stores_run_wal_with_verified_pragmas() { std::fs::remove_dir_all(path.parent().unwrap()).ok(); } +#[test] +fn completed_import_store_restores_durable_wal_mode() { + let path = temp_store_path("import-pragmas"); + let font = shift_font::test_support::sample_font(); + let mut store = ShiftStore::open_for_import(&path).expect("open import store"); + let mut writer = store.begin_import(&font).expect("begin import"); + for glyph in font.glyphs() { + writer.write_glyph(glyph).expect("write glyph"); + } + writer.finish().expect("commit import"); + store.finish_import().expect("finish import"); + drop(store); + + let conn = rusqlite::Connection::open(&path).expect("reopen raw"); + let journal: String = conn + .query_row("PRAGMA journal_mode", [], |row| row.get(0)) + .expect("journal_mode"); + assert_eq!(journal, "wal"); + let integrity: String = conn + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .expect("integrity_check"); + assert_eq!(integrity, "ok"); + + std::fs::remove_dir_all(path.parent().unwrap()).ok(); +} + +#[test] +fn import_mode_refuses_a_published_workspace_destination() { + let path = temp_store_path("occupied-import"); + let mut store = ShiftStore::open(&path).expect("open"); + store + .set_workspace_state(WorkspaceState::untitled(None)) + .expect("publish workspace"); + drop(store); + + let error = match ShiftStore::open_for_import(&path) { + Ok(_) => panic!("published store must be retained"), + Err(error) => error, + }; + assert!(matches!( + error, + shift_store::StoreError::ImportDestinationNotEmpty(existing) if existing == path + )); + + std::fs::remove_dir_all(path.parent().unwrap()).ok(); +} + #[test] fn reopen_preserves_written_contents_and_integrity() { let path = temp_store_path("reopen"); diff --git a/crates/shift-workspace/Cargo.toml b/crates/shift-workspace/Cargo.toml index 06425016..20d7c506 100644 --- a/crates/shift-workspace/Cargo.toml +++ b/crates/shift-workspace/Cargo.toml @@ -8,10 +8,11 @@ shift-backends = { workspace = true } shift-font = { workspace = true } shift-source = { workspace = true } shift-store = { workspace = true } +tempfile = "3" thiserror = "2" [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Security", "Win32_Storage_FileSystem"] } [dev-dependencies] -tempfile = "3" +rusqlite = "0.37.0" diff --git a/crates/shift-workspace/docs/DOCS.md b/crates/shift-workspace/docs/DOCS.md index 50dd3120..98118897 100644 --- a/crates/shift-workspace/docs/DOCS.md +++ b/crates/shift-workspace/docs/DOCS.md @@ -4,13 +4,18 @@ Backend runtime object for an open Shift font workspace. ## Architecture Invariants -- **Architecture Invariant:** `FontWorkspace` composes the live `shift-font::Font`, the user-selected `shift-source` package, and the working `shift-store` database. +- **Architecture Invariant:** `FontWorkspace` composes a directory-complete, payload-lazy `shift-font::Font`, the user-selected `shift-source` package, and the working `shift-store` database. +- **Architecture Invariant:** Resuming SQLite loads metadata and glyph/layer directory facts only. `acquire_glyphs` performs explicit bounded payload I/O; synchronous `font()` reads never initiate I/O. +- **Architecture Invariant:** TTF/OTF, UFO, and Designspace imports consume bounded `FontImport` batches and write through one `FontImportWriter`; they never construct a complete geometry-resident `Font`. Format readers, MessagePack encoding, BLAKE3 hashing, and independent per-layer compression use Rayon, while SQLite has one transaction owner. +- **Architecture Invariant:** `LayerResidency` is the sole owner of loaded-layer membership and placeholder replacement. A loaded layer is only a cache of already-committed authored state; apply, undo, and redo reacquire their complete layer read sets before mutation, and `evict_glyphs` replaces only committed layers with directory placeholders. - **Architecture Invariant:** The `.shift` source package path and SQLite working store path are separate inputs. - **Architecture Invariant:** Package recovery policy is not ranked in Rust. `FontWorkspace` exposes package and working-store inspection primitives; the utility process owns binding and lifecycle decisions. - **Architecture Invariant:** The workspace is the domain object future bridge or utility-process transports should wrap. +- **Architecture Invariant:** Ledger layer pairs retain the original touched-layer structural classification. Values-only undo/redo restores the target snapshot's canonical numeric values without rebuilding identity indexes or emitting structure; structural replay installs and emits the complete target structure in both directions. - **Architecture Invariant:** Ledger replay restores complete named-instance collections after axis topology so undo/redo never observes an instance against the wrong external-axis shape. - **Architecture Invariant:** Metadata ledger entries store complete pre/post snapshots and replay them independently of font metrics. - **Architecture Invariant:** Metric-definition ledger state replays before complete source snapshots so source metric IDs are always valid during undo and redo. +- **Architecture Invariant:** Undo and redo retain at most 100 entries per stack. Extending either stack drops that stack's oldest entry; a fresh apply clears redo. ## Codemap @@ -19,7 +24,13 @@ crates/shift-workspace/src/ lib.rs -- public API barrel new_workspace.rs -- creation options for a fresh workspace source_identity.rs -- package identity snapshots and source-save validation - workspace.rs -- `FontWorkspace` and workspace errors + import_pipeline.rs -- bounded parser/parallel-packer/single-writer pipeline and progress observer + import_staging.rs -- sibling staged-store creation, atomic installation, and parent sync + layer_residency.rs -- complete read sets, bounded acquisition, and safe eviction + ledger.rs -- bounded snapshot-pair undo/redo entries + workspace.rs -- `FontWorkspace` orchestration and workspace errors +crates/shift-workspace/examples/ + profile_streaming_import.rs -- shared release-mode foreign/native import profiler ``` ## Key Types @@ -35,7 +46,7 @@ crates/shift-workspace/src/ `FontWorkspace::create(source_path, store_path, options)` creates a placeholder `.shift` package, opens the working SQLite store, writes initial font metadata, and starts with an empty `shift-font::Font`. -`FontWorkspace::open(path, store_path)` detects `.shift` paths as source packages. Other supported font paths are imported through `shift-backends` into an unsaved workspace with no save target. +`FontWorkspace::open(path, store_path)` detects `.shift` paths as source packages. TTF/OTF, UFO, and Designspace paths use a metadata/directory-first backend cursor, parse batches of at most 512 glyphs and 1,024 authored layers, parallel-pack/hash/compress those layers, and insert them into a disposable sibling staging database. The legal import transitions are **Staging** (foreign source remains authoritative), **Durable** (stream committed, indexes restored, workspace state written, database synced), then **Published** (closed staging file atomically installed and parent directory synced). Failure before Published removes staging and leaves the previous destination untouched. The returned workspace contains directory placeholders and zero loaded layer payloads. This synchronous API still returns only after finalization; publishing the directory and binary packed-outline grid while import continues requires the separate app import-session boundary. Other supported foreign formats retain the eager compatibility path until they gain a bounded reader. `FontWorkspace::save()` succeeds for saved `.shift` workspaces and returns `NeedsSaveAs` for imported workspaces. `save_as(path)` creates a `.shift` package and makes it the save target. @@ -43,6 +54,33 @@ crates/shift-workspace/src/ `FontWorkspace::inspect_package_draft(store_path)` reads the working-store package ownership record without resuming it. It returns the package id, source path, base fingerprint, document id, and dirty flag so the utility process can choose an explicit open transition. +`FontWorkspace::resume(store_path)` builds the eager directory skeleton without reading any layer BLOB. `acquire_glyphs(ids, AcquireScope::Glyphs)` fetches only requested layers; `AcquireScope::ComponentClosure` first expands component dependencies from the relational index. Acquisition passes the complete request to the store's shared count- and decoded-byte-aware planner. Each internal batch reads directory facts, payloads, and component indexes for at most 512 layers and 256 MiB decoded bytes, decompresses and verifies exact lengths plus BLAKE3 in parallel, accumulates the canonical results, and validates the complete replacement before mutating the uniquely owned live font in place. Validated identity sets become the final index entries rather than a temporary duplicate; shared font snapshots still use copy-on-write. A malformed batch does not replace the live cache. Save/export explicitly acquire all layers before creating their complete snapshots. + +## Profiling + +`profile_streaming_import` uses the same public `stream_into` three-stage pipeline as the workspace and reports foreign-directory, parse, MessagePack encode, compression, SQLite write, commit, durable-finalization, native-directory materialization, reopen, decoded/stored BLOB, and database measurements without putting machine-specific timing assertions in tests: + +```bash +cargo build --release -p shift-workspace --example profile_streaming_import +/usr/bin/time -v target/release/examples/profile_streaming_import \ + /path/to/font-or-project /tmp/import.sqlite +``` + +Use `RAYON_NUM_THREADS`, `SHIFT_IMPORT_BATCH_GLYPHS`, and `SHIFT_IMPORT_BATCH_LAYERS` to compare worker and bounded-batch limits. `SHIFT_IMPORT_PROGRESS_BATCHES` controls machine-readable periodic lines containing cumulative batches, glyphs, layers, parse, pack, compression, SQLite, wall time, and throughput. MessagePack encoding, hashing, and compression use Rayon; the SQLite writer remains single-threaded. + +A repeatable ignored corpus gate exercises streaming import, directory-only +resume, and bounded acquisition without checking large fonts into Git: + +```bash +SHIFT_STRESS_FONT=/path/to/SourceHanSans-VF.ttf \ +SHIFT_STRESS_MIN_GLYPHS=65000 SHIFT_STRESS_MIN_LAYERS=65000 \ +cargo test -p shift-workspace configured_large_corpus_streams_resumes_and_acquires \ + -- --ignored --nocapture +``` + +Set both expectations for the selected, checksum-pinned local corpus. Timing and +RSS remain profiler observations rather than CI assertions. + ## Verification - `cargo test -p shift-workspace` diff --git a/crates/shift-workspace/examples/profile_streaming_import.rs b/crates/shift-workspace/examples/profile_streaming_import.rs new file mode 100644 index 00000000..56c40137 --- /dev/null +++ b/crates/shift-workspace/examples/profile_streaming_import.rs @@ -0,0 +1,168 @@ +use std::{ + env, fs, + path::PathBuf, + time::{Duration, Instant}, +}; + +use shift_backends::{ImportBatchLimit, font_loader::FontLoader}; +use shift_store::{ShiftStore, WorkspaceState}; +use shift_workspace::{AcquireScope, FontWorkspace, stream_into}; + +fn ms(duration: Duration) -> f64 { + duration.as_secs_f64() * 1_000.0 +} + +fn main() -> Result<(), Box> { + let batch_glyphs = env::var("SHIFT_IMPORT_BATCH_GLYPHS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|value| *value > 0) + .unwrap_or(512); + let batch_layers = env::var("SHIFT_IMPORT_BATCH_LAYERS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|value| *value > 0) + .unwrap_or(1_024); + let batch_limit = ImportBatchLimit::new(batch_glyphs, batch_layers); + let progress_batches = env::var("SHIFT_IMPORT_PROGRESS_BATCHES") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|value| *value > 0) + .unwrap_or(32); + + let source = PathBuf::from(env::args_os().nth(1).ok_or("missing source path")?); + let store_path = PathBuf::from(env::args_os().nth(2).ok_or("missing store path")?); + let _ = fs::remove_file(&store_path); + let source_str = source.to_str().ok_or("invalid source path")?; + let total_started = Instant::now(); + + let started = Instant::now(); + let import = FontLoader::new().stream_font(source_str)?; + println!("directory_ms={:.3}", ms(started.elapsed())); + let started = Instant::now(); + let directory = import.directory(); + println!("publish_directory_ms={:.3}", ms(started.elapsed())); + println!("declared_glyph_count={}", import.glyph_count()); + println!("published_glyph_count={}", directory.len()); + drop(directory); + println!("batch_glyph_limit={batch_glyphs}"); + println!("batch_layer_limit={batch_layers}"); + println!("progress_batch_interval={progress_batches}"); + + let started = Instant::now(); + let mut store = ShiftStore::open_for_import(&store_path)?; + let mut writer = store.begin_import(import.header())?; + println!("store_setup_ms={:.3}", ms(started.elapsed())); + + let pipeline_started = Instant::now(); + let mut parse = Duration::ZERO; + let mut pack = Duration::ZERO; + let mut compression = Duration::ZERO; + let mut sqlite = Duration::ZERO; + let mut glyph_count = 0; + let mut layer_count = 0; + let mut batches = 0; + stream_into(import, &mut writer, batch_limit, |batch| { + parse += batch.parse_elapsed; + pack += batch.pack_elapsed; + compression += batch.compression_elapsed; + sqlite += batch.sqlite_elapsed; + glyph_count += batch.glyph_count; + layer_count += batch.layer_count; + batches += 1; + + if batches % progress_batches == 0 { + let wall = pipeline_started.elapsed(); + println!( + "progress batches={batches} glyphs={glyph_count} layers={layer_count} parse_ms={:.3} pack_ms={:.3} compression_ms={:.3} sqlite_ms={:.3} wall_ms={:.3} glyphs_per_sec={:.1}", + ms(parse), + ms(pack), + ms(compression), + ms(sqlite), + ms(wall), + glyph_count as f64 / wall.as_secs_f64() + ); + } + })?; + println!("parse_ms={:.3}", ms(parse)); + println!("pack_ms={:.3}", ms(pack)); + println!("compression_ms={:.3}", ms(compression)); + println!("sqlite_ms={:.3}", ms(sqlite)); + println!("pipeline_wall_ms={:.3}", ms(pipeline_started.elapsed())); + println!("batch_count={batches}"); + println!("glyph_count={glyph_count}"); + println!("streamed_layer_count={layer_count}"); + + let started = Instant::now(); + writer.finish()?; + println!("commit_ms={:.3}", ms(started.elapsed())); + let started = Instant::now(); + store.finish_import()?; + println!("finalize_import_ms={:.3}", ms(started.elapsed())); + store.set_workspace_state(WorkspaceState::imported(&source, None))?; + + let started = Instant::now(); + let directory = store.load_font_directory()?; + println!("materialize_directory_ms={:.3}", ms(started.elapsed())); + println!("foreign_import_ms={:.3}", ms(total_started.elapsed())); + println!("materialized_glyph_count={}", directory.glyph_count()); + println!("store_bytes={}", fs::metadata(&store_path)?.len()); + let started = Instant::now(); + let layer_directory = store.list_glyph_layer_directory()?; + println!("layer_count={}", layer_directory.len()); + println!( + "packed_bytes={}", + layer_directory + .iter() + .map(|entry| entry.decoded_byte_length) + .sum::() + ); + println!( + "stored_payload_bytes={}", + layer_directory + .iter() + .map(|entry| entry.stored_byte_length) + .sum::() + ); + println!("inspection_ms={:.3}", ms(started.elapsed())); + drop(directory); + drop(store); + + let started = Instant::now(); + let mut workspace = FontWorkspace::resume(&store_path)?; + println!("native_resume_ms={:.3}", ms(started.elapsed())); + println!("resumed_glyph_count={}", workspace.font().glyph_count()); + println!("resumed_loaded_layers={}", workspace.loaded_layer_count()); + let first_glyph_id = workspace + .font() + .glyph_id_by_name(".notdef") + .or_else(|| workspace.font().glyphs().next().map(|glyph| glyph.id())); + if let Some(glyph_id) = first_glyph_id { + let layer_ids = workspace + .font() + .glyph(glyph_id.clone()) + .unwrap() + .layers() + .keys() + .cloned() + .collect::>(); + let started = Instant::now(); + workspace.acquire_glyphs(std::slice::from_ref(&glyph_id), AcquireScope::Glyphs)?; + println!("one_glyph_acquire_ms={:.3}", ms(started.elapsed())); + println!("one_glyph_loaded_layers={}", workspace.loaded_layer_count()); + + let probe_store = ShiftStore::open(&store_path)?; + let started = Instant::now(); + let layers = probe_store.load_glyph_layers(&layer_ids)?; + println!("one_glyph_batch_decode_ms={:.3}", ms(started.elapsed())); + let mut probe_font = workspace.font().clone(); + let started = Instant::now(); + probe_font.replace_glyph_layers(layers)?; + println!("one_glyph_install_ms={:.3}", ms(started.elapsed())); + + let started = Instant::now(); + workspace.evict_glyphs(&[glyph_id])?; + println!("one_glyph_evict_ms={:.3}", ms(started.elapsed())); + } + Ok(()) +} diff --git a/crates/shift-workspace/src/import_pipeline.rs b/crates/shift-workspace/src/import_pipeline.rs new file mode 100644 index 00000000..67f4a2f2 --- /dev/null +++ b/crates/shift-workspace/src/import_pipeline.rs @@ -0,0 +1,114 @@ +use std::time::{Duration, Instant}; + +use shift_backends::{FontImport, ImportBatchLimit}; +use shift_font::Glyph; +use shift_store::{FontImportWriter, GlyphWriteBatch, encode_glyph_batch}; + +use crate::WorkspaceError; + +#[derive(Clone, Copy, Debug)] +pub struct ImportBatchProgress { + pub glyph_count: usize, + pub layer_count: usize, + pub parse_elapsed: Duration, + pub pack_elapsed: Duration, + pub compression_elapsed: Duration, + pub sqlite_elapsed: Duration, +} + +struct ParsedImportBatch { + glyphs: Vec, + glyph_count: usize, + layer_count: usize, + parse_elapsed: Duration, +} + +struct PackedImportBatch { + glyphs: GlyphWriteBatch, + glyph_count: usize, + layer_count: usize, + parse_elapsed: Duration, +} + +/// Runs a bounded parser/packer/SQLite pipeline with one writer. Rendezvous +/// channels keep one batch active per stage without retaining queued batches. +/// The observer runs after each batch is staged in the open transaction; it +/// must remain lightweight so it does not throttle the writer. +pub fn stream_into( + mut import: FontImport, + writer: &mut FontImportWriter<'_>, + batch_limit: ImportBatchLimit, + mut observe: impl FnMut(ImportBatchProgress), +) -> Result<(), WorkspaceError> { + let (parsed_sender, parsed_receiver) = std::sync::mpsc::sync_channel(0); + let (packed_sender, packed_receiver) = std::sync::mpsc::sync_channel(0); + std::thread::scope(|scope| -> Result<(), WorkspaceError> { + scope.spawn(move || { + loop { + let started = Instant::now(); + match import.next_batch(batch_limit) { + Ok(glyphs) if glyphs.is_empty() => break, + Ok(glyphs) => { + let batch = ParsedImportBatch { + glyph_count: glyphs.len(), + layer_count: glyphs.iter().map(|glyph| glyph.layers().len()).sum(), + glyphs, + parse_elapsed: started.elapsed(), + }; + if parsed_sender.send(Ok(batch)).is_err() { + break; + } + } + Err(error) => { + let _ = parsed_sender.send(Err(error)); + break; + } + } + } + }); + + scope.spawn(move || { + for result in parsed_receiver { + let batch = match result { + Ok(batch) => batch, + Err(error) => { + let _ = packed_sender.send(Err(WorkspaceError::from(error))); + break; + } + }; + let glyphs = match encode_glyph_batch(batch.glyphs) { + Ok(glyphs) => glyphs, + Err(error) => { + let _ = packed_sender.send(Err(WorkspaceError::from(error))); + break; + } + }; + let batch = PackedImportBatch { + glyphs, + glyph_count: batch.glyph_count, + layer_count: batch.layer_count, + parse_elapsed: batch.parse_elapsed, + }; + if packed_sender.send(Ok(batch)).is_err() { + break; + } + } + }); + + for result in packed_receiver { + let batch = result?; + let pack_elapsed = batch.glyphs.pack_elapsed(); + let compression_elapsed = batch.glyphs.compression_elapsed(); + let sqlite_elapsed = writer.write_packed_glyph_batch(batch.glyphs)?; + observe(ImportBatchProgress { + glyph_count: batch.glyph_count, + layer_count: batch.layer_count, + parse_elapsed: batch.parse_elapsed, + pack_elapsed, + compression_elapsed, + sqlite_elapsed, + }); + } + Ok(()) + }) +} diff --git a/crates/shift-workspace/src/import_staging.rs b/crates/shift-workspace/src/import_staging.rs new file mode 100644 index 00000000..eb074279 --- /dev/null +++ b/crates/shift-workspace/src/import_staging.rs @@ -0,0 +1,83 @@ +use std::{ + ffi::OsString, + io, + path::{Path, PathBuf}, +}; + +use tempfile::TempPath; + +pub(crate) fn create_import_staging_path(destination: &Path) -> io::Result { + let parent = match destination.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + }; + let staged = tempfile::Builder::new() + .prefix(".shift-import-") + .suffix(".sqlite") + .tempfile_in(parent)?; + Ok(staged.into_temp_path()) +} + +pub(crate) fn install_import_store(staged: TempPath, destination: &Path) -> io::Result<()> { + remove_staging_sidecars(&staged)?; + staged.persist(destination).map_err(|error| error.error)?; + sync_parent_directory(destination) +} + +fn remove_staging_sidecars(staged: &Path) -> io::Result<()> { + for suffix in ["-wal", "-shm"] { + match std::fs::remove_file(sqlite_sidecar_path(staged, suffix)) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(()) +} + +fn sqlite_sidecar_path(path: &Path, suffix: &str) -> PathBuf { + let mut sidecar = OsString::from(path.as_os_str()); + sidecar.push(suffix); + PathBuf::from(sidecar) +} + +#[cfg(unix)] +pub(crate) fn sync_parent_directory(path: &Path) -> io::Result<()> { + if let Some(parent) = path.parent() + && !parent.as_os_str().is_empty() + { + std::fs::File::open(parent)?.sync_all()?; + } + Ok(()) +} + +#[cfg(not(unix))] +pub(crate) fn sync_parent_directory(_path: &Path) -> io::Result<()> { + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn staged_install_replaces_destination_and_cleans_staging_file() { + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("working.sqlite"); + std::fs::write(&destination, b"old").unwrap(); + let staged = create_import_staging_path(&destination).unwrap(); + let staged_path = staged.to_path_buf(); + let staged_wal_path = sqlite_sidecar_path(&staged_path, "-wal"); + let staged_shm_path = sqlite_sidecar_path(&staged_path, "-shm"); + std::fs::write(&staged_path, b"new").unwrap(); + std::fs::write(&staged_wal_path, b"").unwrap(); + std::fs::write(&staged_shm_path, b"").unwrap(); + + install_import_store(staged, &destination).unwrap(); + + assert_eq!(std::fs::read(&destination).unwrap(), b"new"); + assert!(!staged_path.exists()); + assert!(!staged_wal_path.exists()); + assert!(!staged_shm_path.exists()); + } +} diff --git a/crates/shift-workspace/src/layer_residency.rs b/crates/shift-workspace/src/layer_residency.rs new file mode 100644 index 00000000..872062c3 --- /dev/null +++ b/crates/shift-workspace/src/layer_residency.rs @@ -0,0 +1,60 @@ +use std::collections::HashSet; + +use shift_font::{Font, LayerId}; + +/// Tracks which directory-listed layers do not currently carry authored +/// geometry in the live font projection. +/// +/// Directory placeholders are an implementation detail of the current lazy +/// workspace. Keeping every residency transition here prevents a valid empty +/// layer from being mistaken for loaded authored state at persistence +/// boundaries. +#[derive(Default)] +pub(crate) struct LayerResidency { + unloaded: HashSet, +} + +impl LayerResidency { + pub(crate) fn with_unloaded(layer_ids: impl IntoIterator) -> Self { + Self { + unloaded: layer_ids.into_iter().collect(), + } + } + + pub(crate) fn is_unloaded(&self, layer_id: &LayerId) -> bool { + self.unloaded.contains(layer_id) + } + + pub(crate) fn requested_unloaded( + &self, + layer_ids: impl IntoIterator, + ) -> Vec { + layer_ids + .into_iter() + .filter(|layer_id| self.is_unloaded(layer_id)) + .collect() + } + + pub(crate) fn unloaded_layer_ids(&self) -> impl Iterator { + self.unloaded.iter() + } + + pub(crate) fn mark_loaded(&mut self, layer_ids: impl IntoIterator) { + for layer_id in layer_ids { + self.unloaded.remove(&layer_id); + } + } + + pub(crate) fn mark_unloaded(&mut self, layer_ids: impl IntoIterator) { + self.unloaded.extend(layer_ids); + } + + pub(crate) fn retain_directory_layers(&mut self, font: &Font) { + self.unloaded + .retain(|layer_id| font.layer(layer_id.clone()).is_some()); + } + + pub(crate) fn loaded_count(&self, directory_layer_count: usize) -> usize { + directory_layer_count.saturating_sub(self.unloaded.len()) + } +} diff --git a/crates/shift-workspace/src/ledger.rs b/crates/shift-workspace/src/ledger.rs index 9ee2a07c..413951c7 100644 --- a/crates/shift-workspace/src/ledger.rs +++ b/crates/shift-workspace/src/ledger.rs @@ -7,14 +7,16 @@ //! a renderer reload (it lives with the workspace process), not a utility //! crash; a SQLite ledger table is the later upgrade if that ever matters. +use std::sync::Arc; + use shift_font::{ - Axis, AxisMapping, FontMetadata, Glyph, GlyphId, GlyphLayer, GlyphName, MetricDefinition, - NamedInstance, Source, SourceId, + Axis, AxisMapping, FontMetadata, Glyph, GlyphId, GlyphLayer, GlyphName, LayerId, + MetricDefinition, NamedInstance, Source, SourceId, }; -/// Generous bound so a marathon session cannot grow memory unboundedly; -/// oldest entries fall off first. -const MAX_ENTRIES: usize = 100; +/// Maximum entries retained independently by each stack. The oldest entry on +/// the stack being extended falls off first; a fresh apply also clears redo. +const MAX_ENTRIES_PER_STACK: usize = 100; #[derive(Clone)] pub enum LedgerStep { @@ -85,8 +87,11 @@ pub struct GlyphIdentity { #[derive(Clone)] pub struct LayerPair { - pub pre: GlyphLayer, - pub post: GlyphLayer, + pub pre: Arc, + pub post: Arc, + /// Fixed when the entry is created. Both replay directions preserve + /// topology when false and must publish complete structure when true. + pub structural: bool, } #[derive(Clone)] @@ -95,6 +100,40 @@ pub struct LedgerEntry { pub steps: Vec, } +impl LedgerEntry { + /// Layers whose persisted current values may be needed before replay. + /// Snapshot-backed replay can then replace loaded authored state without + /// leaving workspace residency bookkeeping pointed at a placeholder. + pub(crate) fn layer_ids(&self) -> Vec { + self.steps + .iter() + .flat_map(|step| match step { + LedgerStep::Layers(pairs) => pairs + .iter() + .flat_map(|pair| [pair.pre.id(), pair.post.id()]) + .collect(), + LedgerStep::Glyph { pre, post } => pre + .iter() + .chain(post.iter()) + .flat_map(|glyph| glyph.layers().keys().cloned()) + .collect(), + LedgerStep::GlyphLayer { pre, post, .. } => pre + .iter() + .chain(post.iter()) + .map(|layer| layer.id()) + .collect(), + LedgerStep::FontMetadata { .. } + | LedgerStep::Axis { .. } + | LedgerStep::AxisMappings { .. } + | LedgerStep::MetricDefinitions { .. } + | LedgerStep::NamedInstances { .. } + | LedgerStep::Source { .. } + | LedgerStep::GlyphIdentity { .. } => Vec::new(), + }) + .collect() + } +} + #[derive(Default)] pub struct Ledger { undo: Vec, @@ -105,11 +144,7 @@ impl Ledger { /// Records an applied entry. A fresh apply truncates the redo stack. pub fn push(&mut self, entry: LedgerEntry) { self.redo.clear(); - self.undo.push(entry); - - if self.undo.len() > MAX_ENTRIES { - self.undo.remove(0); - } + push_bounded(&mut self.undo, entry); } /// Pops the entry to undo; the caller replays its pre states and must @@ -121,12 +156,12 @@ impl Ledger { } pub fn record_undone(&mut self, entry: LedgerEntry) { - self.redo.push(entry); + push_bounded(&mut self.redo, entry); } /// Hands a popped undo entry back after a failed replay. pub fn restore_undo(&mut self, entry: LedgerEntry) { - self.undo.push(entry); + push_bounded(&mut self.undo, entry); } /// Pops the entry to redo; hand back via [`Ledger::record_redone`] after @@ -137,11 +172,59 @@ impl Ledger { } pub fn record_redone(&mut self, entry: LedgerEntry) { - self.undo.push(entry); + push_bounded(&mut self.undo, entry); } /// Hands a popped redo entry back after a failed replay. pub fn restore_redo(&mut self, entry: LedgerEntry) { - self.redo.push(entry); + push_bounded(&mut self.redo, entry); + } +} + +fn push_bounded(stack: &mut Vec, entry: LedgerEntry) { + stack.push(entry); + if stack.len() > MAX_ENTRIES_PER_STACK { + stack.remove(0); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn each_stack_drops_its_oldest_entry_independently() { + let mut ledger = Ledger::default(); + for index in 0..=MAX_ENTRIES_PER_STACK { + ledger.push(entry(index)); + } + assert_eq!(ledger.undo.len(), MAX_ENTRIES_PER_STACK); + assert_eq!(ledger.undo[0].label.as_deref(), Some("1")); + + let undo = std::mem::take(&mut ledger.undo); + for entry in undo { + ledger.record_undone(entry); + } + ledger.record_undone(entry(MAX_ENTRIES_PER_STACK + 1)); + assert_eq!(ledger.redo.len(), MAX_ENTRIES_PER_STACK); + assert_eq!(ledger.redo[0].label.as_deref(), Some("2")); + } + + #[test] + fn fresh_apply_clears_the_bounded_redo_stack() { + let mut ledger = Ledger::default(); + ledger.record_undone(entry(1)); + + ledger.push(entry(2)); + + assert!(ledger.redo.is_empty()); + assert_eq!(ledger.undo.len(), 1); + } + + fn entry(index: usize) -> LedgerEntry { + LedgerEntry { + label: Some(index.to_string()), + steps: Vec::new(), + } } } diff --git a/crates/shift-workspace/src/lib.rs b/crates/shift-workspace/src/lib.rs index 3306d266..b835dcfb 100644 --- a/crates/shift-workspace/src/lib.rs +++ b/crates/shift-workspace/src/lib.rs @@ -1,9 +1,13 @@ +mod import_pipeline; +mod import_staging; +mod layer_residency; mod ledger; mod new_workspace; mod source_identity; mod workspace; +pub use import_pipeline::{ImportBatchProgress, stream_into}; pub use ledger::{LayerPair, Ledger, LedgerEntry, LedgerStep}; pub use new_workspace::NewWorkspace; pub use source_identity::{PackageDraft, PackageIdentity}; -pub use workspace::{FontWorkspace, WorkspaceError, WorkspaceSource}; +pub use workspace::{AcquireScope, FontWorkspace, WorkspaceError, WorkspaceSource}; diff --git a/crates/shift-workspace/src/workspace.rs b/crates/shift-workspace/src/workspace.rs index 47fe275b..aa6d7469 100644 --- a/crates/shift-workspace/src/workspace.rs +++ b/crates/shift-workspace/src/workspace.rs @@ -1,23 +1,29 @@ use std::{ + collections::{HashMap, HashSet}, io, path::{Path, PathBuf}, + sync::Arc, }; -use shift_backends::{FontExportRequest, FontExportResult, FontExporter, font_loader::FontLoader}; +use shift_backends::{ + FontExportRequest, FontExportResult, FontExporter, ImportBatchLimit, font_loader::FontLoader, +}; use shift_font::{ AppliedIntents, Axis, AxisId, FontChange, FontChangeSet, FontIntent, FontIntentSet, - FontMetadata, Glyph, GlyphId, GlyphLayer, MetricDefinition, NamedInstance, Source, SourceId, - TouchedLayer, error::CoreError, + FontMetadata, Glyph, GlyphId, GlyphLayer, LayerId, MetricDefinition, NamedInstance, Source, + SourceId, TouchedLayer, error::CoreError, }; use shift_source::ShiftSourcePackage; use shift_store::{ShiftStore, SourceIdentitySnapshot, WorkspaceSourceKind, WorkspaceState}; -use crate::NewWorkspace; +use crate::import_staging::{create_import_staging_path, install_import_store}; +use crate::layer_residency::LayerResidency; use crate::ledger::{GlyphIdentity, LayerPair, Ledger, LedgerEntry, LedgerStep}; use crate::source_identity::{ PackageDraft, PackageIdentity, package_identity, source_identity_snapshot, validate_source_identity_for_save, }; +use crate::{NewWorkspace, stream_into}; #[derive(Debug, thiserror::Error)] pub enum WorkspaceError { @@ -51,6 +57,9 @@ pub enum WorkspaceError { #[error("corrupt working store: {0}")] CorruptWorkingStore(String), + #[error("refusing to persist unloaded glyph layer {0}")] + UnloadedLayerMutation(LayerId), + #[error("invalid UTF-8 in workspace path: {0}")] InvalidPathUtf8(PathBuf), @@ -65,14 +74,36 @@ pub enum WorkspaceSource { Imported { original_path: PathBuf }, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AcquireScope { + Glyphs, + ComponentClosure, +} + pub struct FontWorkspace { font: shift_font::Font, source: WorkspaceSource, store: ShiftStore, ledger: Ledger, + residency: LayerResidency, } impl FontWorkspace { + fn from_store( + font: shift_font::Font, + source: WorkspaceSource, + store: ShiftStore, + residency: LayerResidency, + ) -> Self { + Self { + font, + source, + store, + ledger: Ledger::default(), + residency, + } + } + pub fn create_untitled( store_path: impl AsRef, new_workspace: NewWorkspace, @@ -84,12 +115,12 @@ impl FontWorkspace { store.replace_font_state(&font)?; store.set_workspace_state(WorkspaceState::untitled(None))?; - Ok(Self { + Ok(Self::from_store( font, - source: WorkspaceSource::Untitled, + WorkspaceSource::Untitled, store, - ledger: Ledger::default(), - }) + LayerResidency::default(), + )) } pub fn create_package( @@ -115,6 +146,7 @@ impl FontWorkspace { } pub fn save(&mut self) -> Result<(), WorkspaceError> { + self.acquire_all_layers()?; match self.source.clone() { WorkspaceSource::Package { path } => { self.validate_save_target(&path)?; @@ -130,6 +162,7 @@ impl FontWorkspace { } pub fn save_as(&mut self, source_path: impl AsRef) -> Result<(), WorkspaceError> { + self.acquire_all_layers()?; let source_package = ShiftSourcePackage::save_font_as(source_path, &self.font)?; let identity = source_identity_snapshot(source_package.path())?; self.source = WorkspaceSource::Package { @@ -145,15 +178,14 @@ impl FontWorkspace { let state = store .workspace_state()? .ok_or_else(|| WorkspaceError::CorruptWorkingStore("missing workspace_state".into()))?; - let font = store.load_font_state()?; + let font = store.load_font_directory()?; + let residency = LayerResidency::with_unloaded( + font.glyphs() + .flat_map(|glyph| glyph.layers().keys().cloned()), + ); let source = source_from_workspace_state(&state)?; - Ok(Self { - font, - source, - store, - ledger: Ledger::default(), - }) + Ok(Self::from_store(font, source, store, residency)) } pub fn resume_for_source( @@ -204,7 +236,11 @@ impl FontWorkspace { }) } - pub fn export(&self, request: FontExportRequest) -> Result { + pub fn export( + &mut self, + request: FontExportRequest, + ) -> Result { + self.acquire_all_layers()?; FontExporter::new() .export(&self.font, request) .map_err(WorkspaceError::from) @@ -219,6 +255,13 @@ impl FontWorkspace { set: FontIntentSet, label: Option, ) -> Result { + let required_layers = set + .intents + .iter() + .flat_map(|intent| intent.required_layer_ids(&self.font)) + .collect::>(); + self.acquire_layers(required_layers)?; + let mut pre = FontLevelPreState::default(); for intent in &set.intents { let Some(layer_id) = intent.layer_id() else { @@ -228,13 +271,14 @@ impl FontWorkspace { if pre.layers.iter().any(|pre| pre.layer.id() == *layer_id) { continue; } - if let Some(layer) = self.font.layer(layer_id.clone()) - && let Some(glyph_id) = self.font.glyph_id_by_layer(layer_id.clone()) + if let Some(glyph_id) = self.font.glyph_id_by_layer(layer_id.clone()) + && let Some(layer) = self + .font + .glyph(glyph_id.clone()) + .and_then(|glyph| glyph.layers().get(layer_id)) + .cloned() { - pre.layers.push(PreLayer { - glyph_id, - layer: layer.clone(), - }); + pre.layers.push(PreLayer { glyph_id, layer }); } } @@ -265,6 +309,7 @@ impl FontWorkspace { .map(|pre| LayerPair { pre: pre.layer.clone(), post, + structural: touched.structural, }) }) .collect(); @@ -396,7 +441,7 @@ impl FontWorkspace { .find(|pre| { pre.glyph_id == change.glyph_id && pre.layer.id() == change.layer_id }) - .map(|pre| Box::new(pre.layer.clone())), + .map(|pre| Box::new(pre.layer.as_ref().clone())), post: None, }); } @@ -484,6 +529,8 @@ impl FontWorkspace { entry: &LedgerEntry, side: ReplaySide, ) -> Result { + self.acquire_layers(entry.layer_ids())?; + let mut named_instances = None; let mut metric_definitions = None; let mut steps = entry @@ -593,8 +640,19 @@ impl FontWorkspace { next_font: shift_font::Font, change_set: FontChangeSet, ) -> Result<(), WorkspaceError> { - self.store.apply_change_set(&change_set)?; + if let Some(layer_id) = change_set + .changes + .iter() + .filter_map(FontChange::layer_id) + .find(|layer_id| self.residency.is_unloaded(layer_id)) + { + return Err(WorkspaceError::UnloadedLayerMutation(layer_id.clone())); + } + + self.store + .apply_change_set_with_font(&change_set, &next_font)?; self.font = next_font; + self.residency.retain_directory_layers(&self.font); Ok(()) } @@ -671,14 +729,14 @@ impl FontWorkspace { let identity = source_identity_snapshot(source_package.path())?; store.set_workspace_state(WorkspaceState::package(identity, None))?; - Ok(Self { + Ok(Self::from_store( font, - source: WorkspaceSource::Package { + WorkspaceSource::Package { path: source_package.path().to_path_buf(), }, store, - ledger: Ledger::default(), - }) + LayerResidency::default(), + )) } fn import_font( @@ -689,22 +747,183 @@ impl FontWorkspace { let import_path_str = import_path .to_str() .ok_or_else(|| WorkspaceError::InvalidPathUtf8(import_path.to_path_buf()))?; - let font = FontLoader::new().read_font(import_path_str)?; + let loader = FontLoader::new(); + match loader.stream_font(import_path_str) { + Ok(import) => { + let store_path = store_path.as_ref(); + if store_path.exists() { + let existing = ShiftStore::open(store_path)?; + if existing.workspace_state()?.is_some() { + return Err(shift_store::StoreError::ImportDestinationNotEmpty( + store_path.to_path_buf(), + ) + .into()); + } + } + + let staged_path = create_import_staging_path(store_path)?; + let mut store = ShiftStore::open_for_import(&staged_path)?; + let mut writer = store.begin_import(import.header())?; + stream_into(import, &mut writer, ImportBatchLimit::default(), |_| {})?; + writer.finish()?; + store.set_workspace_state(WorkspaceState::imported(import_path, None))?; + store.finish_import()?; + let font = store.load_font_directory()?; + let residency = LayerResidency::with_unloaded( + font.glyphs() + .flat_map(|glyph| glyph.layers().keys().cloned()), + ); + drop(store); + install_import_store(staged_path, store_path)?; + let store = ShiftStore::open(store_path)?; + + return Ok(Self::from_store( + font, + WorkspaceSource::Imported { + original_path: import_path.to_path_buf(), + }, + store, + residency, + )); + } + Err(shift_backends::BackendError::StreamingUnsupported { .. }) => {} + Err(error) => return Err(error.into()), + } + + let font = loader.read_font(import_path_str)?; let mut store = ShiftStore::open(store_path)?; store.set_font_info(font_info_from_font(&font))?; store.replace_font_state(&font)?; store.set_workspace_state(WorkspaceState::imported(import_path, None))?; - Ok(Self { + Ok(Self::from_store( font, - source: WorkspaceSource::Imported { + WorkspaceSource::Imported { original_path: import_path.to_path_buf(), }, store, - ledger: Ledger::default(), - }) + LayerResidency::default(), + )) } + /// Explicitly acquires requested glyph payloads. Component closure expands + /// dependencies from relational indexes before any BLOB is read. + pub fn acquire_glyphs( + &mut self, + glyph_ids: &[GlyphId], + scope: AcquireScope, + ) -> Result<(), WorkspaceError> { + let glyph_ids = if scope == AcquireScope::ComponentClosure { + self.store + .referenced_glyph_closure(glyph_ids.iter().cloned())? + } else { + glyph_ids.to_vec() + }; + let layer_ids = glyph_ids + .into_iter() + .flat_map(|glyph_id| { + self.font + .glyph(glyph_id) + .into_iter() + .flat_map(|glyph| glyph.layers().keys().cloned()) + .collect::>() + }) + .collect::>(); + self.acquire_layers(layer_ids) + } + + pub fn acquire_all_layers(&mut self) -> Result<(), WorkspaceError> { + let layer_ids = self + .residency + .unloaded_layer_ids() + .cloned() + .collect::>(); + self.acquire_layers(layer_ids) + } + + fn acquire_layers( + &mut self, + layer_ids: impl IntoIterator, + ) -> Result<(), WorkspaceError> { + let mut layer_ids = self.residency.requested_unloaded(layer_ids); + layer_ids.sort_by(|left, right| left.as_str().cmp(right.as_str())); + layer_ids.dedup(); + if layer_ids.is_empty() { + return Ok(()); + } + + // Decode through bounded SQLite reads. Font validates the complete + // replacement batch before mutating, so malformed input leaves the + // live cache unchanged without copying the complete directory. + let layers = self.store.load_glyph_layers(&layer_ids)?; + self.font.replace_glyph_layers(layers)?; + self.residency.mark_loaded(layer_ids); + Ok(()) + } + + /// Drops clean in-memory payloads back to directory placeholders. Every + /// authored edit is committed before the live font swap, so eviction can + /// never discard unpersisted state. + /// + /// This is currently a test/profiler surface; the bridge does not yet own + /// a production eviction policy. + pub fn evict_glyphs(&mut self, glyph_ids: &[GlyphId]) -> Result<(), WorkspaceError> { + let mut placeholders = Vec::new(); + let mut evicted = Vec::new(); + let mut seen_layer_ids = HashSet::new(); + for glyph_id in glyph_ids { + let Some(glyph) = self.font.glyph(glyph_id.clone()) else { + continue; + }; + for layer in glyph.layers().values().map(|layer| layer.as_ref()) { + if self.residency.is_unloaded(&layer.id()) || !seen_layer_ids.insert(layer.id()) { + continue; + } + let mut placeholder = + GlyphLayer::with_width(layer.id(), layer.source_id(), layer.width()); + placeholder.set_height(layer.height()); + placeholders.push(placeholder); + evicted.push(layer.id()); + } + } + + self.font.replace_glyph_layers(placeholders)?; + self.residency.mark_unloaded(evicted); + Ok(()) + } + + /// Residency instrumentation for tests and import profiling. + pub fn loaded_layer_count(&self) -> usize { + let directory_layer_count = self.font.glyphs().map(|glyph| glyph.layers().len()).sum(); + self.residency.loaded_count(directory_layer_count) + } + + pub fn glyph_component_references( + &self, + ) -> Result>, WorkspaceError> { + self.store.glyph_component_references().map_err(Into::into) + } + + pub fn referenced_glyph_ids_for_glyph( + &self, + glyph_id: &GlyphId, + ) -> Result, WorkspaceError> { + self.store + .referenced_glyph_ids_for_glyph(glyph_id) + .map_err(Into::into) + } + + pub fn dependent_glyph_ids_for_layers( + &self, + layer_ids: &[LayerId], + ) -> Result, WorkspaceError> { + self.store + .dependent_glyph_ids_for_layers(layer_ids) + .map_err(Into::into) + } + + /// Metadata and directory are always present. Layer payloads are present + /// only after explicit acquisition. pub fn font(&self) -> &shift_font::Font { &self.font } @@ -749,7 +968,7 @@ impl FontWorkspace { #[derive(Clone)] struct PreLayer { glyph_id: GlyphId, - layer: GlyphLayer, + layer: Arc, } #[derive(Default)] @@ -822,7 +1041,12 @@ fn capture_font_level_pre_state( } for glyph in font.glyphs() { - let Some(layer) = glyph.layer_for_source(source_id.clone()) else { + let Some(layer) = glyph + .layers() + .values() + .find(|layer| layer.source_id() == *source_id) + .cloned() + else { continue; }; if pre.layers.iter().any(|pre| pre.layer.id() == layer.id()) { @@ -830,7 +1054,7 @@ fn capture_font_level_pre_state( } pre.layers.push(PreLayer { glyph_id: glyph.id(), - layer: layer.clone(), + layer, }); } } @@ -897,25 +1121,28 @@ fn replay_layer_pairs( changes: &mut FontChangeSet, touched: &mut Vec, ) -> Result<(), WorkspaceError> { + let mut replayed = Vec::with_capacity(pairs.len()); + let mut structural_replacements = Vec::with_capacity(pairs.len()); for pair in pairs { let replacement = match side { ReplaySide::Pre => pair.pre, ReplaySide::Post => pair.post, }; - let layer_id = replacement.id(); - let layer = font - .layer_mut(layer_id.clone()) - .ok_or(CoreError::LayerNotFound(layer_id))?; - *layer = replacement; + if pair.structural { + structural_replacements.push(replacement.clone()); + } else { + font.replace_glyph_layer_values(replacement.id(), &replacement.interpolation_values())?; + } + replayed.push((replacement, pair.structural)); + } + font.replace_glyph_layers(structural_replacements)?; + for (layer, structural) in replayed { // Geometry replace persists contours only; metrics ride their // own change so width/height restores reach SQLite too. - changes.push(FontChange::layer_geometry_replaced(layer)); - changes.push(FontChange::layer_metrics_changed(layer)); - touched.push(TouchedLayer { - layer: layer.clone(), - structural: true, - }); + changes.push(FontChange::layer_geometry_replaced(layer.as_ref())); + changes.push(FontChange::layer_metrics_changed(layer.as_ref())); + touched.push(TouchedLayer { layer, structural }); } Ok(()) @@ -939,14 +1166,9 @@ fn replay_glyph( changes.push(FontChange::glyph_created(&glyph)); for layer in glyph.layers().values() { - let layer = layer.as_ref().clone(); - changes.push(FontChange::glyph_layer_created( - glyph.id(), - Some(glyph.glyph_name().clone()), - &layer, - )); + changes.push(FontChange::glyph_layer_created(glyph.id(), layer.as_ref())); touched.push(TouchedLayer { - layer, + layer: layer.clone(), structural: true, }); } @@ -1113,17 +1335,10 @@ fn replay_glyph_layer( } if let Some(layer) = to { - let glyph_name = font - .glyph(glyph_id.clone()) - .map(|glyph| glyph.glyph_name().clone()); - changes.push(FontChange::glyph_layer_created( - glyph_id.clone(), - glyph_name, - &layer, - )); + changes.push(FontChange::glyph_layer_created(glyph_id.clone(), &layer)); font.insert_glyph_layer(glyph_id, layer.clone())?; touched.push(TouchedLayer { - layer, + layer: Arc::new(layer), structural: true, }); } diff --git a/crates/shift-workspace/tests/workspace_test.rs b/crates/shift-workspace/tests/workspace_test.rs index 52ead6f8..509cdc30 100644 --- a/crates/shift-workspace/tests/workspace_test.rs +++ b/crates/shift-workspace/tests/workspace_test.rs @@ -1,12 +1,12 @@ use std::{fs, path::PathBuf}; use shift_font::{ - AnchorId, AnchorSeed, Axis, AxisId, BooleanOp, ContourId, FontChange, FontIntent, - FontIntentSet, GlyphId, GlyphName, LayerId, Location, NamedInstance, NamedInstanceId, PointId, - PointSeed, PointType, SourceId, error::CoreError, + AnchorId, AnchorSeed, Axis, AxisId, BooleanOp, ContourId, Font, FontChange, FontIntent, + FontIntentSet, Glyph, GlyphId, GlyphLayer, GlyphName, LayerId, Location, NamedInstance, + NamedInstanceId, PointId, PointSeed, PointType, SourceId, error::CoreError, }; use shift_source::ShiftSourcePackage; -use shift_workspace::{FontWorkspace, NewWorkspace, WorkspaceError, WorkspaceSource}; +use shift_workspace::{AcquireScope, FontWorkspace, NewWorkspace, WorkspaceError, WorkspaceSource}; fn create_glyph_intent(name: &str, unicodes: Vec) -> FontIntent { FontIntent::CreateGlyph { @@ -145,6 +145,366 @@ fn imports_external_fonts_without_a_save_target() { .family_name .is_some() ); + let entries = fs::read_dir(temp.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect::>(); + assert!( + entries + .iter() + .all(|name| !name.to_string_lossy().starts_with(".shift-import-")), + "staging artifacts remain: {entries:?}" + ); +} + +#[test] +fn large_ttf_reopens_directory_first_and_acquires_only_requested_layers() { + let temp = tempfile::tempdir().unwrap(); + let source_path = fixture("apps/desktop/src/renderer/src/assets/fonts/Inter-VariableFont.ttf"); + let store_path = temp.path().join("inter.sqlite"); + + let mut workspace = FontWorkspace::open(&source_path, &store_path).unwrap(); + let glyph_count = workspace.font().glyph_count(); + let total_layers = workspace + .font() + .glyphs() + .map(|glyph| glyph.layers().len()) + .sum::(); + assert!( + glyph_count > 2_000, + "fixture should exercise a large glyph directory" + ); + assert!(total_layers >= glyph_count); + + let requested = workspace + .font() + .glyphs() + .filter(|glyph| !glyph.layers().is_empty()) + .take(520) + .map(|glyph| glyph.id()) + .collect::>(); + assert!(requested.len() > 512); + assert_eq!(workspace.loaded_layer_count(), 0); + workspace + .acquire_glyphs(&requested, AcquireScope::Glyphs) + .unwrap(); + let expected = requested + .iter() + .flat_map(|glyph_id| { + workspace + .font() + .glyph(glyph_id.clone()) + .unwrap() + .layers() + .values() + .map(|layer| (layer.id(), layer.as_ref().clone())) + .collect::>() + }) + .collect::>(); + drop(workspace); + + let mut resumed = FontWorkspace::resume(&store_path).unwrap(); + assert_eq!(resumed.font().glyph_count(), glyph_count); + assert_eq!(resumed.loaded_layer_count(), 0); + + resumed + .acquire_glyphs(&requested, AcquireScope::Glyphs) + .unwrap(); + + assert_eq!(resumed.loaded_layer_count(), expected.len()); + assert!(resumed.loaded_layer_count() < total_layers); + for (layer_id, layer) in expected { + assert_eq!(resumed.font().layer(layer_id).unwrap(), &layer); + } +} + +/// Manual corpus gate for large fonts that cannot live in the repository. +/// +/// Run with, for example: +/// `SHIFT_STRESS_FONT=/path/to/SourceHanSans-VF.ttf SHIFT_STRESS_MIN_GLYPHS=65000 SHIFT_STRESS_MIN_LAYERS=65000 cargo test -p shift-workspace configured_large_corpus_streams_resumes_and_acquires -- --ignored --nocapture`. +#[test] +#[ignore = "requires SHIFT_STRESS_FONT and explicit corpus expectations"] +fn configured_large_corpus_streams_resumes_and_acquires() { + let source_path = PathBuf::from( + std::env::var_os("SHIFT_STRESS_FONT").expect("SHIFT_STRESS_FONT must name a font source"), + ); + let min_glyphs = std::env::var("SHIFT_STRESS_MIN_GLYPHS") + .expect("SHIFT_STRESS_MIN_GLYPHS must document the selected corpus") + .parse::() + .expect("SHIFT_STRESS_MIN_GLYPHS must be an integer"); + let min_layers = std::env::var("SHIFT_STRESS_MIN_LAYERS") + .expect("SHIFT_STRESS_MIN_LAYERS must document the selected corpus") + .parse::() + .expect("SHIFT_STRESS_MIN_LAYERS must be an integer"); + let temp = tempfile::tempdir().unwrap(); + let store_path = temp.path().join("corpus.sqlite"); + let started = std::time::Instant::now(); + + let mut workspace = FontWorkspace::open(&source_path, &store_path).unwrap(); + let glyph_count = workspace.font().glyph_count(); + let layer_count = workspace + .font() + .glyphs() + .map(|glyph| glyph.layers().len()) + .sum::(); + assert!(glyph_count >= min_glyphs, "got {glyph_count} glyphs"); + assert!(layer_count >= min_layers, "got {layer_count} layers"); + assert_eq!(workspace.loaded_layer_count(), 0); + + let glyph_ids = workspace + .font() + .glyphs() + .filter(|glyph| !glyph.layers().is_empty()) + .take(16) + .map(|glyph| glyph.id()) + .collect::>(); + workspace + .acquire_glyphs(&glyph_ids, AcquireScope::Glyphs) + .unwrap(); + assert!(workspace.loaded_layer_count() > 0); + assert!(workspace.loaded_layer_count() < layer_count); + drop(workspace); + + let resumed = FontWorkspace::resume(&store_path).unwrap(); + assert_eq!(resumed.font().glyph_count(), glyph_count); + assert_eq!(resumed.loaded_layer_count(), 0); + eprintln!( + "corpus={} glyphs={glyph_count} layers={layer_count} store_bytes={} elapsed_ms={:.3}", + source_path.display(), + fs::metadata(&store_path).unwrap().len(), + started.elapsed().as_secs_f64() * 1_000.0 + ); +} + +#[test] +fn designspace_and_ufo_sources_roundtrip_through_lazy_component_acquisition() { + let temp = tempfile::tempdir().unwrap(); + let source_path = fixture("fixtures/fonts/mutatorsans-variable/MutatorSans.designspace"); + let store_path = temp.path().join("mutatorsans.sqlite"); + + let mut workspace = FontWorkspace::open(&source_path, &store_path).unwrap(); + let glyph_count = workspace.font().glyph_count(); + let total_layers = workspace + .font() + .glyphs() + .map(|glyph| glyph.layers().len()) + .sum::(); + assert!(workspace.font().sources().len() >= 4); + assert!(total_layers >= glyph_count * 4); + + let root = workspace + .font() + .glyphs() + .find(|glyph| { + !workspace + .store() + .referenced_glyph_ids_for_glyph(&glyph.id()) + .unwrap() + .is_empty() + }) + .expect("fixture should contain a composite glyph") + .id(); + let closure = workspace + .store() + .referenced_glyph_closure([root.clone()]) + .unwrap(); + assert!(closure.len() > 1); + let closure_layer_count = closure + .iter() + .map(|glyph_id| { + workspace + .font() + .glyph(glyph_id.clone()) + .unwrap() + .layers() + .len() + }) + .sum::(); + assert_eq!(workspace.loaded_layer_count(), 0); + workspace + .acquire_glyphs(std::slice::from_ref(&root), AcquireScope::ComponentClosure) + .unwrap(); + let expected_root_layers = workspace + .font() + .glyph(root.clone()) + .unwrap() + .layers() + .values() + .map(|layer| (layer.id(), layer.as_ref().clone())) + .collect::>(); + drop(workspace); + + let mut resumed = FontWorkspace::resume(&store_path).unwrap(); + assert_eq!(resumed.loaded_layer_count(), 0); + resumed + .acquire_glyphs(std::slice::from_ref(&root), AcquireScope::ComponentClosure) + .unwrap(); + + assert_eq!(resumed.loaded_layer_count(), closure_layer_count); + assert!(resumed.loaded_layer_count() < total_layers); + for (layer_id, layer) in expected_root_layers { + assert_eq!(resumed.font().layer(layer_id).unwrap(), &layer); + } +} + +#[test] +fn ufo_source_roundtrips_through_lazy_acquisition() { + let temp = tempfile::tempdir().unwrap(); + let source_path = fixture("fixtures/fonts/mutatorsans/MutatorSansLightCondensed.ufo"); + let store_path = temp.path().join("mutatorsans-ufo.sqlite"); + + let mut workspace = FontWorkspace::open(&source_path, &store_path).unwrap(); + let glyph_id = workspace + .font() + .glyphs() + .find(|glyph| !glyph.layers().is_empty()) + .expect("fixture should contain a glyph layer") + .id(); + assert_eq!(workspace.loaded_layer_count(), 0); + + workspace + .acquire_glyphs(std::slice::from_ref(&glyph_id), AcquireScope::Glyphs) + .unwrap(); + let expected = workspace + .font() + .glyph(glyph_id.clone()) + .unwrap() + .layers() + .values() + .map(|layer| (layer.id(), layer.as_ref().clone())) + .collect::>(); + assert!(!expected.is_empty()); + drop(workspace); + + let mut resumed = FontWorkspace::resume(&store_path).unwrap(); + assert_eq!(resumed.loaded_layer_count(), 0); + resumed + .acquire_glyphs(std::slice::from_ref(&glyph_id), AcquireScope::Glyphs) + .unwrap(); + + assert_eq!(resumed.loaded_layer_count(), expected.len()); + for (layer_id, layer) in expected { + assert_eq!(resumed.font().layer(layer_id).unwrap(), &layer); + } +} + +#[test] +fn failed_streaming_import_removes_staging_and_preserves_the_destination() { + let temp = tempfile::tempdir().unwrap(); + let source_path = temp.path().join("Broken.ufo"); + let store_path = temp.path().join("existing.sqlite"); + let mut font = Font::new(); + let mut glyph = Glyph::new("A"); + let mut layer = GlyphLayer::new(LayerId::new(), font.default_source_id().unwrap()); + let mut contour = shift_font::Contour::new(); + contour.add_point(0.0, 0.0, PointType::OnCurve, false); + layer.add_contour(contour); + glyph.set_layer(layer); + font.insert_glyph(glyph).unwrap(); + shift_backends::font_loader::FontLoader::new() + .write_font(&font, source_path.to_str().unwrap()) + .unwrap(); + let glif_path = fs::read_dir(source_path.join("glyphs")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| { + path.extension() + .is_some_and(|extension| extension == "glif") + }) + .unwrap(); + fs::remove_file(glif_path).unwrap(); + drop(shift_store::ShiftStore::open(&store_path).unwrap()); + let destination_before = fs::read(&store_path).unwrap(); + + assert!(FontWorkspace::open(&source_path, &store_path).is_err()); + + assert_eq!(fs::read(&store_path).unwrap(), destination_before); + assert!( + shift_store::ShiftStore::open(&store_path) + .unwrap() + .workspace_state() + .unwrap() + .is_none() + ); + assert!(fs::read_dir(temp.path()).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".shift-import-") + })); +} + +#[test] +fn foreign_import_refuses_to_replace_a_published_workspace_store() { + let temp = tempfile::tempdir().unwrap(); + let source_path = fixture("fixtures/fonts/mutatorsans/MutatorSansLightCondensed.ufo"); + let store_path = temp.path().join("published.sqlite"); + let workspace = FontWorkspace::create_untitled(&store_path, NewWorkspace::new()).unwrap(); + drop(workspace); + + let error = match FontWorkspace::open(&source_path, &store_path) { + Ok(_) => panic!("foreign import must not replace a published workspace"), + Err(error) => error, + }; + + assert!(matches!( + error, + WorkspaceError::Store(shift_store::StoreError::ImportDestinationNotEmpty(path)) + if path == store_path + )); + let resumed = FontWorkspace::resume(&store_path).unwrap(); + assert_eq!(resumed.font().glyph_count(), 0); +} + +#[test] +fn failed_acquisition_keeps_placeholders_and_allows_an_unrelated_retry() { + let temp = tempfile::tempdir().unwrap(); + let source_path = fixture("fixtures/fonts/mutatorsans/MutatorSansLightCondensed.ufo"); + let store_path = temp.path().join("corrupt-acquisition.sqlite"); + let mut workspace = FontWorkspace::open(&source_path, &store_path).unwrap(); + let requested = workspace + .font() + .glyphs() + .filter(|glyph| !glyph.layers().is_empty()) + .take(2) + .map(|glyph| glyph.id()) + .collect::>(); + assert_eq!(requested.len(), 2); + let corrupt_layer_id = workspace + .font() + .glyph(requested[0].clone()) + .unwrap() + .layers() + .keys() + .next() + .unwrap() + .clone(); + let conn = rusqlite::Connection::open(&store_path).unwrap(); + conn.execute( + "UPDATE glyph_layer_payloads + SET payload = X'00', stored_byte_length = 1 + WHERE layer_id = ?1", + [corrupt_layer_id.to_string()], + ) + .unwrap(); + drop(conn); + + let error = workspace + .acquire_glyphs(std::slice::from_ref(&requested[0]), AcquireScope::Glyphs) + .unwrap_err(); + + assert!(matches!( + error, + WorkspaceError::Store(shift_store::StoreError::LayerDecompression(_)) + )); + assert_eq!(workspace.loaded_layer_count(), 0); + assert!(workspace.font().layer(corrupt_layer_id).unwrap().is_empty()); + + workspace + .acquire_glyphs(std::slice::from_ref(&requested[1]), AcquireScope::Glyphs) + .unwrap(); + assert!(workspace.loaded_layer_count() > 0); } #[test] @@ -237,6 +597,115 @@ fn resume_rebuilds_dirty_untitled_workspace_from_store() { assert!(workspace.font().glyph_id_by_name("A").is_some()); } +#[test] +fn resumed_layers_are_acquired_and_evictable_without_losing_authored_state() { + let temp = tempfile::tempdir().unwrap(); + let store_path = temp.path().join("working.sqlite"); + let mut workspace = FontWorkspace::create_untitled(&store_path, NewWorkspace::new()).unwrap(); + let source_id = workspace.font().default_source_id().unwrap(); + let glyph_id = create_glyph(&mut workspace, "A", vec![65]); + let layer_id = create_glyph_layer(&mut workspace, glyph_id.clone(), source_id); + add_square_contour(&mut workspace, &layer_id, (10.0, 20.0), 100.0); + drop(workspace); + + let mut workspace = FontWorkspace::resume(&store_path).unwrap(); + assert_eq!(workspace.loaded_layer_count(), 0); + assert!(workspace.font().layer(layer_id.clone()).unwrap().is_empty()); + + workspace + .acquire_glyphs(std::slice::from_ref(&glyph_id), AcquireScope::Glyphs) + .unwrap(); + assert_eq!(workspace.loaded_layer_count(), 1); + let authored = workspace.font().layer(layer_id.clone()).unwrap().clone(); + assert_eq!(authored.contours_iter().next().unwrap().points().len(), 4); + + workspace + .evict_glyphs(&[glyph_id.clone(), glyph_id.clone()]) + .unwrap(); + assert_eq!(workspace.loaded_layer_count(), 0); + assert!(workspace.font().layer(layer_id.clone()).unwrap().is_empty()); + + workspace + .acquire_glyphs(std::slice::from_ref(&glyph_id), AcquireScope::Glyphs) + .unwrap(); + assert_eq!(workspace.font().layer(layer_id).unwrap(), &authored); +} + +#[test] +fn intent_on_directory_layer_acquires_before_persisting() { + let temp = tempfile::tempdir().unwrap(); + let store_path = temp.path().join("working.sqlite"); + let mut workspace = FontWorkspace::create_untitled(&store_path, NewWorkspace::new()).unwrap(); + let source_id = workspace.font().default_source_id().unwrap(); + let glyph_id = create_glyph(&mut workspace, "A", vec![65]); + let layer_id = create_glyph_layer(&mut workspace, glyph_id.clone(), source_id); + add_square_contour(&mut workspace, &layer_id, (10.0, 20.0), 100.0); + drop(workspace); + + let mut workspace = FontWorkspace::resume(&store_path).unwrap(); + assert_eq!(workspace.loaded_layer_count(), 0); + workspace + .apply( + FontIntentSet { + intents: vec![FontIntent::SetXAdvance { + layer_id: layer_id.clone(), + width: 777.0, + }], + }, + None, + ) + .unwrap(); + + let layer = workspace.font().layer(layer_id.clone()).unwrap(); + assert_eq!(workspace.loaded_layer_count(), 1); + assert_eq!(layer.width(), 777.0); + assert_eq!(layer.contours_iter().next().unwrap().points().len(), 4); + drop(workspace); + + let mut resumed = FontWorkspace::resume(&store_path).unwrap(); + resumed + .acquire_glyphs(std::slice::from_ref(&glyph_id), AcquireScope::Glyphs) + .unwrap(); + let layer = resumed.font().layer(layer_id).unwrap(); + assert_eq!(layer.width(), 777.0); + assert_eq!(layer.contours_iter().next().unwrap().points().len(), 4); +} + +#[test] +fn undo_after_eviction_reacquires_and_tracks_restored_layer_as_loaded() { + let temp = tempfile::tempdir().unwrap(); + let store_path = temp.path().join("working.sqlite"); + let mut workspace = FontWorkspace::create_untitled(&store_path, NewWorkspace::new()).unwrap(); + let source_id = workspace.font().default_source_id().unwrap(); + let glyph_id = create_glyph(&mut workspace, "A", vec![65]); + let layer_id = create_glyph_layer(&mut workspace, glyph_id.clone(), source_id); + add_square_contour(&mut workspace, &layer_id, (10.0, 20.0), 100.0); + let original_width = workspace.font().layer(layer_id.clone()).unwrap().width(); + + workspace + .apply( + FontIntentSet { + intents: vec![FontIntent::SetXAdvance { + layer_id: layer_id.clone(), + width: 777.0, + }], + }, + Some("Change advance".to_string()), + ) + .unwrap(); + workspace + .evict_glyphs(std::slice::from_ref(&glyph_id)) + .unwrap(); + assert_eq!(workspace.loaded_layer_count(), 0); + + workspace.undo().unwrap().expect("advance edit should undo"); + + let layer = workspace.font().layer(layer_id).unwrap(); + assert_eq!(workspace.loaded_layer_count(), 1); + assert_eq!(layer.width(), original_width); + assert_eq!(layer.contours_iter().next().unwrap().points().len(), 4); +} + #[test] fn resume_package_workspace_can_save_unsaved_store_state() { let temp = tempfile::tempdir().unwrap(); @@ -898,19 +1367,39 @@ fn set_contour_closed_undo_redo_restores_layer() { } #[test] -fn move_points_undo_redo_restores_layer() { +fn move_points_undo_redo_restores_exact_coordinates_without_structure() { let (_temp, mut workspace, layer_id) = workspace_with_layer(); let (_, point_ids) = add_square_contour(&mut workspace, &layer_id, (0.0, 0.0), 100.0); + let pre = workspace.font().layer(layer_id.clone()).unwrap().clone(); - assert_layer_undo_redo( - &mut workspace, - &layer_id, - vec![FontIntent::MovePoints { - layer_id: layer_id.clone(), - point_ids: vec![point_ids[0].clone(), point_ids[1].clone()], - coords: vec![5.0, 6.0, 105.0, 6.0], - }], - ); + let applied = workspace + .apply( + FontIntentSet { + intents: vec![FontIntent::MovePoints { + layer_id: layer_id.clone(), + point_ids: vec![point_ids[0].clone(), point_ids[1].clone()], + coords: vec![5.0, 6.0, 105.0, 6.0], + }], + }, + None, + ) + .unwrap(); + let post = workspace.font().layer(layer_id.clone()).unwrap().clone(); + assert!(!applied.layers[0].structural); + + let undone = workspace + .undo() + .unwrap() + .expect("position edit should undo"); + assert!(!undone.layers[0].structural); + assert_eq!(workspace.font().layer(layer_id.clone()).unwrap(), &pre); + + let redone = workspace + .redo() + .unwrap() + .expect("position edit should redo"); + assert!(!redone.layers[0].structural); + assert_eq!(workspace.font().layer(layer_id).unwrap(), &post); } #[test] diff --git a/docs/architecture/index.md b/docs/architecture/index.md index bd7d5442..6c16daed 100644 --- a/docs/architecture/index.md +++ b/docs/architecture/index.md @@ -17,9 +17,9 @@ Central routing table for Shift's distributed documentation. Before creating new | --------------------------- | ---------------------------------------------------------------------------------- | ------------------------------------------------------------- | | `crates/shift-backends/**` | [`crates/shift-backends/docs/DOCS.md`](../../crates/shift-backends/docs/DOCS.md) | Font format backends for reading/writing various font formats | | `crates/shift-font/**` | [`crates/shift-font/docs/DOCS.md`](../../crates/shift-font/docs/DOCS.md) | First-class Rust font object model and editing behavior | -| `crates/shift-glyph-codec/**` | [`crates/shift-glyph-codec/SPECIFICATION.md`](../../crates/shift-glyph-codec/SPECIFICATION.md) | Packed glyph framing, validation, and compatibility | | `crates/shift-slug/**` | [`crates/shift-slug/docs/DOCS.md`](../../crates/shift-slug/docs/DOCS.md) | GPU-independent Slug curves, bands, and atlas packing | | `crates/shift-source/**` | [`crates/shift-source/docs/DOCS.md`](../../crates/shift-source/docs/DOCS.md) | User-authored `.shift` source package layout | +| `crates/shift-store/**` | [`crates/shift-store/README.md`](../../crates/shift-store/README.md) | SQLite directory, independently compressed canonical layer payloads, and reference indexes | | `crates/shift-workspace/**` | [`crates/shift-workspace/docs/DOCS.md`](../../crates/shift-workspace/docs/DOCS.md) | Open font workspace runtime over source, store, and font | | `crates/shift-bridge/**` | [`crates/shift-bridge/docs/DOCS.md`](../../crates/shift-bridge/docs/DOCS.md) | NAPI bridge exposing Rust to Node.js/Electron | @@ -30,6 +30,7 @@ Central routing table for Shift's distributed documentation. Before creating new | `apps/desktop/src/main/**` | [`apps/desktop/src/main/docs/DOCS.md`](../../apps/desktop/src/main/docs/DOCS.md) | Electron main process: lifecycle, windows, menus, document state | | `apps/desktop/src/preload/**` | [`apps/desktop/src/preload/docs/DOCS.md`](../../apps/desktop/src/preload/docs/DOCS.md) | Preload script bridging native Rust FontEngine to renderer | | `apps/desktop/src/shared/bridge/**` | [`apps/desktop/src/shared/bridge/docs/DOCS.md`](../../apps/desktop/src/shared/bridge/docs/DOCS.md) | Type-safe preload bridge system (FontEngineAPI contract) | +| `apps/desktop/e2e/**` | [`apps/desktop/e2e/README.md`](../../apps/desktop/e2e/README.md) | Playwright visual, GPU correctness, and performance suites | ### Desktop app — Renderer @@ -50,7 +51,6 @@ Central routing table for Shift's distributed documentation. Before creating new | `packages/types/**` | [`packages/types/docs/DOCS.md`](../../packages/types/docs/DOCS.md) | Branded IDs, bridge DTO facade, and legacy editor migration types | | `packages/geo/**` | [`packages/geo/docs/DOCS.md`](../../packages/geo/docs/DOCS.md) | Geometry utilities (Vec2, Curve, Polygon, Mat) | | `packages/glyph-state/**` | [`packages/glyph-state/docs/DOCS.md`](../../packages/glyph-state/docs/DOCS.md) | Glyph-domain geometry (contour traversal, segment parsing, bounds) | -| `packages/glyph-codec/**` | [`packages/glyph-codec/README.md`](../../packages/glyph-codec/README.md) | DOM-free TypeScript packed glyph validation and views | | `packages/ui/**` | [`packages/ui/docs/DOCS.md`](../../packages/ui/docs/DOCS.md) | UI component library wrapping Base UI primitives | | `packages/validation/**` | [`packages/validation/docs/DOCS.md`](../../packages/validation/docs/DOCS.md) | Point sequence validation and persistence schemas | | `packages/rules/**` | [`packages/rules/docs/DOCS.md`](../../packages/rules/docs/DOCS.md) | Point editing rules engine for geometric constraints | diff --git a/fixtures/glyph-codec/outline-v1/closed-line.bin b/fixtures/glyph-codec/outline-v1/closed-line.bin deleted file mode 100644 index 7c516978..00000000 Binary files a/fixtures/glyph-codec/outline-v1/closed-line.bin and /dev/null differ diff --git a/fixtures/glyph-codec/outline-v1/empty.bin b/fixtures/glyph-codec/outline-v1/empty.bin deleted file mode 100644 index 054a3800..00000000 Binary files a/fixtures/glyph-codec/outline-v1/empty.bin and /dev/null differ diff --git a/fixtures/glyph-codec/outline-v1/multiple-contours.bin b/fixtures/glyph-codec/outline-v1/multiple-contours.bin deleted file mode 100644 index a81597ad..00000000 Binary files a/fixtures/glyph-codec/outline-v1/multiple-contours.bin and /dev/null differ diff --git a/fixtures/glyph-codec/outline-v1/open.bin b/fixtures/glyph-codec/outline-v1/open.bin deleted file mode 100644 index d4479bba..00000000 Binary files a/fixtures/glyph-codec/outline-v1/open.bin and /dev/null differ diff --git a/fixtures/glyph-codec/outline-v1/quadratic-cubic.bin b/fixtures/glyph-codec/outline-v1/quadratic-cubic.bin deleted file mode 100644 index 2d091b34..00000000 Binary files a/fixtures/glyph-codec/outline-v1/quadratic-cubic.bin and /dev/null differ diff --git a/fixtures/glyph-codec/outline-v1/vectors.json b/fixtures/glyph-codec/outline-v1/vectors.json deleted file mode 100644 index f13a5fd0..00000000 --- a/fixtures/glyph-codec/outline-v1/vectors.json +++ /dev/null @@ -1,45 +0,0 @@ -[ - { - "name": "empty", - "file": "empty.bin", - "commands": [] - }, - { - "name": "closed-line", - "file": "closed-line.bin", - "commands": [ - { "kind": "move", "coordinates": [0, 0] }, - { "kind": "line", "coordinates": [100, 200] }, - { "kind": "close", "coordinates": [] } - ] - }, - { - "name": "open", - "file": "open.bin", - "commands": [ - { "kind": "move", "coordinates": [-12.5, 0.25] }, - { "kind": "line", "coordinates": [123.75, -456.5] } - ] - }, - { - "name": "quadratic-cubic", - "file": "quadratic-cubic.bin", - "commands": [ - { "kind": "move", "coordinates": [-1.5, 2.25] }, - { "kind": "quad", "coordinates": [0.5, -0.75, 1000.125, -2000.5] }, - { "kind": "cubic", "coordinates": [-3, 4.5, 5.25, -6.75, 100000000000000000000, -100000000000000000000] }, - { "kind": "close", "coordinates": [] } - ] - }, - { - "name": "multiple-contours", - "file": "multiple-contours.bin", - "commands": [ - { "kind": "move", "coordinates": [0, 0] }, - { "kind": "line", "coordinates": [10, 0] }, - { "kind": "close", "coordinates": [] }, - { "kind": "move", "coordinates": [100.5, -100.25] }, - { "kind": "line", "coordinates": [200, 300] } - ] - } -] diff --git a/package.json b/package.json index e45b38b9..6f9da3a9 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,12 @@ "test:integration": "pnpm test:native && cargo test --workspace", "test:lint": "pnpm --filter @shift/desktop run lint:check", "test:perf": "pnpm generate:glyph-info && pnpm --filter @shift/desktop exec vitest bench --run", - "test:playwright": "pnpm --filter @shift/desktop test:e2e", + "test:e2e": "pnpm --filter @shift/desktop test:e2e", + "test:e2e:visual": "pnpm --filter @shift/desktop test:e2e:visual", + "test:e2e:visual:update": "pnpm --filter @shift/desktop test:e2e:visual:update", + "test:e2e:gpu": "pnpm --filter @shift/desktop test:e2e:gpu", + "test:e2e:perf": "pnpm --filter @shift/desktop test:e2e:perf", + "test:playwright": "pnpm test:e2e", "test:watch": "turbo run test:watch", "test:native": "turbo run test --filter=shift-bridge", "typecheck": "pnpm run generate:types && turbo run typecheck", diff --git a/packages/glyph-codec/README.md b/packages/glyph-codec/README.md deleted file mode 100644 index 71476a5d..00000000 --- a/packages/glyph-codec/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# @shift/glyph-codec - -DOM-free TypeScript implementation of Shift's packed glyph payload family. - -The normative byte layout and compatibility rules live in -[`crates/shift-glyph-codec/SPECIFICATION.md`](../../crates/shift-glyph-codec/SPECIFICATION.md). -This package owns strict validation and the opaque `PackedGlyphOutline` value; -transport, persistence, `Path2D`, SVG debugging, and Slug transformations belong -to their respective consumers. diff --git a/packages/glyph-codec/package.json b/packages/glyph-codec/package.json deleted file mode 100644 index 31c51b84..00000000 --- a/packages/glyph-codec/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "@shift/glyph-codec", - "version": "0.1.0", - "private": true, - "description": "Strict packed glyph payload codecs for Shift", - "type": "module", - "main": "./src/index.ts", - "types": "./src/index.ts", - "exports": { - ".": { - "types": "./src/index.ts", - "import": "./src/index.ts" - } - }, - "scripts": { - "typecheck": "tsgo --noEmit", - "test": "vitest run", - "test:watch": "vitest" - }, - "devDependencies": { - "@types/node": "^25.3.0", - "typescript": "^5.5.4", - "vitest": "^4.1.10" - } -} diff --git a/packages/glyph-codec/src/index.ts b/packages/glyph-codec/src/index.ts deleted file mode 100644 index f180933b..00000000 --- a/packages/glyph-codec/src/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -export { - decodeOutline, - GlyphCodecError, - MAX_COMMAND_COUNT, - MAX_COORDINATE_COUNT, - MAX_PAYLOAD_BYTES, - PackedGlyphOutline, - packOutline, -} from "./outline"; -export type { GlyphCodecErrorCode, OutlineCommand } from "./types"; diff --git a/packages/glyph-codec/src/outline.test.ts b/packages/glyph-codec/src/outline.test.ts deleted file mode 100644 index 3e268772..00000000 --- a/packages/glyph-codec/src/outline.test.ts +++ /dev/null @@ -1,275 +0,0 @@ -import { readFileSync } from "node:fs"; -import { describe, expect, it } from "vitest"; -import { decodeOutline, GlyphCodecError, MAX_COMMAND_COUNT, packOutline } from "./outline"; -import type { GlyphCodecErrorCode, OutlineCommand } from "./types"; - -type GoldenVector = { - readonly name: string; - readonly file: string; - readonly commands: readonly OutlineCommand[]; -}; - -const vectors: readonly GoldenVector[] = [ - { name: "empty", file: "empty.bin", commands: [] }, - { - name: "closed-line", - file: "closed-line.bin", - commands: [{ kind: "move", x: 0, y: 0 }, { kind: "line", x: 100, y: 200 }, { kind: "close" }], - }, - { - name: "open", - file: "open.bin", - commands: [ - { kind: "move", x: -12.5, y: 0.25 }, - { kind: "line", x: 123.75, y: -456.5 }, - ], - }, - { - name: "quadratic-cubic", - file: "quadratic-cubic.bin", - commands: [ - { kind: "move", x: -1.5, y: 2.25 }, - { kind: "quad", cx: 0.5, cy: -0.75, x: 1000.125, y: -2000.5 }, - { - kind: "cubic", - c1x: -3, - c1y: 4.5, - c2x: 5.25, - c2y: -6.75, - x: 1e20, - y: -1e20, - }, - { kind: "close" }, - ], - }, - { - name: "multiple-contours", - file: "multiple-contours.bin", - commands: [ - { kind: "move", x: 0, y: 0 }, - { kind: "line", x: 10, y: 0 }, - { kind: "close" }, - { kind: "move", x: 100.5, y: -100.25 }, - { kind: "line", x: 200, y: 300 }, - ], - }, -]; - -function fixture(file: string): Uint8Array { - return readFileSync(new URL(`../../../fixtures/glyph-codec/outline-v1/${file}`, import.meta.url)); -} - -function bytesEqual(left: Uint8Array, right: Uint8Array): boolean { - return left.length === right.length && left.every((byte, index) => byte === right[index]); -} - -function f32Commands(commands: readonly OutlineCommand[]): OutlineCommand[] { - return commands.map((command) => { - switch (command.kind) { - case "move": - case "line": - return { kind: command.kind, x: Math.fround(command.x), y: Math.fround(command.y) }; - case "quad": - return { - kind: "quad", - cx: Math.fround(command.cx), - cy: Math.fround(command.cy), - x: Math.fround(command.x), - y: Math.fround(command.y), - }; - case "cubic": - return { - kind: "cubic", - c1x: Math.fround(command.c1x), - c1y: Math.fround(command.c1y), - c2x: Math.fround(command.c2x), - c2y: Math.fround(command.c2y), - x: Math.fround(command.x), - y: Math.fround(command.y), - }; - case "close": - return command; - } - }); -} - -function rawOutline(commands: readonly number[], coordinates: readonly number[]): Uint8Array { - const alignedCommands = Math.ceil(commands.length / 4) * 4; - const bytes = new Uint8Array(16 + alignedCommands + coordinates.length * 4); - bytes.set([0x53, 0x48, 0x46, 0x54, 1, 1], 0); - const view = new DataView(bytes.buffer); - view.setUint32(8, commands.length, true); - view.setUint32(12, coordinates.length, true); - bytes.set(commands, 16); - coordinates.forEach((coordinate, index) => - view.setFloat32(16 + alignedCommands + index * 4, coordinate, true), - ); - return bytes; -} - -function codecErrorCode(action: () => unknown): GlyphCodecErrorCode | null { - try { - action(); - return null; - } catch (error) { - if (!(error instanceof GlyphCodecError)) throw error; - return error.code; - } -} - -describe("shift.glyph-outline.v1 shared compatibility", () => { - it.each(vectors)("decodes and canonically re-encodes $name", ({ file, commands }) => { - const bytes = fixture(file); - const outline = decodeOutline(bytes); - - expect([...outline]).toEqual(f32Commands(commands)); - expect([...packOutline(commands).toUint8Array()]).toEqual([...bytes]); - expect([...packOutline([...outline]).toUint8Array()]).toEqual([...bytes]); - }); - - it("covers every command-count alignment remainder", () => { - const remainders = vectors.map(({ file }) => decodeOutline(fixture(file)).commandCount % 4); - - expect(new Set(remainders)).toEqual(new Set([0, 1, 2, 3])); - }); - - it("does not expose mutable validated storage", () => { - const source = fixture("open.bin"); - const outline = decodeOutline(source); - const exported = outline.toUint8Array(); - source[0] = 0; - exported[1] = 0; - - expect([...outline]).toEqual(f32Commands(vectors[2].commands)); - }); -}); - -describe("strict outline decoding", () => { - it("rejects every truncation and trailing bytes", () => { - const bytes = fixture("quadratic-cubic.bin"); - for (let length = 0; length < bytes.length; length += 1) { - expect(codecErrorCode(() => decodeOutline(bytes.slice(0, length)))).not.toBeNull(); - } - - const trailing = new Uint8Array(bytes.length + 1); - trailing.set(bytes); - expect(codecErrorCode(() => decodeOutline(trailing))).toBe("length-mismatch"); - }); - - it("rejects wrong framing and unknown flags", () => { - const mutations: ReadonlyArray = [ - [0, 0, "wrong-magic"], - [4, 2, "unsupported-kind"], - [5, 2, "unsupported-version"], - [6, 1, "unknown-flags"], - ]; - - for (const [offset, value, code] of mutations) { - const bytes = fixture("open.bin"); - bytes[offset] = value; - expect(codecErrorCode(() => decodeOutline(bytes))).toBe(code); - } - }); - - it("rejects unknown commands and illegal command transitions", () => { - const cases: ReadonlyArray = [ - [rawOutline([9], []), "unknown-command"], - [rawOutline([1], [1, 2]), "invalid-command-order"], - [rawOutline([4], []), "invalid-command-order"], - [rawOutline([0, 4], [0, 0]), "invalid-command-order"], - [rawOutline([0, 1, 4, 1], [0, 0, 1, 1, 2, 2]), "invalid-command-order"], - ]; - - for (const [bytes, code] of cases) { - expect(codecErrorCode(() => decodeOutline(bytes))).toBe(code); - } - }); - - it("rejects arity mismatch and non-zero padding", () => { - expect(codecErrorCode(() => decodeOutline(rawOutline([0, 1], [0, 0])))).toBe( - "coordinate-count-mismatch", - ); - - const padding = fixture("open.bin"); - padding[18] = 1; - expect(codecErrorCode(() => decodeOutline(padding))).toBe("non-zero-padding"); - }); - - it.each([Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY])( - "rejects decoded non-finite coordinate %s", - (coordinate) => { - expect(codecErrorCode(() => decodeOutline(rawOutline([0], [coordinate, 0])))).toBe( - "non-finite-coordinate", - ); - }, - ); - - it("rejects illegally ordered encoder input", () => { - const cases: ReadonlyArray = [ - [{ kind: "line", x: 1, y: 2 }], - [{ kind: "close" }], - [{ kind: "move", x: 0, y: 0 }, { kind: "close" }], - ]; - - for (const commands of cases) { - expect(codecErrorCode(() => packOutline(commands))).toBe("invalid-command-order"); - } - }); - - it.each([Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY])( - "rejects non-finite input coordinate %s", - (x) => { - expect(codecErrorCode(() => packOutline([{ kind: "move", x, y: 0 }]))).toBe( - "non-finite-input-coordinate", - ); - }, - ); - - it("rejects finite values that overflow f32", () => { - expect(codecErrorCode(() => packOutline([{ kind: "move", x: Number.MAX_VALUE, y: 0 }]))).toBe( - "coordinate-out-of-f32-range", - ); - }); - - it("applies count limits before reading a declared body", () => { - const bytes = fixture("empty.bin"); - new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).setUint32( - 8, - MAX_COMMAND_COUNT + 1, - true, - ); - - expect(codecErrorCode(() => decodeOutline(bytes))).toBe("limit-exceeded"); - }); - - it("never accepts a mutated payload with a non-canonical encoding", () => { - const commands = vectors.flatMap((vector) => vector.commands); - const seed = packOutline(commands).toUint8Array(); - let random = 0x7a114e29; - - for (let iteration = 0; iteration < 10_000; iteration += 1) { - random ^= random << 13; - random ^= random >>> 17; - random ^= random << 5; - const candidate = seed.slice(); - const offset = (random >>> 0) % candidate.length; - candidate[offset] ^= ((random >>> 24) | 1) & 0xff; - const mutated = - iteration % 7 === 0 - ? candidate.slice(0, (random >>> 0) % candidate.length) - : iteration % 11 === 0 - ? Uint8Array.from([...candidate, (random >>> 16) & 0xff]) - : candidate; - - try { - const outline = decodeOutline(mutated); - const canonical = packOutline([...outline]).toUint8Array(); - if (!bytesEqual(canonical, mutated)) { - throw new Error("decoder accepted a non-canonical outline encoding"); - } - } catch (error) { - if (!(error instanceof GlyphCodecError)) throw error; - } - } - }); -}); diff --git a/packages/glyph-codec/src/outline.ts b/packages/glyph-codec/src/outline.ts deleted file mode 100644 index d3e90dc9..00000000 --- a/packages/glyph-codec/src/outline.ts +++ /dev/null @@ -1,400 +0,0 @@ -import type { GlyphCodecErrorCode, OutlineCommand } from "./types"; - -const MAGIC = [0x53, 0x48, 0x46, 0x54] as const; -const OUTLINE_KIND = 0x01; -const OUTLINE_VERSION = 0x01; -const HEADER_LENGTH = 16; - -export const MAX_COMMAND_COUNT = 1_000_000; -export const MAX_COORDINATE_COUNT = 6_000_000; -export const MAX_PAYLOAD_BYTES = 32 * 1024 * 1024; - -export class GlyphCodecError extends Error { - readonly code: GlyphCodecErrorCode; - - constructor(code: GlyphCodecErrorCode, message: string) { - super(message); - this.name = "GlyphCodecError"; - this.code = code; - } -} - -type Header = { - readonly commandCount: number; - readonly coordinateCount: number; - readonly coordinateOffset: number; -}; - -type CommandState = "between" | "active-empty" | "active-drawing"; - -/** - * Opaque, immutable, validated `shift.glyph-outline.v1` bytes. - * - * The constructor is intentionally private. Decode copies caller-owned bytes so - * later mutation cannot invalidate the checked command/coordinate view; packed - * bytes are allocated and retained directly. `toUint8Array()` likewise returns - * a copy rather than exposing mutable internal storage. - */ -export class PackedGlyphOutline implements Iterable { - readonly #bytes: Uint8Array; - readonly #header: Header; - - private constructor(bytes: Uint8Array, header: Header) { - this.#bytes = bytes; - this.#header = header; - } - - static decode(data: Uint8Array): PackedGlyphOutline { - checkLimit("payload byte length", data.byteLength, MAX_PAYLOAD_BYTES); - const bytes = Uint8Array.from(data); - const header = validate(bytes); - return new PackedGlyphOutline(bytes, header); - } - - static pack(commands: readonly OutlineCommand[]): PackedGlyphOutline { - checkLimit("command count", commands.length, MAX_COMMAND_COUNT); - - let state: CommandState = "between"; - let coordinateCount = 0; - for (const [index, command] of commands.entries()) { - state = nextState(state, commandByte(command), index); - coordinateCount = checkedAdd(coordinateCount, commandArity(command)); - checkLimit("coordinate count", coordinateCount, MAX_COORDINATE_COUNT); - } - - const alignedCommands = align4(commands.length); - const byteLength = checkedAdd( - checkedAdd(HEADER_LENGTH, alignedCommands), - checkedMultiply(coordinateCount, 4), - ); - checkLimit("payload byte length", byteLength, MAX_PAYLOAD_BYTES); - - const bytes = new Uint8Array(byteLength); - bytes.set(MAGIC, 0); - bytes[4] = OUTLINE_KIND; - bytes[5] = OUTLINE_VERSION; - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); - view.setUint16(6, 0, true); - view.setUint32(8, commands.length, true); - view.setUint32(12, coordinateCount, true); - - let commandOffset = HEADER_LENGTH; - let coordinateIndex = 0; - const coordinateOffset = HEADER_LENGTH + alignedCommands; - for (const command of commands) { - bytes[commandOffset] = commandByte(command); - commandOffset += 1; - for (const coordinate of commandCoordinates(command)) { - view.setFloat32( - coordinateOffset + coordinateIndex * 4, - convertCoordinate(coordinate, coordinateIndex), - true, - ); - coordinateIndex += 1; - } - } - - return new PackedGlyphOutline(bytes, { - commandCount: commands.length, - coordinateCount, - coordinateOffset, - }); - } - - get byteLength(): number { - return this.#bytes.byteLength; - } - - get commandCount(): number { - return this.#header.commandCount; - } - - get coordinateCount(): number { - return this.#header.coordinateCount; - } - - /** Returns a mutable copy suitable for transport. */ - toUint8Array(): Uint8Array { - return this.#bytes.slice(); - } - - /** Iterates validated commands without allocating a point-object graph. */ - *commands(): IterableIterator { - const view = new DataView(this.#bytes.buffer, this.#bytes.byteOffset, this.#bytes.byteLength); - let coordinateOffset = this.#header.coordinateOffset; - const next = (): number => { - const value = view.getFloat32(coordinateOffset, true); - coordinateOffset += 4; - return value; - }; - - for (let index = 0; index < this.#header.commandCount; index += 1) { - switch (this.#bytes[HEADER_LENGTH + index]) { - case 0: - yield { kind: "move", x: next(), y: next() }; - break; - case 1: - yield { kind: "line", x: next(), y: next() }; - break; - case 2: - yield { kind: "quad", cx: next(), cy: next(), x: next(), y: next() }; - break; - case 3: - yield { - kind: "cubic", - c1x: next(), - c1y: next(), - c2x: next(), - c2y: next(), - x: next(), - y: next(), - }; - break; - case 4: - yield { kind: "close" }; - break; - default: - throw new Error("PackedGlyphOutline command invariant violated"); - } - } - } - - /** Iterates all finite f32 values in command-stream order. */ - *coordinates(): IterableIterator { - const view = new DataView(this.#bytes.buffer, this.#bytes.byteOffset, this.#bytes.byteLength); - for (let index = 0; index < this.#header.coordinateCount; index += 1) { - yield view.getFloat32(this.#header.coordinateOffset + index * 4, true); - } - } - - [Symbol.iterator](): IterableIterator { - return this.commands(); - } -} - -/** Canonically encodes drawing commands into an opaque outline payload. */ -export function packOutline(commands: readonly OutlineCommand[]): PackedGlyphOutline { - return PackedGlyphOutline.pack(commands); -} - -/** Strictly validates and copies an outline payload. */ -export function decodeOutline(data: Uint8Array): PackedGlyphOutline { - return PackedGlyphOutline.decode(data); -} - -function validate(bytes: Uint8Array): Header { - if (bytes.byteLength < HEADER_LENGTH) { - fail( - "header-truncated", - `outline header is truncated: ${bytes.byteLength} of ${HEADER_LENGTH} bytes`, - ); - } - checkLimit("payload byte length", bytes.byteLength, MAX_PAYLOAD_BYTES); - - if (MAGIC.some((byte, index) => bytes[index] !== byte)) { - fail("wrong-magic", "wrong glyph-codec magic"); - } - if (bytes[4] !== OUTLINE_KIND) { - fail("unsupported-kind", `unsupported glyph-codec payload kind ${hex(bytes[4])}`); - } - if (bytes[5] !== OUTLINE_VERSION) { - fail("unsupported-version", `unsupported glyph-outline version ${bytes[5]}`); - } - - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); - const flags = view.getUint16(6, true); - if (flags !== 0) { - fail("unknown-flags", `unknown glyph-outline flags ${hex(flags, 4)}`); - } - - const commandCount = view.getUint32(8, true); - const coordinateCount = view.getUint32(12, true); - checkLimit("command count", commandCount, MAX_COMMAND_COUNT); - checkLimit("coordinate count", coordinateCount, MAX_COORDINATE_COUNT); - - const alignedCommands = align4(commandCount); - const expectedLength = checkedAdd( - checkedAdd(HEADER_LENGTH, alignedCommands), - checkedMultiply(coordinateCount, 4), - ); - checkLimit("payload byte length", expectedLength, MAX_PAYLOAD_BYTES); - if (bytes.byteLength !== expectedLength) { - fail( - "length-mismatch", - `outline payload length mismatch: expected ${expectedLength} bytes, got ${bytes.byteLength}`, - ); - } - - let state: CommandState = "between"; - let expectedCoordinateCount = 0; - for (let index = 0; index < commandCount; index += 1) { - const command = bytes[HEADER_LENGTH + index]; - const arity = byteArity(command, index); - state = nextState(state, command, index); - expectedCoordinateCount = checkedAdd(expectedCoordinateCount, arity); - } - if (expectedCoordinateCount !== coordinateCount) { - fail( - "coordinate-count-mismatch", - `outline command arity requires ${expectedCoordinateCount} coordinates, header declares ${coordinateCount}`, - ); - } - - const commandsEnd = HEADER_LENGTH + commandCount; - const coordinateOffset = HEADER_LENGTH + alignedCommands; - for (let offset = commandsEnd; offset < coordinateOffset; offset += 1) { - if (bytes[offset] !== 0) { - fail("non-zero-padding", `non-zero outline alignment padding at byte ${offset}`); - } - } - - for (let index = 0; index < coordinateCount; index += 1) { - const coordinate = view.getFloat32(coordinateOffset + index * 4, true); - if (!Number.isFinite(coordinate)) { - fail("non-finite-coordinate", `non-finite outline coordinate at index ${index}`); - } - } - - return { commandCount, coordinateCount, coordinateOffset }; -} - -function nextState(state: CommandState, command: number, index: number): CommandState { - switch (command) { - case 0: - return "active-empty"; - case 1: - case 2: - case 3: - if (state === "between") { - fail( - "invalid-command-order", - `invalid outline command ordering at index ${index}: drawing command requires an active contour`, - ); - } - return "active-drawing"; - case 4: - if (state === "between") { - fail( - "invalid-command-order", - `invalid outline command ordering at index ${index}: close requires an active contour`, - ); - } - if (state === "active-empty") { - fail( - "invalid-command-order", - `invalid outline command ordering at index ${index}: close requires at least one drawing segment`, - ); - } - return "between"; - default: - fail("unknown-command", `unknown outline command ${hex(command)} at index ${index}`); - } -} - -function commandByte(command: OutlineCommand): number { - switch (command.kind) { - case "move": - return 0; - case "line": - return 1; - case "quad": - return 2; - case "cubic": - return 3; - case "close": - return 4; - } -} - -function commandArity(command: OutlineCommand): number { - switch (command.kind) { - case "move": - case "line": - return 2; - case "quad": - return 4; - case "cubic": - return 6; - case "close": - return 0; - } -} - -function byteArity(command: number, index: number): number { - switch (command) { - case 0: - case 1: - return 2; - case 2: - return 4; - case 3: - return 6; - case 4: - return 0; - default: - fail("unknown-command", `unknown outline command ${hex(command)} at index ${index}`); - } -} - -function commandCoordinates(command: OutlineCommand): readonly number[] { - switch (command.kind) { - case "move": - case "line": - return [command.x, command.y]; - case "quad": - return [command.cx, command.cy, command.x, command.y]; - case "cubic": - return [command.c1x, command.c1y, command.c2x, command.c2y, command.x, command.y]; - case "close": - return []; - } -} - -function convertCoordinate(value: number, index: number): number { - if (!Number.isFinite(value)) { - fail("non-finite-input-coordinate", `non-finite input coordinate at index ${index}`); - } - - const converted = Math.fround(value); - if (!Number.isFinite(converted)) { - fail( - "coordinate-out-of-f32-range", - `input coordinate at index ${index} is outside finite f32 range`, - ); - } - return converted; -} - -function align4(value: number): number { - return checkedMultiply(Math.ceil(value / 4), 4); -} - -function checkedAdd(left: number, right: number): number { - const value = left + right; - if (!Number.isSafeInteger(value)) { - fail("length-overflow", "outline payload length arithmetic overflowed"); - } - return value; -} - -function checkedMultiply(left: number, right: number): number { - const value = left * right; - if (!Number.isSafeInteger(value)) { - fail("length-overflow", "outline payload length arithmetic overflowed"); - } - return value; -} - -function checkLimit(field: string, actual: number, limit: number): void { - if (actual > limit) { - fail("limit-exceeded", `outline ${field} exceeds implementation limit ${limit}: got ${actual}`); - } -} - -function hex(value: number, width = 2): string { - return `0x${value.toString(16).padStart(width, "0")}`; -} - -function fail(code: GlyphCodecErrorCode, message: string): never { - throw new GlyphCodecError(code, message); -} diff --git a/packages/glyph-codec/src/types.ts b/packages/glyph-codec/src/types.ts deleted file mode 100644 index 635a899c..00000000 --- a/packages/glyph-codec/src/types.ts +++ /dev/null @@ -1,37 +0,0 @@ -export type OutlineCommand = - | { readonly kind: "move"; readonly x: number; readonly y: number } - | { readonly kind: "line"; readonly x: number; readonly y: number } - | { - readonly kind: "quad"; - readonly cx: number; - readonly cy: number; - readonly x: number; - readonly y: number; - } - | { - readonly kind: "cubic"; - readonly c1x: number; - readonly c1y: number; - readonly c2x: number; - readonly c2y: number; - readonly x: number; - readonly y: number; - } - | { readonly kind: "close" }; - -export type GlyphCodecErrorCode = - | "header-truncated" - | "wrong-magic" - | "unsupported-kind" - | "unsupported-version" - | "unknown-flags" - | "unknown-command" - | "invalid-command-order" - | "coordinate-count-mismatch" - | "non-zero-padding" - | "non-finite-coordinate" - | "non-finite-input-coordinate" - | "coordinate-out-of-f32-range" - | "length-overflow" - | "length-mismatch" - | "limit-exceeded"; diff --git a/packages/glyph-codec/tsconfig.json b/packages/glyph-codec/tsconfig.json deleted file mode 100644 index f53b6d51..00000000 --- a/packages/glyph-codec/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../tsconfig/library.json", - "compilerOptions": { - "rootDir": "./src", - "outDir": "./dist", - "types": ["node"] - }, - "include": ["src"] -} diff --git a/packages/glyph-codec/vitest.config.ts b/packages/glyph-codec/vitest.config.ts deleted file mode 100644 index 20a45a26..00000000 --- a/packages/glyph-codec/vitest.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { defineConfig } from "vitest/config"; - -export default defineConfig({ - test: { - include: ["src/**/*.test.ts"], - passWithNoTests: false, - }, -}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 23ecf6ed..e8568a86 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -211,18 +211,6 @@ importers: specifier: ^4.1.10 version: 4.1.10(@types/node@25.3.0)(jsdom@26.1.0)(vite@6.4.3(@types/node@25.3.0)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.49.0)(tsx@4.21.0)) - packages/glyph-codec: - devDependencies: - '@types/node': - specifier: ^25.3.0 - version: 25.3.0 - typescript: - specifier: ^5.5.4 - version: 5.9.3 - vitest: - specifier: ^4.1.10 - version: 4.1.10(@types/node@25.3.0)(jsdom@26.1.0)(vite@6.4.3(@types/node@25.3.0)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.49.0)(tsx@4.21.0)) - packages/glyph-info: dependencies: minisearch: