diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e9aa65..e0e518d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: - name: Build, run and compare run: sh tools/check.sh $(ls apps | grep -E -- '-(python|rust)$') - # The documentation tells readers that newer toolchains usually work. This + # The documentation says that newer toolchains usually work. This # tests that claim nightly, against whatever those projects have released, # with no cached interpreter so CPython is built from source as a reader # would. Only wasi-sdk stays fixed, since a new one moves the sysroot the diff --git a/AGENTS.md b/AGENTS.md index d36451f..db024b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,22 +14,32 @@ make build APP= # one module into build/.wasm make run APP= # build, then both passes against fixtures/ make run APP= FIXTURES=fixtures/no-call make run APP= FIXTURES=fixtures/unanswered +OPTIONAL=off make run APP= # as if the owner declined every optional grant +INPUTS= make run APP= # answers from , not apps//inputs sh tools/check.sh # what CI runs: links, builds, runs, ports ``` `make run` mounts only the paths an app's manifest declares, read-only, and -runs it with `/` as the data directory, the way an Ark does. The three fixture -roots are described in `fixtures/README.md`. An app whose grants a root lacks -stops before its run pass there, which is expected. +runs it with `/` as the data directory, the way an Ark does. An optional path +is mounted when the fixture root holds it, and an app's answers at `/inputs` +when it declares inputs. The three fixture roots are described in +`fixtures/README.md`. An app whose required grants a root lacks stops before +its report pass there, which is expected. ## Adding an app - Copy the nearest example. One source file, in `apps/--/` for a mini and `apps/-/` for a full app, with a short `README.md` that says what it shows and how to run it. -- The manifest names the app, its version and the narrowest grants that answer - the question. Spell every path exactly as `ark data paths` shows it, with - `v1/` kept and no trailing `/`. `docs/02-manifest.md` has the rules. +- The manifest gives the app's name as the owner should see it, its version and + the narrowest grants that answer the question. Spell every path exactly as + `ark data paths` shows it, with `v1/` kept and no trailing `/`. A full + `[listing]` ends it, with a purpose for every grant, and every language port + of an app prints the same one apart from `source`. `docs/02-manifest.md` has + the rules. +- An app that asks the owner something declares `[inputs.]` tables and + keeps sample answers in its `inputs/` folder, one file per input with no + trailing newline, which `make run` mounts. - Read grants as plain files. Absent means "not found" and is a result. Any other error goes to standard error with a non-zero exit. The shapes a genotype can take are in `docs/04-reading-data.md`. @@ -62,7 +72,7 @@ Follow `docs/06-reports.md`. In short: - Pick one voice and keep it. Light or serious, never both in one report. Genomics words, never clinic words. - Plain Markdown only. No images, HTML, footnotes, encoded content, dates or - run ids. Tables with as few columns as carry the evidence, rows about 80 + task ids. Tables with as few columns as carry the evidence, rows about 80 characters. ## Never diff --git a/Makefile b/Makefile index d42220b..c968dcc 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,8 @@ # make build APP=01-hello-rust build one app # make run build and run every app # make run APP=01-hello-rust build and run one app +# make run APP=02-permissions OPTIONAL=off decline optional datasets +# make run APP=03-sundae-order INPUTS=answers override apps//inputs # make clean remove build outputs # # Every app builds to a single module at build/.wasm, whatever its @@ -13,12 +15,13 @@ # the data an Ark would mount. See docs/05-running.md. FIXTURES ?= fixtures +INPUTS ?= apps/$*/inputs BUILD ?= build APPS := $(sort $(notdir $(wildcard apps/*))) SELECTED := $(or $(strip $(APP)),$(APPS)) # The scripts read these, and any toolchain override, from the environment. -export BUILD WASMTIME WASI_SDK PYTHON TINYGO WASM_OPT +export BUILD WASMTIME WASI_SDK PYTHON TINYGO WASM_OPT OPTIONAL INPUTS .PHONY: all build run clean list FORCE @@ -39,7 +42,7 @@ build-%: FORCE run-%: FORCE @if [ -z "$(strip $(APP))" ]; then echo; echo "===== $* ====="; fi @sh tools/build.sh "$*" - @sh tools/run.sh "$(BUILD)/$*.wasm" "$(FIXTURES)" + @sh tools/run.sh "$(BUILD)/$*.wasm" "$(FIXTURES)" "$(INPUTS)" clean: @rm -rf $(BUILD) apps/*/target apps/*/*.wasm diff --git a/README.md b/README.md index 1ad47c1..e542523 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,11 @@ Worked examples for writing apps that run on an Ark. An Ark holds a person's genome on a device only they control. Apps never receive a copy of that data. They are sent to it instead, and run in a deterministic sandbox on the Ark with no network and no writable storage. An app reads the -data it was granted as plain files and prints a report. The owner approves every -run on their phone and decides whether to release the result. Because the -sandbox contains the code, anyone can write an app and anyone can run anyone -else's. The [whitepaper](https://dark.bio/whitepaper.pdf) lays out the trust -model. +data it was granted as plain files and prints a report. The owner approves each +app on their phone before it runs, and decides whether to release its report. +Because the sandbox contains the code, anyone can write an app and anyone can +run anyone else's. The [whitepaper](https://dark.bio/whitepaper.pdf) lays out +the trust model. These examples teach that model one idea at a time, from printing a line to scanning a whole call file. Every app is a single source file. @@ -29,7 +29,7 @@ An app is a [WebAssembly](https://webassembly.org/) module built for ```rust fn main() { if std::env::args().nth(1).is_none() { - print!("[package]\nname = \"hello\"\nversion = \"0.1.0\"\ndatasets = []\n"); + print!("manifest = 1\n\n[app]\nname = \"hello\"\nversion = \"0.1.0\"\n"); return; } println!("Hello from an Ark app."); @@ -47,7 +47,7 @@ make run # every app ``` Run it on an Ark with the [`ark`](https://github.com/dark-bio/cli) tool. The -owner approves the run on their phone, in Ark Companion for +owner approves it on their phone, in Ark Companion for [iOS](https://apps.apple.com/app/id6751324700) or [Android](https://play.google.com/store/apps/details?id=bio.dark.companion). Without hardware, [Ark Emulator](https://github.com/dark-bio/emulator) boots @@ -67,30 +67,31 @@ what a laptop can't reproduce, and running on an Ark or an emulator. Most data comes in two versions. A *mini* shows the bare read in a few lines, and a full app turns the same read into a report. Each row is a folder under `apps/`, with the language appended for the minis, such as -`03-cilantro-mini-rust`, and that folder name is what `APP=` takes. +`04-cilantro-mini-rust`, and that folder name is what `APP=` takes. | App | What it shows | Languages | | :-- | :-- | :-- | -| `01-hello` | the manifest pass and the run pass | Rust, Go, C, Python | -| `02-permissions` | requesting data, least privilege and the `develop` flag | Rust | -| `03-cilantro-mini` | one variant through `rsids/` | Rust, Go, C, Python | -| `03-cilantro-soapiness` | the same variant as a report | Rust | -| `04-drunk-o-type` | a panel of variants | Rust | -| `05-genes-mini` | one gene through `genes/` | Rust, Go, C, Python | -| `05-bitter-meter` | the gene and its changes as a report | Rust | -| `06-regions-mini` | an interval through `regions/` | Rust, Go, C, Python | -| `06-powerhouse-of-the-cell` | the mitochondrial genome as a report | Rust | -| `07-vcf-mini` | streaming the raw call file | Rust, Go, C, Python | -| `07-vcf-roll-call` | parsing the whole call file as a report | Rust | -| `08-motif-finder` | computing over a gene's reference sequence | Rust | -| `09-fortune-cookie` | why the sandbox is deterministic | Rust | +| `01-hello` | the manifest pass and the report pass | Rust, Go, C, Python | +| `02-permissions` | requesting data, optional grants, least privilege and the `develop` flag | Rust | +| `03-sundae-order` | asking the owner questions with inputs | Rust | +| `04-cilantro-mini` | one variant through `rsids/` | Rust, Go, C, Python | +| `04-cilantro-soapiness` | the same variant as a report | Rust | +| `05-drunk-o-type` | a panel of variants | Rust | +| `06-genes-mini` | one gene through `genes/` | Rust, Go, C, Python | +| `06-bitter-meter` | the gene and its changes as a report | Rust | +| `07-regions-mini` | an interval through `regions/` | Rust, Go, C, Python | +| `07-powerhouse-of-the-cell` | the mitochondrial genome as a report | Rust | +| `08-vcf-mini` | streaming the raw call file | Rust, Go, C, Python | +| `08-vcf-roll-call` | parsing the whole call file as a report | Rust | +| `09-motif-finder` | computing over a gene's reference sequence | Rust | +| `10-fortune-cookie` | why the sandbox is deterministic | Rust | ## Documentation - [01-app-model.md](docs/01-app-model.md) covers the two passes, the checks, the sandbox and its limits. -- [02-manifest.md](docs/02-manifest.md) covers the manifest and how to grant - data. +- [02-manifest.md](docs/02-manifest.md) covers the manifest, how to grant data + and how to ask the owner questions. - [03-data-paths.md](docs/03-data-paths.md) maps the data tree and the rules every path follows. - [04-reading-data.md](docs/04-reading-data.md) covers absence, errors, @@ -103,7 +104,7 @@ and a full app turns the same read into a report. Each row is a folder under ## A note on scope These apps show how to read data and present a result. Their trait readouts are -illustrations, not medical advice, and one variant rarely tells a whole story. +illustrations, not medical advice, and one variant is rarely the whole story. ## License diff --git a/apps/01-hello-c/README.md b/apps/01-hello-c/README.md index 36d73c5..defa661 100644 --- a/apps/01-hello-c/README.md +++ b/apps/01-hello-c/README.md @@ -23,5 +23,5 @@ To see each pass by hand, build the module and run it twice: ```sh make build APP=01-hello-c wasmtime build/01-hello-c.wasm # manifest pass -wasmtime build/01-hello-c.wasm / # run pass +wasmtime build/01-hello-c.wasm / # report pass ``` diff --git a/apps/01-hello-c/main.c b/apps/01-hello-c/main.c index 079f986..b66fbfb 100644 --- a/apps/01-hello-c/main.c +++ b/apps/01-hello-c/main.c @@ -7,18 +7,34 @@ int main(int argc, char *argv[]) { (void)argv; - // The manifest pass: no data directory was given, so describe the app and - // stop. This app reads nothing, so its dataset list is empty. + // The manifest pass has no data directory, so describe the app and stop. + // This app reads nothing, so it has no reads table. if (argc < 2) { printf( - "[package]\n" - "name = \"hello-c\"\n" - "version = \"0.1.0\"\n" - "datasets = []\n"); + "manifest = 1\n\n" + "[app]\n" + "name = \"Hello, Ark\"\n" + "version = \"0.1.0\"\n\n" + "[listing]\n" + "language = \"en\"\n" + "icon = \"👋\"\n" + "summary = \"The two passes of an app, in the fewest lines. Start here.\"\n" + "category = \"developer\"\n" + "license = \"BSD-3-Clause\"\n" + "source = \"https://github.com/dark-bio/examples/tree/main/apps/01-hello-c\"\n" + "keywords = [\"tutorial\", \"manifest\", \"two passes\", \"WASI\"]\n" + "description = '''\n" + "An Ark app is one WebAssembly file built for WASI Preview 1. The Ark runs it twice.\n\n" + "**The manifest pass.** With no arguments, the app prints a short TOML manifest naming itself, its version and the data it wants. The Ark checks the manifest and shows it on your phone.\n\n" + "**The report pass.** With a data directory as its first argument, the app reads the files it was granted and prints its report. Reports are Markdown, which is how Ark Hub shows them.\n\n" + "This app asks for no data, so approving it grants nothing. It proves the round trip, from upload through approval to a rendered report, in the fewest lines possible. Pick a language, read the source, then run it on your Ark.\n\n" + "Each language's folder in the examples repository builds and runs it with one `make run` command.\n\n" + "The docs cover the two passes, the manifest and the sandbox limits in detail.\n" + "'''\n"); return 0; } - // The run pass: a data directory was given. A real app would read it here. + // The report pass receives a data directory. This app reads nothing. printf("Hello from an Ark app written in C.\n"); return 0; } diff --git a/apps/01-hello-go/README.md b/apps/01-hello-go/README.md index ad38945..9dc0e70 100644 --- a/apps/01-hello-go/README.md +++ b/apps/01-hello-go/README.md @@ -15,5 +15,5 @@ To see each pass by hand, build the module and run it twice: ```sh make build APP=01-hello-go wasmtime build/01-hello-go.wasm # manifest pass -wasmtime build/01-hello-go.wasm / # run pass +wasmtime build/01-hello-go.wasm / # report pass ``` diff --git a/apps/01-hello-go/main.go b/apps/01-hello-go/main.go index fe128ea..f131629 100644 --- a/apps/01-hello-go/main.go +++ b/apps/01-hello-go/main.go @@ -11,17 +11,33 @@ import ( ) func main() { - // The manifest pass: no data directory was given, so describe the app and - // stop. This app reads nothing, so its dataset list is empty. + // The manifest pass has no data directory, so describe the app and stop. + // This app reads nothing, so it has no reads table. if len(os.Args) < 2 { fmt.Print( - "[package]\n" + - "name = \"hello-go\"\n" + - "version = \"0.1.0\"\n" + - "datasets = []\n") + "manifest = 1\n\n" + + "[app]\n" + + "name = \"Hello, Ark\"\n" + + "version = \"0.1.0\"\n\n" + + "[listing]\n" + + "language = \"en\"\n" + + "icon = \"👋\"\n" + + "summary = \"The two passes of an app, in the fewest lines. Start here.\"\n" + + "category = \"developer\"\n" + + "license = \"BSD-3-Clause\"\n" + + "source = \"https://github.com/dark-bio/examples/tree/main/apps/01-hello-go\"\n" + + "keywords = [\"tutorial\", \"manifest\", \"two passes\", \"WASI\"]\n" + + "description = '''\n" + + "An Ark app is one WebAssembly file built for WASI Preview 1. The Ark runs it twice.\n\n" + + "**The manifest pass.** With no arguments, the app prints a short TOML manifest naming itself, its version and the data it wants. The Ark checks the manifest and shows it on your phone.\n\n" + + "**The report pass.** With a data directory as its first argument, the app reads the files it was granted and prints its report. Reports are Markdown, which is how Ark Hub shows them.\n\n" + + "This app asks for no data, so approving it grants nothing. It proves the round trip, from upload through approval to a rendered report, in the fewest lines possible. Pick a language, read the source, then run it on your Ark.\n\n" + + "Each language's folder in the examples repository builds and runs it with one `make run` command.\n\n" + + "The docs cover the two passes, the manifest and the sandbox limits in detail.\n" + + "'''\n") return } - // The run pass: a data directory was given. A real app would read it here. + // The report pass receives a data directory. This app reads nothing. fmt.Println("Hello from an Ark app written in Go.") } diff --git a/apps/01-hello-python/README.md b/apps/01-hello-python/README.md index f349d21..a14e18b 100644 --- a/apps/01-hello-python/README.md +++ b/apps/01-hello-python/README.md @@ -18,5 +18,5 @@ To see each pass by hand, build the module and run it twice: ```sh make build APP=01-hello-python wasmtime build/01-hello-python.wasm # manifest pass -wasmtime build/01-hello-python.wasm / # run pass +wasmtime build/01-hello-python.wasm / # report pass ``` diff --git a/apps/01-hello-python/main.py b/apps/01-hello-python/main.py index 4323644..67bb7cd 100644 --- a/apps/01-hello-python/main.py +++ b/apps/01-hello-python/main.py @@ -5,10 +5,36 @@ def main(): # With no data directory, print the manifest and stop. if len(sys.argv) < 2: - print('[package]\nname = "hello-python"\nversion = "0.1.0"\ndatasets = []') + print("""manifest = 1 + +[app] +name = "Hello, Ark" +version = "0.1.0" + +[listing] +language = "en" +icon = "👋" +summary = "The two passes of an app, in the fewest lines. Start here." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/01-hello-python" +keywords = ["tutorial", "manifest", "two passes", "WASI"] +description = ''' +An Ark app is one WebAssembly file built for WASI Preview 1. The Ark runs it twice. + +**The manifest pass.** With no arguments, the app prints a short TOML manifest naming itself, its version and the data it wants. The Ark checks the manifest and shows it on your phone. + +**The report pass.** With a data directory as its first argument, the app reads the files it was granted and prints its report. Reports are Markdown, which is how Ark Hub shows them. + +This app asks for no data, so approving it grants nothing. It proves the round trip, from upload through approval to a rendered report, in the fewest lines possible. Pick a language, read the source, then run it on your Ark. + +Each language's folder in the examples repository builds and runs it with one `make run` command. + +The docs cover the two passes, the manifest and the sandbox limits in detail. +'''""") return - # The run pass receives a data directory, even when no data is requested. + # The report pass receives a data directory, even when no data is requested print("Hello from an Ark app written in Python.") diff --git a/apps/01-hello-rust/README.md b/apps/01-hello-rust/README.md index fc992dd..f0fc589 100644 --- a/apps/01-hello-rust/README.md +++ b/apps/01-hello-rust/README.md @@ -16,5 +16,5 @@ To see each pass by hand, build the module and run it twice: ```sh make build APP=01-hello-rust wasmtime build/01-hello-rust.wasm # manifest pass -wasmtime build/01-hello-rust.wasm / # run pass +wasmtime build/01-hello-rust.wasm / # report pass ``` diff --git a/apps/01-hello-rust/src/main.rs b/apps/01-hello-rust/src/main.rs index 418f4bb..2599610 100644 --- a/apps/01-hello-rust/src/main.rs +++ b/apps/01-hello-rust/src/main.rs @@ -5,18 +5,42 @@ //! ../../docs/01-app-model.md for the two passes. fn main() { - // The manifest pass: no data directory was given, so describe the app and - // stop. This app reads nothing, so its dataset list is empty. + // The manifest pass has no data directory, so describe the app and stop. + // This app reads nothing, so it has no reads table. if std::env::args().nth(1).is_none() { print!( - "[package]\n\ - name = \"hello-rust\"\n\ - version = \"0.1.0\"\n\ - datasets = []\n" + r#"manifest = 1 + +[app] +name = "Hello, Ark" +version = "0.1.0" + +[listing] +language = "en" +icon = "👋" +summary = "The two passes of an app, in the fewest lines. Start here." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/01-hello-rust" +keywords = ["tutorial", "manifest", "two passes", "WASI"] +description = ''' +An Ark app is one WebAssembly file built for WASI Preview 1. The Ark runs it twice. + +**The manifest pass.** With no arguments, the app prints a short TOML manifest naming itself, its version and the data it wants. The Ark checks the manifest and shows it on your phone. + +**The report pass.** With a data directory as its first argument, the app reads the files it was granted and prints its report. Reports are Markdown, which is how Ark Hub shows them. + +This app asks for no data, so approving it grants nothing. It proves the round trip, from upload through approval to a rendered report, in the fewest lines possible. Pick a language, read the source, then run it on your Ark. + +Each language's folder in the examples repository builds and runs it with one `make run` command. + +The docs cover the two passes, the manifest and the sandbox limits in detail. +''' +"# ); return; } - // The run pass: a data directory was given. A real app would read it here. + // The report pass receives a data directory. This app reads nothing. println!("Hello from an Ark app written in Rust."); } diff --git a/apps/02-permissions/Cargo.lock b/apps/02-permissions/Cargo.lock index 6e336bc..8474144 100644 --- a/apps/02-permissions/Cargo.lock +++ b/apps/02-permissions/Cargo.lock @@ -4,4 +4,4 @@ version = 4 [[package]] name = "permissions" -version = "0.1.0" +version = "0.2.0" diff --git a/apps/02-permissions/Cargo.toml b/apps/02-permissions/Cargo.toml index 808922c..b673aba 100644 --- a/apps/02-permissions/Cargo.toml +++ b/apps/02-permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "permissions" -version = "0.1.0" +version = "0.2.0" edition = "2024" # A smaller module uploads and starts faster on an Ark. diff --git a/apps/02-permissions/README.md b/apps/02-permissions/README.md index e6870ef..2f16c24 100644 --- a/apps/02-permissions/README.md +++ b/apps/02-permissions/README.md @@ -1,11 +1,16 @@ # 02 - permissions -A grant covers one directory and everything beneath it, and nothing else. This -app grants `v1/genome/rsids/rs72921001` and reads its genotype, then tries to -open the call file it never asked for, and shows that the read is blocked. +A grant covers only one directory and everything beneath it. This app grants +`v1/genome/rsids/rs72921001` and reads its genotype. It also asks for +`v1/genome/rsids/rs671` as an optional grant, which the owner may switch off, +and reports whether it was mounted. Then it tries to open the call file it never +asked for, and shows that the read is blocked. -Its manifest sets `develop = true`, so an Ark also returns what it prints to -standard error. Leave that flag out of apps you ship. +A declined optional grant and one whose data the Ark lacks are both left +unmounted, so the app can only say that it was one of the two. + +Its manifest sets `develop = true`, so its standard error goes to the owner's +review as well. Leave that flag out of apps you ship. [02-manifest.md](../../docs/02-manifest.md) covers grants, and [01-app-model.md](../../docs/01-app-model.md) the flag. @@ -13,6 +18,7 @@ standard error. Leave that flag out of apps you ship. ```sh make run APP=02-permissions +OPTIONAL=off make run APP=02-permissions # as if the owner declined rs671 make run APP=02-permissions FIXTURES=fixtures/no-call make run APP=02-permissions FIXTURES=fixtures/unanswered ``` diff --git a/apps/02-permissions/src/main.rs b/apps/02-permissions/src/main.rs index 3580125..49e23e3 100644 --- a/apps/02-permissions/src/main.rs +++ b/apps/02-permissions/src/main.rs @@ -1,34 +1,68 @@ -//! A directory grant and the develop flag. +//! Required and optional directory grants and the develop flag. //! -//! The app reads a granted genotype if it has an answer, then checks that an -//! undeclared call file is blocked. The local runner mounts only declared paths. -//! With develop enabled, an Ark returns stderr and stdout even on failure. -//! The local runner always shows both streams. Ship without develop. +//! The app reads a required genotype and checks whether an optional grant is +//! mounted before reading its answer. It then checks that an undeclared call +//! file is blocked. A declined or unavailable optional grant stays unmounted. +//! With develop enabled, the owner also reviews stderr, and stdout when the app +//! fails. The local runner always shows both streams. Ship without develop. use std::path::Path; use std::{fs, io}; -/// Declared in the manifest, so the Ark mounts it. +/// Required grant that the Ark mounts before the report pass. const GRANTED: &str = "v1/genome/rsids/rs72921001"; -/// Not declared, so it is never mounted and reading it fails. +/// Optional grant that the Ark mounts when available and approved. +const OPTIONAL: &str = "v1/genome/rsids/rs671"; +/// Undeclared call file that stays unmounted, so reading it fails. const DENIED: &str = "v1/genome/snp-indel/vcf"; +/// Prints the manifest or reports which grants can be read. fn main() { + // Print the manifest when no data root was passed let Some(dir) = std::env::args().nth(1) else { print!( - "[package]\n\ - name = \"permissions\"\n\ - version = \"0.1.0\"\n\ - datasets = [\"v1/genome/rsids/rs72921001\"]\n\ - develop = true\n" + r#"manifest = 1 + +[app] +name = "Permissions" +version = "0.2.0" +develop = true + +[reads] +paths = ["{GRANTED}"] +optional = ["{OPTIONAL}"] + +[listing] +language = "en" +icon = "🔒" +summary = "One grant, one optional grant, one blocked read, and the develop flag." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/02-permissions" +keywords = ["tutorial", "grants", "optional grants", "develop flag", "sandbox"] +description = ''' +A grant covers only one directory and everything beneath it. This app grants one variant, reads its genotype, then tries to open the call file it never asked for. The read fails, and the report says so. + +It also asks for a second variant as an optional grant, which starts switched off when you approve. A declined grant reads exactly like missing data, so the app cannot know whether you said no. + +The manifest also sets `develop = true`, so what the app prints to standard error goes to your review along with its report, even when it fails. Your phone shows a developer mode warning when approving such an app. Leave the flag out of apps you ship. + +Approving this app shares one variant, or two with the optional one switched on. The blocked read shows that the boundary is enforced by the Ark, not by the app's good manners. +''' + +[listing.purposes] +"v1/genome/rsids/rs72921001" = "The variant this demo reads, to show a granted read" +"v1/genome/rsids/rs671" = "An optional second variant, to show what a declined grant looks like" +"# ); return; }; - let root = Path::new(&dir); + // Open the report before checking each grant + let root = Path::new(&dir); println!("## Permissions\n"); - // The grant allows the read; the genotype can still have no answer. + // The required grant allows the read, but the genotype can have no answer match fs::read_to_string(root.join(GRANTED).join("genotype")) { Ok(genotype) => println!("- granted `{GRANTED}`: read genotype `{}`", genotype), Err(err) if err.kind() == io::ErrorKind::NotFound => { @@ -42,8 +76,33 @@ fn main() { } } - // The undeclared path was never mounted, so this fails. On a device the same - // holds: an app cannot reach data the owner did not approve. + // Check whether the optional directory is mounted before reading its answer + match fs::read_dir(root.join(OPTIONAL)) { + Ok(_) => match fs::read_to_string(root.join(OPTIONAL).join("genotype")) { + Ok(genotype) => println!("- optional `{OPTIONAL}`: read genotype `{genotype}`"), + Err(err) if err.kind() == io::ErrorKind::NotFound => { + println!("- optional `{OPTIONAL}`: no genotype answer"); + } + Err(err) => { + eprintln!("Could not read the optional genotype: {err}"); + std::process::exit(1); + } + }, + Err(err) + if matches!( + err.kind(), + io::ErrorKind::NotFound | io::ErrorKind::PermissionDenied + ) => + { + println!("- optional `{OPTIONAL}`: not mounted, so either declined or not on this Ark"); + } + Err(err) => { + eprintln!("Could not check the optional grant: {err}"); + std::process::exit(1); + } + } + + // The undeclared path stays unmounted, so reading it fails match fs::read_to_string(root.join(DENIED)) { Ok(_) => { eprintln!("The undeclared path was readable; check the sandbox mounts."); @@ -63,7 +122,6 @@ fn main() { } } - // This goes to standard error. The Ark returns stderr only when the manifest - // sets develop = true; otherwise you would never see it. + // The owner sees this diagnostic only because the manifest sets develop eprintln!("debug: this line is on stderr, visible only because develop = true"); } diff --git a/apps/03-cilantro-mini-go/README.md b/apps/03-cilantro-mini-go/README.md deleted file mode 100644 index 2baaf55..0000000 --- a/apps/03-cilantro-mini-go/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# 03 - cilantro-mini (Go) - -[03-cilantro-mini-rust](../03-cilantro-mini-rust) written in Go. It tells an -absent genotype from a failed read with `errors.Is(err, fs.ErrNotExist)`. - -## Build and run - -```sh -make run APP=03-cilantro-mini-go -make run APP=03-cilantro-mini-go FIXTURES=fixtures/no-call -make run APP=03-cilantro-mini-go FIXTURES=fixtures/unanswered -``` diff --git a/apps/03-cilantro-mini-python/README.md b/apps/03-cilantro-mini-python/README.md deleted file mode 100644 index c99e0c0..0000000 --- a/apps/03-cilantro-mini-python/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# 03 - cilantro-mini (Python) - -[03-cilantro-mini-rust](../03-cilantro-mini-rust) written in Python. An absent -genotype arrives as `FileNotFoundError` and reads as no answer, while every -other `OSError` stops the app. [01-hello-python](../01-hello-python) covers what -a Python module costs. - -## Build and run - -```sh -make run APP=03-cilantro-mini-python -make run APP=03-cilantro-mini-python FIXTURES=fixtures/no-call -make run APP=03-cilantro-mini-python FIXTURES=fixtures/unanswered -``` diff --git a/apps/03-sundae-order/Cargo.lock b/apps/03-sundae-order/Cargo.lock new file mode 100644 index 0000000..8ba359f --- /dev/null +++ b/apps/03-sundae-order/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "sundae-order" +version = "0.1.0" diff --git a/apps/03-sundae-order/Cargo.toml b/apps/03-sundae-order/Cargo.toml new file mode 100644 index 0000000..1535657 --- /dev/null +++ b/apps/03-sundae-order/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "sundae-order" +version = "0.1.0" +edition = "2024" + +# A smaller module uploads and starts faster on an Ark. +[profile.release] +opt-level = "z" +lto = "fat" +codegen-units = 1 +panic = "abort" +strip = "symbols" diff --git a/apps/03-sundae-order/README.md b/apps/03-sundae-order/README.md new file mode 100644 index 0000000..9cf45cb --- /dev/null +++ b/apps/03-sundae-order/README.md @@ -0,0 +1,21 @@ +# 03 - sundae order + +An app can ask the owner questions when they approve it. This one asks for a +flavor, any number of toppings and the words on the card, then prints the +sundae. It reads none of the owner's data, so its manifest has no `[reads]`. + +Each answer arrives as a file under `inputs/` in the data directory, named after +its input. `flavor` holds the picked choice, `toppings` the picks one per line, +and `card` the text, each without a trailing newline. The report quotes every +file exactly as the app read it. [02-manifest.md](../../docs/02-manifest.md) +covers inputs. + +## Build and run + +`make run` takes the answers from this app's `inputs/` folder, and `INPUTS` +points it at another directory of answer files. + +```sh +make run APP=03-sundae-order +INPUTS=my-answers make run APP=03-sundae-order +``` diff --git a/apps/03-sundae-order/inputs/card b/apps/03-sundae-order/inputs/card new file mode 100644 index 0000000..b2889d8 --- /dev/null +++ b/apps/03-sundae-order/inputs/card @@ -0,0 +1 @@ +Happy birthday! \ No newline at end of file diff --git a/apps/03-sundae-order/inputs/flavor b/apps/03-sundae-order/inputs/flavor new file mode 100644 index 0000000..1224880 --- /dev/null +++ b/apps/03-sundae-order/inputs/flavor @@ -0,0 +1 @@ +Pistachio \ No newline at end of file diff --git a/apps/03-sundae-order/inputs/toppings b/apps/03-sundae-order/inputs/toppings new file mode 100644 index 0000000..bdf82f1 --- /dev/null +++ b/apps/03-sundae-order/inputs/toppings @@ -0,0 +1,2 @@ +Sprinkles +Cherry \ No newline at end of file diff --git a/apps/03-sundae-order/src/main.rs b/apps/03-sundae-order/src/main.rs new file mode 100644 index 0000000..cea315c --- /dev/null +++ b/apps/03-sundae-order/src/main.rs @@ -0,0 +1,133 @@ +//! A sundae order built from the owner's answers to three questions. +//! +//! The manifest declares a choice, a multiple choice and a text input. The Ark +//! mounts each answer as a file under `inputs/` in the data directory, and the +//! report quotes every file exactly as it was read. + +use std::fs; +use std::path::Path; + +/// Prints the manifest or reports the owner's sundae order. +fn main() { + // Print the questions when no data root was passed + let Some(dir) = std::env::args().nth(1) else { + print!( + r#"manifest = 1 + +[app] +name = "Sundae Order" +version = "0.1.0" + +[inputs.flavor] +type = "choice" +prompt = "Which flavor should the scoop be?" +choices = ["Vanilla", "Chocolate", "Strawberry", "Pistachio"] +default = "Vanilla" + +[inputs.toppings] +type = "choice" +prompt = "Which toppings go on top?" +choices = ["Sprinkles", "Hot fudge", "Whipped cream", "Cherry"] +multiple = true +default = ["Hot fudge", "Cherry"] + +[inputs.card] +type = "text" +prompt = "What should the card say?" +max = 40 + +[listing] +language = "en" +icon = "🍨" +summary = "A flavor, some toppings and a card. The first app that asks you questions." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/03-sundae-order" +keywords = ["tutorial", "inputs", "choices", "text input"] +description = ''' +An app can ask you questions when you approve it. This one asks for a flavor, any number of toppings and the words on the card, then prints your sundae. + +Each answer reaches the app as a plain file under `inputs`. A choice holds the picked choice, a multiple choice holds one pick per line, and a text holds exactly what you typed. The app reads none of your genome, so approving it shares only your answers with it. + +Answers never change what an app can read. The grants on the approval are the whole of it. +''' +"# + ); + return; + }; + + // Read each answer exactly as the Ark mounted it, without trimming + let inputs = Path::new(&dir).join("inputs"); + let flavor = read_input(&inputs, "flavor"); + let toppings = read_input(&inputs, "toppings"); + let card = read_input(&inputs, "card"); + + // Multiple choices hold one pick per line, with an empty file for no picks + let picks: Vec = toppings.lines().map(str::to_owned).collect(); + let topping_names = if picks.is_empty() { + "no toppings".to_owned() + } else { + join_picks(&picks) + }; + + // Put the order first, followed by each input's value beside its path + println!("# Your Sundae\n"); + println!("One scoop of {flavor} with {topping_names}, and a card that reads \"{card}\".\n"); + println!("## Evidence\n"); + println!("- `inputs/flavor` holds {}.", inline_code(&flavor)); + match picks.as_slice() { + [] => println!("- `inputs/toppings` is empty, so no toppings were picked."), + [pick] => println!("- `inputs/toppings` holds 1 pick: {}.", inline_code(pick)), + _ => { + let values: Vec = picks.iter().map(|pick| inline_code(pick)).collect(); + println!( + "- `inputs/toppings` holds {} picks, one per line: {}.", + picks.len(), + join_picks(&values) + ); + } + } + println!("- `inputs/card` holds {}.", inline_code(&card)); +} + +/// Reads one input, reporting an error and exiting if its file cannot be read. +fn read_input(inputs: &Path, name: &str) -> String { + match fs::read_to_string(inputs.join(name)) { + Ok(value) => value, + Err(err) => { + eprintln!("Could not read inputs/{name}: {err}"); + std::process::exit(1); + } + } +} + +/// Joins picks in their file order with commas and a final conjunction. +fn join_picks(picks: &[String]) -> String { + match picks { + [] => String::new(), + [one] => one.clone(), + [first, second] => format!("{first} and {second}"), + [rest @ .., last] => format!("{}, and {last}", rest.join(", ")), + } +} + +/// Wraps a value in inline code, preserving backticks and surrounding spaces. +fn inline_code(value: &str) -> String { + // Use a fence longer than any run of backticks inside the value + let longest = value + .split(|character| character != '`') + .map(str::len) + .max() + .unwrap_or(0); + let fence = "`".repeat(longest + 1); + + // Padding keeps edge backticks and paired spaces inside the code span + let pad = value.starts_with('`') + || value.ends_with('`') + || (value.starts_with(' ') && value.ends_with(' ') && !value.chars().all(|c| c == ' ')); + if pad { + format!("{fence} {value} {fence}") + } else { + format!("{fence}{value}{fence}") + } +} diff --git a/apps/03-cilantro-mini-c/README.md b/apps/04-cilantro-mini-c/README.md similarity index 53% rename from apps/03-cilantro-mini-c/README.md rename to apps/04-cilantro-mini-c/README.md index 6c4c3c3..5c88573 100644 --- a/apps/03-cilantro-mini-c/README.md +++ b/apps/04-cilantro-mini-c/README.md @@ -1,6 +1,6 @@ -# 03 - cilantro-mini (C) +# 04 - cilantro-mini (C) -[03-cilantro-mini-rust](../03-cilantro-mini-rust) written in C. Only +[04-cilantro-mini-rust](../04-cilantro-mini-rust) written in C. Only `errno == ENOENT` means no answer, and any other failure to open or read a file stops the app. Each file is read into a buffer that grows to fit, since a genotype can hold long alleles. [01-hello-c](../01-hello-c) covers the @@ -9,7 +9,7 @@ toolchain. ## Build and run ```sh -make run APP=03-cilantro-mini-c -make run APP=03-cilantro-mini-c FIXTURES=fixtures/no-call -make run APP=03-cilantro-mini-c FIXTURES=fixtures/unanswered +make run APP=04-cilantro-mini-c +make run APP=04-cilantro-mini-c FIXTURES=fixtures/no-call +make run APP=04-cilantro-mini-c FIXTURES=fixtures/unanswered ``` diff --git a/apps/03-cilantro-mini-c/main.c b/apps/04-cilantro-mini-c/main.c similarity index 71% rename from apps/03-cilantro-mini-c/main.c rename to apps/04-cilantro-mini-c/main.c index 681203f..5804a4d 100644 --- a/apps/03-cilantro-mini-c/main.c +++ b/apps/04-cilantro-mini-c/main.c @@ -4,7 +4,7 @@ // olfactory receptor, tracks the trait: the more copies of the C allele you // carry, the more soapy cilantro tends to taste (Eriksson et al., 2012). // -// It is the same program as ../03-cilantro-mini-rust, in C. The rsids/ lens hands +// This is ../04-cilantro-mini-rust in C. The rsids/ lens hands // back the genotype and coordinate as plain files; the app never opens a // variant file. See ../../docs/04-reading-data.md. #include @@ -62,10 +62,27 @@ static char *read_leaf(const char *dir, const char *name) { int main(int argc, char *argv[]) { if (argc < 2) { - printf("[package]\n" - "name = \"cilantro-mini\"\n" - "version = \"0.1.0\"\n" - "datasets = [\"" LENS "\"]\n"); + printf("manifest = 1\n\n" + "[app]\n" + "name = \"One Variant\"\n" + "version = \"0.1.0\"\n\n" + "[reads]\n" + "paths = [\"" LENS "\"]\n\n" + "[listing]\n" + "language = \"en\"\n" + "icon = \"🌱\"\n" + "summary = \"The smallest real data access, through the rsids lens, in four languages.\"\n" + "category = \"developer\"\n" + "license = \"BSD-3-Clause\"\n" + "source = \"https://github.com/dark-bio/examples/tree/main/apps/04-cilantro-mini-c\"\n" + "keywords = [\"tutorial\", \"rsids\", \"genotype\", \"four languages\"]\n" + "description = '''\n" + "The rsids lens hands back one variant as plain files. Grant `v1/genome/rsids/rs72921001` and read `genotype`, `chromosome`, `position` and `reference` from that directory. The app never opens the call file.\n\n" + "Three outcomes every app has to handle. A missing allele, as in `./.`, makes a copy count inconclusive. An absent genotype means no answer, never two reference alleles. Any other read error stops the app.\n\n" + "The same program is written in Rust, Go, C and Python. Compare them, then see the full report version, Cilantro Taste Test.\n" + "'''\n\n" + "[listing.purposes]\n" + "\"v1/genome/rsids/rs72921001\" = \"The one variant this tutorial reads\"\n"); return 0; } diff --git a/apps/04-cilantro-mini-go/README.md b/apps/04-cilantro-mini-go/README.md new file mode 100644 index 0000000..20b86e5 --- /dev/null +++ b/apps/04-cilantro-mini-go/README.md @@ -0,0 +1,12 @@ +# 04 - cilantro-mini (Go) + +[04-cilantro-mini-rust](../04-cilantro-mini-rust) written in Go. It distinguishes +an absent genotype from a failed read with `errors.Is(err, fs.ErrNotExist)`. + +## Build and run + +```sh +make run APP=04-cilantro-mini-go +make run APP=04-cilantro-mini-go FIXTURES=fixtures/no-call +make run APP=04-cilantro-mini-go FIXTURES=fixtures/unanswered +``` diff --git a/apps/03-cilantro-mini-go/go.mod b/apps/04-cilantro-mini-go/go.mod similarity index 100% rename from apps/03-cilantro-mini-go/go.mod rename to apps/04-cilantro-mini-go/go.mod diff --git a/apps/03-cilantro-mini-go/main.go b/apps/04-cilantro-mini-go/main.go similarity index 63% rename from apps/03-cilantro-mini-go/main.go rename to apps/04-cilantro-mini-go/main.go index 0e14fff..2af656b 100644 --- a/apps/03-cilantro-mini-go/main.go +++ b/apps/04-cilantro-mini-go/main.go @@ -4,7 +4,7 @@ // olfactory receptor, tracks the trait: the more copies of the C allele you // carry, the more soapy cilantro tends to taste (Eriksson et al., 2012). // -// It is the same program as ../03-cilantro-mini-rust, in Go. The rsids/ lens hands +// This is ../04-cilantro-mini-rust in Go. The rsids/ lens hands // back the genotype and coordinate as plain files; the app never opens a // variant file. See ../../docs/04-reading-data.md. package main @@ -25,16 +25,33 @@ const ( ) func main() { - // The manifest pass: name the app and ask for the one variant directory. + // The manifest pass names the app and asks for the one variant directory if len(os.Args) < 2 { - fmt.Printf("[package]\n"+ - "name = \"cilantro-mini\"\n"+ - "version = \"0.1.0\"\n"+ - "datasets = [\"%s\"]\n", lens) + fmt.Printf("manifest = 1\n\n"+ + "[app]\n"+ + "name = \"One Variant\"\n"+ + "version = \"0.1.0\"\n\n"+ + "[reads]\n"+ + "paths = [\"%s\"]\n\n"+ + "[listing]\n"+ + "language = \"en\"\n"+ + "icon = \"🌱\"\n"+ + "summary = \"The smallest real data access, through the rsids lens, in four languages.\"\n"+ + "category = \"developer\"\n"+ + "license = \"BSD-3-Clause\"\n"+ + "source = \"https://github.com/dark-bio/examples/tree/main/apps/04-cilantro-mini-go\"\n"+ + "keywords = [\"tutorial\", \"rsids\", \"genotype\", \"four languages\"]\n"+ + "description = '''\n"+ + "The rsids lens hands back one variant as plain files. Grant `v1/genome/rsids/rs72921001` and read `genotype`, `chromosome`, `position` and `reference` from that directory. The app never opens the call file.\n\n"+ + "Three outcomes every app has to handle. A missing allele, as in `./.`, makes a copy count inconclusive. An absent genotype means no answer, never two reference alleles. Any other read error stops the app.\n\n"+ + "The same program is written in Rust, Go, C and Python. Compare them, then see the full report version, Cilantro Taste Test.\n"+ + "'''\n\n"+ + "[listing.purposes]\n"+ + "\"v1/genome/rsids/rs72921001\" = \"The one variant this tutorial reads\"\n", lens) return } - // The run pass: everything comes from the one lens directory. + // The report pass reads everything from the one lens directory base := filepath.Join(os.Args[1], lens) // ENOENT means no answer, including reference sites in variants-only calls. diff --git a/apps/04-cilantro-mini-python/README.md b/apps/04-cilantro-mini-python/README.md new file mode 100644 index 0000000..79e7c00 --- /dev/null +++ b/apps/04-cilantro-mini-python/README.md @@ -0,0 +1,14 @@ +# 04 - cilantro-mini (Python) + +[04-cilantro-mini-rust](../04-cilantro-mini-rust) written in Python. An absent +genotype arrives as `FileNotFoundError` and reads as no answer, while every +other `OSError` stops the app. [01-hello-python](../01-hello-python) covers what +a Python module costs. + +## Build and run + +```sh +make run APP=04-cilantro-mini-python +make run APP=04-cilantro-mini-python FIXTURES=fixtures/no-call +make run APP=04-cilantro-mini-python FIXTURES=fixtures/unanswered +``` diff --git a/apps/03-cilantro-mini-python/main.py b/apps/04-cilantro-mini-python/main.py similarity index 65% rename from apps/03-cilantro-mini-python/main.py rename to apps/04-cilantro-mini-python/main.py index 1d353d2..ce5faaa 100644 --- a/apps/03-cilantro-mini-python/main.py +++ b/apps/04-cilantro-mini-python/main.py @@ -19,8 +19,34 @@ def leaf(base, name): def main(): if len(sys.argv) < 2: - print('[package]\nname = "cilantro-mini"\nversion = "0.1.0"\n' - f'datasets = ["{LENS}"]') + print(f"""manifest = 1 + +[app] +name = "One Variant" +version = "0.1.0" + +[reads] +paths = ["{LENS}"] + +[listing] +language = "en" +icon = "🌱" +summary = "The smallest real data access, through the rsids lens, in four languages." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/04-cilantro-mini-python" +keywords = ["tutorial", "rsids", "genotype", "four languages"] +description = ''' +The rsids lens hands back one variant as plain files. Grant `v1/genome/rsids/rs72921001` and read `genotype`, `chromosome`, `position` and `reference` from that directory. The app never opens the call file. + +Three outcomes every app has to handle. A missing allele, as in `./.`, makes a copy count inconclusive. An absent genotype means no answer, never two reference alleles. Any other read error stops the app. + +The same program is written in Rust, Go, C and Python. Compare them, then see the full report version, Cilantro Taste Test. +''' + +[listing.purposes] +"v1/genome/rsids/rs72921001" = "The one variant this tutorial reads" +""", end="") return base = f"{sys.argv[1]}/{LENS}" diff --git a/apps/03-cilantro-mini-rust/Cargo.lock b/apps/04-cilantro-mini-rust/Cargo.lock similarity index 100% rename from apps/03-cilantro-mini-rust/Cargo.lock rename to apps/04-cilantro-mini-rust/Cargo.lock diff --git a/apps/03-cilantro-mini-rust/Cargo.toml b/apps/04-cilantro-mini-rust/Cargo.toml similarity index 100% rename from apps/03-cilantro-mini-rust/Cargo.toml rename to apps/04-cilantro-mini-rust/Cargo.toml diff --git a/apps/03-cilantro-mini-rust/README.md b/apps/04-cilantro-mini-rust/README.md similarity index 61% rename from apps/03-cilantro-mini-rust/README.md rename to apps/04-cilantro-mini-rust/README.md index 2739d25..9a1d1f5 100644 --- a/apps/03-cilantro-mini-rust/README.md +++ b/apps/04-cilantro-mini-rust/README.md @@ -1,4 +1,4 @@ -# 03 - cilantro-mini (Rust) +# 04 - cilantro-mini (Rust) The smallest real data access. The app grants one variant, `v1/genome/rsids/rs72921001`, reads its genotype and counts copies of the `C` @@ -10,14 +10,14 @@ makes the count inconclusive. An absent genotype is reported as no answer, never as two reference alleles. Any other read error stops the app. [04-reading-data.md](../../docs/04-reading-data.md) explains why. -The same mini exists in [Go](../03-cilantro-mini-go), -[C](../03-cilantro-mini-c) and [Python](../03-cilantro-mini-python), and -[03-cilantro-soapiness](../03-cilantro-soapiness) turns it into a report. +The same mini exists in [Go](../04-cilantro-mini-go), +[C](../04-cilantro-mini-c) and [Python](../04-cilantro-mini-python), and +[04-cilantro-soapiness](../04-cilantro-soapiness) turns it into a report. ## Build and run ```sh -make run APP=03-cilantro-mini-rust -make run APP=03-cilantro-mini-rust FIXTURES=fixtures/no-call -make run APP=03-cilantro-mini-rust FIXTURES=fixtures/unanswered +make run APP=04-cilantro-mini-rust +make run APP=04-cilantro-mini-rust FIXTURES=fixtures/no-call +make run APP=04-cilantro-mini-rust FIXTURES=fixtures/unanswered ``` diff --git a/apps/03-cilantro-mini-rust/src/main.rs b/apps/04-cilantro-mini-rust/src/main.rs similarity index 71% rename from apps/03-cilantro-mini-rust/src/main.rs rename to apps/04-cilantro-mini-rust/src/main.rs index ee5b1db..89f55aa 100644 --- a/apps/03-cilantro-mini-rust/src/main.rs +++ b/apps/04-cilantro-mini-rust/src/main.rs @@ -18,18 +18,42 @@ const SOAPY_ALLELE: &str = "C"; const LENS: &str = "v1/genome/rsids/rs72921001"; fn main() { - // The manifest pass: name the app and ask for the one variant directory. + // The manifest pass names the app and asks for the one variant directory let Some(dir) = std::env::args().nth(1) else { print!( - "[package]\n\ - name = \"cilantro-mini\"\n\ - version = \"0.1.0\"\n\ - datasets = [\"{LENS}\"]\n" + r#"manifest = 1 + +[app] +name = "One Variant" +version = "0.1.0" + +[reads] +paths = ["{LENS}"] + +[listing] +language = "en" +icon = "🌱" +summary = "The smallest real data access, through the rsids lens, in four languages." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/04-cilantro-mini-rust" +keywords = ["tutorial", "rsids", "genotype", "four languages"] +description = ''' +The rsids lens hands back one variant as plain files. Grant `v1/genome/rsids/rs72921001` and read `genotype`, `chromosome`, `position` and `reference` from that directory. The app never opens the call file. + +Three outcomes every app has to handle. A missing allele, as in `./.`, makes a copy count inconclusive. An absent genotype means no answer, never two reference alleles. Any other read error stops the app. + +The same program is written in Rust, Go, C and Python. Compare them, then see the full report version, Cilantro Taste Test. +''' + +[listing.purposes] +"v1/genome/rsids/rs72921001" = "The one variant this tutorial reads" +"# ); return; }; - // The run pass: everything comes from the one lens directory. + // The report pass reads everything from the one lens directory let base = Path::new(&dir).join(LENS); // ENOENT means no answer, including reference sites in variants-only calls. diff --git a/apps/03-cilantro-soapiness/Cargo.lock b/apps/04-cilantro-soapiness/Cargo.lock similarity index 100% rename from apps/03-cilantro-soapiness/Cargo.lock rename to apps/04-cilantro-soapiness/Cargo.lock diff --git a/apps/03-cilantro-soapiness/Cargo.toml b/apps/04-cilantro-soapiness/Cargo.toml similarity index 100% rename from apps/03-cilantro-soapiness/Cargo.toml rename to apps/04-cilantro-soapiness/Cargo.toml diff --git a/apps/03-cilantro-soapiness/README.md b/apps/04-cilantro-soapiness/README.md similarity index 66% rename from apps/03-cilantro-soapiness/README.md rename to apps/04-cilantro-soapiness/README.md index 2b6a313..a07b068 100644 --- a/apps/03-cilantro-soapiness/README.md +++ b/apps/04-cilantro-soapiness/README.md @@ -1,6 +1,6 @@ -# 03 - cilantro-soapiness +# 04 - cilantro-soapiness -The single variant from [03-cilantro-mini-rust](../03-cilantro-mini-rust), +The single variant from [04-cilantro-mini-rust](../04-cilantro-mini-rust), turned into a full report in the shape [06-reports.md](../../docs/06-reports.md) describes, with a finding, the evidence behind it, the method, its limitations and sources. A no-call and an absent genotype each get their @@ -11,7 +11,7 @@ read for one file, the assembly build the coordinate is reported on. ## Build and run ```sh -make run APP=03-cilantro-soapiness -make run APP=03-cilantro-soapiness FIXTURES=fixtures/no-call -make run APP=03-cilantro-soapiness FIXTURES=fixtures/unanswered +make run APP=04-cilantro-soapiness +make run APP=04-cilantro-soapiness FIXTURES=fixtures/no-call +make run APP=04-cilantro-soapiness FIXTURES=fixtures/unanswered ``` diff --git a/apps/03-cilantro-soapiness/src/main.rs b/apps/04-cilantro-soapiness/src/main.rs similarity index 82% rename from apps/03-cilantro-soapiness/src/main.rs rename to apps/04-cilantro-soapiness/src/main.rs index 58792bf..3fc5e6c 100644 --- a/apps/03-cilantro-soapiness/src/main.rs +++ b/apps/04-cilantro-soapiness/src/main.rs @@ -1,7 +1,7 @@ //! Cilantro Soapiness: the cilantro taste test as a full Ark report. //! //! This is the full version of the cilantro example. The mini version -//! (../03-cilantro-mini-rust) shows the bare lens read in a handful of lines; +//! (../04-cilantro-mini-rust) shows the bare lens read in a handful of lines; //! this one wraps the same single genotype in a complete report in the shape //! docs/06-reports.md describes: a finding, the evidence behind it, the method, //! its limitations and sources. The data access is identical. @@ -26,10 +26,37 @@ fn main() { // coordinate is reported on. let Some(dir) = std::env::args().nth(1) else { print!( - "[package]\n\ - name = \"cilantro-soapiness\"\n\ - version = \"0.4.0\"\n\ - datasets = [\"v1/genome/rsids/rs72921001\", \"v1/genome/reference\"]\n" + r#"manifest = 1 + +[app] +name = "Cilantro Taste Test" +version = "0.4.0" + +[reads] +paths = ["v1/genome/rsids/rs72921001", "v1/genome/reference"] + +[listing] +language = "en" +icon = "🌿" +summary = "Does cilantro taste like soap to you? One variant near OR6A2 has a say." +category = "traits" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/04-cilantro-soapiness" +keywords = ["cilantro", "taste", "smell", "OR6A2", "rs72921001"] +description = ''' +Some people love cilantro. Others taste dish soap. A 2012 study by Eriksson and colleagues tied the difference to one variant, `rs72921001`, near the _OR6A2_ olfactory receptor gene on chromosome 11. Each copy of the C allele makes the soapy reading more likely. + +The app reads that one variant and the reference base beside it. The report says which reading your genotype is, the evidence it rests on and what it does not establish. A missing call or an absent genotype gets its own explanation instead of a verdict. + +**How to read the result.** Two C copies is the genotype most strongly associated with the soapy taste. One copy sits in between. Zero copies is the fresh, herby reading. Taste is polygenic and shaped by diet and exposure, so the verdict is a tendency, not a fact about you. + +**Further reading.** Eriksson N et al. (2012), A genetic variant near olfactory receptor genes influences cilantro preference, _Flavour_ 1:22. +''' + +[listing.purposes] +"v1/genome/rsids/rs72921001" = "The variant near OR6A2 tied to tasting cilantro as soap" +"v1/genome/reference" = "The reference build, to name the variant's coordinates" +"# ); return; }; @@ -81,7 +108,7 @@ fn main() { "**No answer.** No genotype is available at {RSID}, so this app has no \ result. An absent genotype is not a reference call. A call file that \ records only variants has no record at a reference site, and none at a \ - site it didn't cover, and the two can't be told apart here." + site it didn't cover, and the two can't be distinguished here." ); } } @@ -93,7 +120,7 @@ fn main() { print_sources(); } -/// What the lens can tell us about the user at this site. +/// What the lens reveals about the user at this site. enum Call { Called(String), Uncertain(String), diff --git a/apps/04-drunk-o-type/Cargo.lock b/apps/05-drunk-o-type/Cargo.lock similarity index 100% rename from apps/04-drunk-o-type/Cargo.lock rename to apps/05-drunk-o-type/Cargo.lock diff --git a/apps/04-drunk-o-type/Cargo.toml b/apps/05-drunk-o-type/Cargo.toml similarity index 100% rename from apps/04-drunk-o-type/Cargo.toml rename to apps/05-drunk-o-type/Cargo.toml diff --git a/apps/04-drunk-o-type/README.md b/apps/05-drunk-o-type/README.md similarity index 68% rename from apps/04-drunk-o-type/README.md rename to apps/05-drunk-o-type/README.md index 6a0f675..02506ae 100644 --- a/apps/04-drunk-o-type/README.md +++ b/apps/05-drunk-o-type/README.md @@ -1,8 +1,8 @@ -# 04 - drunk-o-type +# 05 - drunk-o-type A panel of eleven variants, scored on three axes of how a body handles alcohol: flushing, reward and histamine. Each variant is its own `rsids/` grant, so the -access pattern is [03-cilantro-mini-rust](../03-cilantro-mini-rust) repeated, +access pattern is [04-cilantro-mini-rust](../04-cilantro-mini-rust) repeated, and the owner sees every site by name before approving. The app counts the allele each study tested, which isn't always the ALT allele. @@ -12,7 +12,7 @@ lowering the score. This is a demonstration, not medical advice. ## Build and run ```sh -make run APP=04-drunk-o-type -make run APP=04-drunk-o-type FIXTURES=fixtures/no-call -make run APP=04-drunk-o-type FIXTURES=fixtures/unanswered +make run APP=05-drunk-o-type +make run APP=05-drunk-o-type FIXTURES=fixtures/no-call +make run APP=05-drunk-o-type FIXTURES=fixtures/unanswered ``` diff --git a/apps/04-drunk-o-type/src/main.rs b/apps/05-drunk-o-type/src/main.rs similarity index 92% rename from apps/04-drunk-o-type/src/main.rs rename to apps/05-drunk-o-type/src/main.rs index 41f8f9e..3440420 100644 --- a/apps/04-drunk-o-type/src/main.rs +++ b/apps/05-drunk-o-type/src/main.rs @@ -776,25 +776,59 @@ fn print_further_reading() { // ── Entry point ──────────────────────────────────────────────────────────── -const METADATA: &str = "\ -[package] -name = \"drunk-o-type\" -version = \"0.3.0\" -datasets = [ - \"v1/genome/rsids/rs671\", - \"v1/genome/rsids/rs1229984\", - \"v1/genome/rsids/rs2066702\", - \"v1/genome/rsids/rs1799971\", - \"v1/genome/rsids/rs1800497\", - \"v1/genome/rsids/rs279858\", - \"v1/genome/rsids/rs10156191\", - \"v1/genome/rsids/rs1049742\", - \"v1/genome/rsids/rs1049793\", - \"v1/genome/rsids/rs2052129\", - \"v1/genome/rsids/rs11558538\", - \"v1/genome/reference\", +/// App identity, grants and listing printed by the manifest pass. +const MANIFEST: &str = r#"manifest = 1 + +[app] +name = "Drunk-o-type" +version = "0.3.0" + +[reads] +paths = [ + "v1/genome/rsids/rs671", + "v1/genome/rsids/rs1229984", + "v1/genome/rsids/rs2066702", + "v1/genome/rsids/rs1799971", + "v1/genome/rsids/rs1800497", + "v1/genome/rsids/rs279858", + "v1/genome/rsids/rs10156191", + "v1/genome/rsids/rs1049742", + "v1/genome/rsids/rs1049793", + "v1/genome/rsids/rs2052129", + "v1/genome/rsids/rs11558538", + "v1/genome/reference", ] -"; + +[listing] +language = "en" +icon = "🍻" +summary = "Eleven variants on the flush, the buzz and how fermented drinks treat you." +category = "traits" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/05-drunk-o-type" +keywords = ["alcohol", "flush", "histamine", "ALDH2", "variant panel"] +description = ''' +Eleven variants across three axes. **Flush** asks whether alcohol makes you sick, through _ALDH2_ and _ADH1B_. **Like** asks whether your brain enjoys it, through _OPRM1_, _ANKK1_ and _GABRA2_. **Junk** asks whether the histamine in fermented drinks wrecks you, through _AOC1_ and _HNMT_. Each axis bins into five tiers and the three tiers combine into your drunk-o-type. + +Every variant is its own grant, so your phone lists all eleven sites by name before you approve. The app counts the allele each study tested, which is not always the ALT allele. A missing allele or an absent genotype makes its axis inconclusive rather than lowering the score. + +This is a demonstration of a variant panel, not medical advice, and the tier names are jokes. +''' + +[listing.purposes] +"v1/genome/rsids/rs671" = "ALDH2, the main driver of the alcohol flush" +"v1/genome/rsids/rs1229984" = "ADH1B, how fast alcohol turns into acetaldehyde" +"v1/genome/rsids/rs2066702" = "ADH1B, a second variant of the same enzyme" +"v1/genome/rsids/rs1799971" = "OPRM1, tied to how rewarding alcohol feels" +"v1/genome/rsids/rs1800497" = "ANKK1 beside DRD2, tied to dopamine reward" +"v1/genome/rsids/rs279858" = "GABRA2, tied to alcohol's unwinding effect" +"v1/genome/rsids/rs10156191" = "AOC1, the enzyme that breaks down histamine from food and drink" +"v1/genome/rsids/rs1049742" = "AOC1, a second variant of the histamine enzyme" +"v1/genome/rsids/rs1049793" = "AOC1, the most common variant of the histamine enzyme" +"v1/genome/rsids/rs2052129" = "AOC1, a variant upstream of the gene that may lower how much is made" +"v1/genome/rsids/rs11558538" = "HNMT, a second enzyme that clears histamine" +"v1/genome/reference" = "The reference build, to name the variants' coordinates" +"#; // ENOENT means no answer; every other read error fails the app. fn read_leaf(base: &Path, name: &str) -> Result, Box> { @@ -838,7 +872,7 @@ fn resolve_target(base: &Path, target: &SnpTarget) -> Result Result<(), Box> { if std::env::args().len() < 2 { - print!("{}", METADATA); + print!("{}", MANIFEST); return Ok(()); } diff --git a/apps/05-genes-mini-go/README.md b/apps/05-genes-mini-go/README.md deleted file mode 100644 index 7e97f98..0000000 --- a/apps/05-genes-mini-go/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# 05 - genes-mini (Go) - -[05-genes-mini-rust](../05-genes-mini-rust) written in Go. It takes the sequence -length from `os.Stat` and tells an absent scalar from a failed read with -`errors.Is(err, fs.ErrNotExist)`. - -## Build and run - -```sh -make run APP=05-genes-mini-go -``` diff --git a/apps/05-bitter-meter/Cargo.lock b/apps/06-bitter-meter/Cargo.lock similarity index 100% rename from apps/05-bitter-meter/Cargo.lock rename to apps/06-bitter-meter/Cargo.lock diff --git a/apps/05-bitter-meter/Cargo.toml b/apps/06-bitter-meter/Cargo.toml similarity index 100% rename from apps/05-bitter-meter/Cargo.toml rename to apps/06-bitter-meter/Cargo.toml diff --git a/apps/05-bitter-meter/README.md b/apps/06-bitter-meter/README.md similarity index 92% rename from apps/05-bitter-meter/README.md rename to apps/06-bitter-meter/README.md index 1424753..ea6484e 100644 --- a/apps/05-bitter-meter/README.md +++ b/apps/06-bitter-meter/README.md @@ -1,4 +1,4 @@ -# 05 - bitter-meter +# 06 - bitter-meter A full report on TAS2R38, the bitter taste receptor gene. It streams the gene's `sequence` for its GC content, and lists the owner's `changes` inside the gene @@ -13,5 +13,5 @@ stops the app instead of reading as no changes. ## Build and run ```sh -make run APP=05-bitter-meter +make run APP=06-bitter-meter ``` diff --git a/apps/05-bitter-meter/src/main.rs b/apps/06-bitter-meter/src/main.rs similarity index 92% rename from apps/05-bitter-meter/src/main.rs rename to apps/06-bitter-meter/src/main.rs index 0429ffb..30ab7b8 100644 --- a/apps/05-bitter-meter/src/main.rs +++ b/apps/06-bitter-meter/src/main.rs @@ -23,10 +23,35 @@ const RULER_WIDTH: usize = 64; fn main() { let Some(dir) = std::env::args().nth(1) else { print!( - "[package]\n\ - name = \"bitter-meter\"\n\ - version = \"0.2.0\"\n\ - datasets = [\"v1/genome/genes/TAS2R38\", \"v1/genome/reference\"]\n" + r#"manifest = 1 + +[app] +name = "Bitter Meter" +version = "0.2.0" + +[reads] +paths = ["v1/genome/genes/TAS2R38", "v1/genome/reference"] + +[listing] +language = "en" +icon = "🥦" +summary = "A postcard from TAS2R38, the bitter taste receptor, with your changes in it." +category = "traits" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/06-bitter-meter" +keywords = ["bitter taste", "TAS2R38", "PTC", "taste receptor"] +description = ''' +_TAS2R38_ is the bitter taste receptor from the PTC and PROP classroom experiments. This app reads the gene through the genes lens. It streams the reference sequence for its GC content and lists every position inside the gene where your call file holds a non reference allele, with the reference base and your genotype at each. + +The report is deliberately a postcard, not a taster prediction. The classic taster story depends on specific coding variants and their phase, and this app does not infer either from the gene view. + +One grant covers the whole gene, including your variants in it. The app walks the changes listing carefully. A position without leaves still counts, genotypes are split around symbolic alleles, and a listing that fails stops the app instead of reading as no changes. +''' + +[listing.purposes] +"v1/genome/genes/TAS2R38" = "The bitter taste receptor gene, its sequence and your changes in it" +"v1/genome/reference" = "The reference build, to name positions" +"# ); return; }; diff --git a/apps/05-genes-mini-c/README.md b/apps/06-genes-mini-c/README.md similarity index 65% rename from apps/05-genes-mini-c/README.md rename to apps/06-genes-mini-c/README.md index 2c06bd0..8e68713 100644 --- a/apps/05-genes-mini-c/README.md +++ b/apps/06-genes-mini-c/README.md @@ -1,6 +1,6 @@ -# 05 - genes-mini (C) +# 06 - genes-mini (C) -[05-genes-mini-rust](../05-genes-mini-rust) written in C. It takes the sequence +[06-genes-mini-rust](../06-genes-mini-rust) written in C. It takes the sequence length from `stat`, reads each scalar into a buffer that grows to fit, and treats only `errno == ENOENT` as no answer. [01-hello-c](../01-hello-c) covers the toolchain. @@ -8,5 +8,5 @@ the toolchain. ## Build and run ```sh -make run APP=05-genes-mini-c +make run APP=06-genes-mini-c ``` diff --git a/apps/05-genes-mini-c/main.c b/apps/06-genes-mini-c/main.c similarity index 74% rename from apps/05-genes-mini-c/main.c rename to apps/06-genes-mini-c/main.c index 678dae8..baed519 100644 --- a/apps/05-genes-mini-c/main.c +++ b/apps/06-genes-mini-c/main.c @@ -64,8 +64,23 @@ static uint64_t coordinate(const char *value) { int main(int argc, char *argv[]) { if (argc < 2) { - printf("[package]\nname = \"genes-mini\"\nversion = \"0.1.0\"\n" - "datasets = [\"" LENS "\"]\n"); + printf("manifest = 1\n\n[app]\nname = \"One Gene\"\nversion = \"0.1.0\"\n\n" + "[reads]\npaths = [\"" LENS "\"]\n\n" + "[listing]\n" + "language = \"en\"\n" + "icon = \"🧬\"\n" + "summary = \"A gene's coordinates and sequence length, through the genes lens.\"\n" + "category = \"developer\"\n" + "license = \"BSD-3-Clause\"\n" + "source = \"https://github.com/dark-bio/examples/tree/main/apps/06-genes-mini-c\"\n" + "keywords = [\"tutorial\", \"genes\", \"sequence\", \"TAS2R38\"]\n" + "description = '''\n" + "Grant `v1/genome/genes/TAS2R38` and read the gene's `chromosome`, `start`, `end`, `strand` and `biotype` as files. The `sequence` file holds exactly end - start + 1 bases, so its size is the gene's length and the gene never has to fit in memory.\n\n" + "An absent value prints as no answer. A missing sequence is an error.\n\n" + "A gene grant also covers your variants inside the gene, even though this app does not read them. Bitter Meter does.\n" + "'''\n\n" + "[listing.purposes]\n" + "\"v1/genome/genes/TAS2R38\" = \"A gene to read coordinates and sequence length from\"\n"); return 0; } diff --git a/apps/06-genes-mini-go/README.md b/apps/06-genes-mini-go/README.md new file mode 100644 index 0000000..0bc697b --- /dev/null +++ b/apps/06-genes-mini-go/README.md @@ -0,0 +1,11 @@ +# 06 - genes-mini (Go) + +[06-genes-mini-rust](../06-genes-mini-rust) written in Go. It takes the sequence +length from `os.Stat` and distinguishes an absent scalar from a failed read +with `errors.Is(err, fs.ErrNotExist)`. + +## Build and run + +```sh +make run APP=06-genes-mini-go +``` diff --git a/apps/05-genes-mini-go/go.mod b/apps/06-genes-mini-go/go.mod similarity index 100% rename from apps/05-genes-mini-go/go.mod rename to apps/06-genes-mini-go/go.mod diff --git a/apps/05-genes-mini-go/main.go b/apps/06-genes-mini-go/main.go similarity index 63% rename from apps/05-genes-mini-go/main.go rename to apps/06-genes-mini-go/main.go index df21cad..5750da8 100644 --- a/apps/05-genes-mini-go/main.go +++ b/apps/06-genes-mini-go/main.go @@ -15,8 +15,23 @@ const lens = "v1/genome/genes/TAS2R38" func main() { if len(os.Args) < 2 { - fmt.Printf("[package]\nname = \"genes-mini\"\nversion = \"0.1.0\"\n"+ - "datasets = [\"%s\"]\n", lens) + fmt.Printf("manifest = 1\n\n[app]\nname = \"One Gene\"\nversion = \"0.1.0\"\n\n"+ + "[reads]\npaths = [\"%s\"]\n\n"+ + "[listing]\n"+ + "language = \"en\"\n"+ + "icon = \"🧬\"\n"+ + "summary = \"A gene's coordinates and sequence length, through the genes lens.\"\n"+ + "category = \"developer\"\n"+ + "license = \"BSD-3-Clause\"\n"+ + "source = \"https://github.com/dark-bio/examples/tree/main/apps/06-genes-mini-go\"\n"+ + "keywords = [\"tutorial\", \"genes\", \"sequence\", \"TAS2R38\"]\n"+ + "description = '''\n"+ + "Grant `v1/genome/genes/TAS2R38` and read the gene's `chromosome`, `start`, `end`, `strand` and `biotype` as files. The `sequence` file holds exactly end - start + 1 bases, so its size is the gene's length and the gene never has to fit in memory.\n\n"+ + "An absent value prints as no answer. A missing sequence is an error.\n\n"+ + "A gene grant also covers your variants inside the gene, even though this app does not read them. Bitter Meter does.\n"+ + "'''\n\n"+ + "[listing.purposes]\n"+ + "\"v1/genome/genes/TAS2R38\" = \"A gene to read coordinates and sequence length from\"\n", lens) return } if err := run(filepath.Join(os.Args[1], lens)); err != nil { diff --git a/apps/05-genes-mini-python/README.md b/apps/06-genes-mini-python/README.md similarity index 63% rename from apps/05-genes-mini-python/README.md rename to apps/06-genes-mini-python/README.md index c7d2963..2f45031 100644 --- a/apps/05-genes-mini-python/README.md +++ b/apps/06-genes-mini-python/README.md @@ -1,6 +1,6 @@ -# 05 - genes-mini (Python) +# 06 - genes-mini (Python) -[05-genes-mini-rust](../05-genes-mini-rust) written in Python. It takes the +[06-genes-mini-rust](../06-genes-mini-rust) written in Python. It takes the sequence length from `os.stat` and reads an absent scalar as no answer when `FileNotFoundError` arrives. [01-hello-python](../01-hello-python) covers what a Python module costs. @@ -8,5 +8,5 @@ Python module costs. ## Build and run ```sh -make run APP=05-genes-mini-python +make run APP=06-genes-mini-python ``` diff --git a/apps/05-genes-mini-python/main.py b/apps/06-genes-mini-python/main.py similarity index 66% rename from apps/05-genes-mini-python/main.py rename to apps/06-genes-mini-python/main.py index 9ae006f..cca175a 100644 --- a/apps/05-genes-mini-python/main.py +++ b/apps/06-genes-mini-python/main.py @@ -31,8 +31,34 @@ def coordinate(value): def main(): if len(sys.argv) < 2: - print('[package]\nname = "genes-mini"\nversion = "0.1.0"\n' - f'datasets = ["{LENS}"]') + print(f"""manifest = 1 + +[app] +name = "One Gene" +version = "0.1.0" + +[reads] +paths = ["{LENS}"] + +[listing] +language = "en" +icon = "🧬" +summary = "A gene's coordinates and sequence length, through the genes lens." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/06-genes-mini-python" +keywords = ["tutorial", "genes", "sequence", "TAS2R38"] +description = ''' +Grant `v1/genome/genes/TAS2R38` and read the gene's `chromosome`, `start`, `end`, `strand` and `biotype` as files. The `sequence` file holds exactly end - start + 1 bases, so its size is the gene's length and the gene never has to fit in memory. + +An absent value prints as no answer. A missing sequence is an error. + +A gene grant also covers your variants inside the gene, even though this app does not read them. Bitter Meter does. +''' + +[listing.purposes] +"v1/genome/genes/TAS2R38" = "A gene to read coordinates and sequence length from" +""", end="") return base = f"{sys.argv[1]}/{LENS}" diff --git a/apps/05-genes-mini-rust/Cargo.lock b/apps/06-genes-mini-rust/Cargo.lock similarity index 100% rename from apps/05-genes-mini-rust/Cargo.lock rename to apps/06-genes-mini-rust/Cargo.lock diff --git a/apps/05-genes-mini-rust/Cargo.toml b/apps/06-genes-mini-rust/Cargo.toml similarity index 100% rename from apps/05-genes-mini-rust/Cargo.toml rename to apps/06-genes-mini-rust/Cargo.toml diff --git a/apps/05-genes-mini-rust/README.md b/apps/06-genes-mini-rust/README.md similarity index 67% rename from apps/05-genes-mini-rust/README.md rename to apps/06-genes-mini-rust/README.md index 9c96a90..7639091 100644 --- a/apps/05-genes-mini-rust/README.md +++ b/apps/06-genes-mini-rust/README.md @@ -1,4 +1,4 @@ -# 05 - genes-mini (Rust) +# 06 - genes-mini (Rust) The app grants one gene, `v1/genome/genes/TAS2R38`, and prints its chromosome, start, end, strand, biotype and sequence length. The length comes from the size @@ -7,13 +7,13 @@ to fit in memory. An absent value prints as no answer, while a missing sequence is an error. The gene grant also covers the owner's `changes` in it, even though this mini -doesn't read them. [05-bitter-meter](../05-bitter-meter) does. +doesn't read them. [06-bitter-meter](../06-bitter-meter) does. -The same mini exists in [Go](../05-genes-mini-go), [C](../05-genes-mini-c) and -[Python](../05-genes-mini-python). +The same mini exists in [Go](../06-genes-mini-go), [C](../06-genes-mini-c) and +[Python](../06-genes-mini-python). ## Build and run ```sh -make run APP=05-genes-mini-rust +make run APP=06-genes-mini-rust ``` diff --git a/apps/05-genes-mini-rust/src/main.rs b/apps/06-genes-mini-rust/src/main.rs similarity index 70% rename from apps/05-genes-mini-rust/src/main.rs rename to apps/06-genes-mini-rust/src/main.rs index c1ea102..c6ad73c 100644 --- a/apps/05-genes-mini-rust/src/main.rs +++ b/apps/06-genes-mini-rust/src/main.rs @@ -1,7 +1,7 @@ //! One gene through the genes/ lens (the mini version). //! //! Reads a gene's scalar metadata leaves and the length of its reference -//! sequence, in a handful of lines. The full report is ../05-bitter-meter. +//! sequence, in a handful of lines. The full report is ../06-bitter-meter. use std::fs; use std::io; @@ -13,10 +13,34 @@ const LENS: &str = "v1/genome/genes/TAS2R38"; fn main() { let Some(dir) = std::env::args().nth(1) else { print!( - "[package]\n\ - name = \"genes-mini\"\n\ - version = \"0.1.0\"\n\ - datasets = [\"v1/genome/genes/TAS2R38\"]\n" + r#"manifest = 1 + +[app] +name = "One Gene" +version = "0.1.0" + +[reads] +paths = ["v1/genome/genes/TAS2R38"] + +[listing] +language = "en" +icon = "🧬" +summary = "A gene's coordinates and sequence length, through the genes lens." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/06-genes-mini-rust" +keywords = ["tutorial", "genes", "sequence", "TAS2R38"] +description = ''' +Grant `v1/genome/genes/TAS2R38` and read the gene's `chromosome`, `start`, `end`, `strand` and `biotype` as files. The `sequence` file holds exactly end - start + 1 bases, so its size is the gene's length and the gene never has to fit in memory. + +An absent value prints as no answer. A missing sequence is an error. + +A gene grant also covers your variants inside the gene, even though this app does not read them. Bitter Meter does. +''' + +[listing.purposes] +"v1/genome/genes/TAS2R38" = "A gene to read coordinates and sequence length from" +"# ); return; }; diff --git a/apps/06-regions-mini-python/main.py b/apps/06-regions-mini-python/main.py deleted file mode 100644 index 915ef5a..0000000 --- a/apps/06-regions-mini-python/main.py +++ /dev/null @@ -1,33 +0,0 @@ -"""One interval's reference length and GC content through the regions/ lens.""" -import sys - -REGION = "chrM:1-16569" -LENS = "v1/genome/regions/chrM/1-16569" - - -def main(): - if len(sys.argv) < 2: - print('[package]\nname = "regions-mini"\nversion = "0.1.0"\n' - f'datasets = ["{LENS}"]') - return - - length = gc = 0 - # Forward-strand sequence has no newline; lowercase bases are soft-masked. - with open(f"{sys.argv[1]}/{LENS}/sequence", "rb") as sequence: - while chunk := sequence.read(8192): - length += len(chunk) - gc += sum(chunk.count(base) for base in (b"G", b"g", b"C", b"c")) - if length != 16569: - raise ValueError("Region sequence length does not match its span") - - print(f"## Region {REGION}\n") - print(f"- Reference length: {length} bp") - print(f"- GC content: {100.0 * gc / length:.1f}%") - - -if __name__ == "__main__": - try: - main() - except (OSError, ValueError) as error: - print(error, file=sys.stderr) - sys.exit(1) diff --git a/apps/06-powerhouse-of-the-cell/Cargo.lock b/apps/07-powerhouse-of-the-cell/Cargo.lock similarity index 100% rename from apps/06-powerhouse-of-the-cell/Cargo.lock rename to apps/07-powerhouse-of-the-cell/Cargo.lock diff --git a/apps/06-powerhouse-of-the-cell/Cargo.toml b/apps/07-powerhouse-of-the-cell/Cargo.toml similarity index 100% rename from apps/06-powerhouse-of-the-cell/Cargo.toml rename to apps/07-powerhouse-of-the-cell/Cargo.toml diff --git a/apps/06-powerhouse-of-the-cell/README.md b/apps/07-powerhouse-of-the-cell/README.md similarity index 73% rename from apps/06-powerhouse-of-the-cell/README.md rename to apps/07-powerhouse-of-the-cell/README.md index b8d6bdf..0591e72 100644 --- a/apps/06-powerhouse-of-the-cell/README.md +++ b/apps/07-powerhouse-of-the-cell/README.md @@ -1,8 +1,8 @@ -# 06 - powerhouse-of-the-cell +# 07 - powerhouse-of-the-cell A full report on the mitochondrial genome, read as one interval. It streams the `sequence` for GC content and density windows, and lists the owner's `changes` -with a text map of where they fall. Like [05-bitter-meter](../05-bitter-meter), +with a text map of where they fall. Like [06-bitter-meter](../06-bitter-meter), it counts listed positions without leaves and splits genotypes around symbolic and breakend alleles. It isn't a haplogroup caller, an ancestry test or a medical report. @@ -10,5 +10,5 @@ medical report. ## Build and run ```sh -make run APP=06-powerhouse-of-the-cell +make run APP=07-powerhouse-of-the-cell ``` diff --git a/apps/06-powerhouse-of-the-cell/src/main.rs b/apps/07-powerhouse-of-the-cell/src/main.rs similarity index 93% rename from apps/06-powerhouse-of-the-cell/src/main.rs rename to apps/07-powerhouse-of-the-cell/src/main.rs index 990e694..6354bc1 100644 --- a/apps/06-powerhouse-of-the-cell/src/main.rs +++ b/apps/07-powerhouse-of-the-cell/src/main.rs @@ -24,10 +24,35 @@ const MAX_CHANGE_ROWS: usize = 28; fn main() { let Some(dir) = std::env::args().nth(1) else { print!( - "[package]\n\ - name = \"powerhouse-of-the-cell\"\n\ - version = \"0.2.0\"\n\ - datasets = [\"v1/genome/regions/chrM/1-16569\", \"v1/genome/reference\"]\n" + r#"manifest = 1 + +[app] +name = "Powerhouse of the Cell" +version = "0.2.0" + +[reads] +paths = ["v1/genome/regions/chrM/1-16569", "v1/genome/reference"] + +[listing] +language = "en" +icon = "⚡" +summary = "Your whole mitochondrial genome on one page, with your changes mapped." +category = "tools" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/07-powerhouse-of-the-cell" +keywords = ["mitochondria", "chrM", "GC content", "variants"] +description = ''' +The mitochondrial chromosome is 16,569 bases long, small enough to read as a single interval. This app grants `chrM:1-16569` through the regions lens, streams the reference sequence for GC content and density windows, and lists your non reference changes with a text map of where they fall along the sequence. + +It counts transitions and transversions, flags indels and mixed allele calls, and reports positions without an answer. It is not a haplogroup caller, an ancestry test or a medical report. + +An interval grant covers your variants inside it, so approving this app shares every mitochondrial change in your call file with it. +''' + +[listing.purposes] +"v1/genome/regions/chrM/1-16569" = "Your whole mitochondrial genome, its sequence and your changes in it" +"v1/genome/reference" = "The reference build, to name positions" +"# ); return; }; diff --git a/apps/06-regions-mini-c/README.md b/apps/07-regions-mini-c/README.md similarity index 59% rename from apps/06-regions-mini-c/README.md rename to apps/07-regions-mini-c/README.md index d3779ee..0d5ea3f 100644 --- a/apps/06-regions-mini-c/README.md +++ b/apps/07-regions-mini-c/README.md @@ -1,11 +1,11 @@ -# 06 - regions-mini (C) +# 07 - regions-mini (C) -[06-regions-mini-rust](../06-regions-mini-rust) written in C. It reads the +[07-regions-mini-rust](../07-regions-mini-rust) written in C. It reads the sequence in 8 KiB chunks with `fread`, so the interval never has to fit in memory. [01-hello-c](../01-hello-c) covers the toolchain. ## Build and run ```sh -make run APP=06-regions-mini-c +make run APP=07-regions-mini-c ``` diff --git a/apps/06-regions-mini-c/main.c b/apps/07-regions-mini-c/main.c similarity index 58% rename from apps/06-regions-mini-c/main.c rename to apps/07-regions-mini-c/main.c index 0ebca08..532f557 100644 --- a/apps/06-regions-mini-c/main.c +++ b/apps/07-regions-mini-c/main.c @@ -10,8 +10,23 @@ int main(int argc, char *argv[]) { if (argc < 2) { - printf("[package]\nname = \"regions-mini\"\nversion = \"0.1.0\"\n" - "datasets = [\"" LENS "\"]\n"); + printf("manifest = 1\n\n[app]\nname = \"One Region\"\nversion = \"0.1.0\"\n\n" + "[reads]\npaths = [\"" LENS "\"]\n\n" + "[listing]\n" + "language = \"en\"\n" + "icon = \"📏\"\n" + "summary = \"Stream an interval's sequence without ever loading it whole.\"\n" + "category = \"developer\"\n" + "license = \"BSD-3-Clause\"\n" + "source = \"https://github.com/dark-bio/examples/tree/main/apps/07-regions-mini-c\"\n" + "keywords = [\"tutorial\", \"regions\", \"streaming\", \"mitochondria\"]\n" + "description = '''\n" + "Grant any interval as `v1/genome/regions//-`. This app takes the whole mitochondrial genome, `chrM:1-16569`, and streams its `sequence` in 8 KiB chunks to report its length and GC content. Memory use does not grow with the interval.\n\n" + "Repeats are soft masked in lowercase, so count both cases. A failed read stops the app.\n\n" + "An interval grant covers your variants inside it, which Powerhouse of the Cell lists.\n" + "'''\n\n" + "[listing.purposes]\n" + "\"v1/genome/regions/chrM/1-16569\" = \"The mitochondrial genome, to stream its sequence\"\n"); return 0; } size_t size = strlen(argv[1]) + sizeof(LENS) + sizeof("/sequence") + 1; diff --git a/apps/06-regions-mini-go/README.md b/apps/07-regions-mini-go/README.md similarity index 52% rename from apps/06-regions-mini-go/README.md rename to apps/07-regions-mini-go/README.md index 333c9a8..a6433ba 100644 --- a/apps/06-regions-mini-go/README.md +++ b/apps/07-regions-mini-go/README.md @@ -1,11 +1,11 @@ -# 06 - regions-mini (Go) +# 07 - regions-mini (Go) -[06-regions-mini-rust](../06-regions-mini-rust) written in Go. It reads the +[07-regions-mini-rust](../07-regions-mini-rust) written in Go. It reads the sequence in 8 KiB chunks with `os.File.Read`, so the interval never has to fit in memory. ## Build and run ```sh -make run APP=06-regions-mini-go +make run APP=07-regions-mini-go ``` diff --git a/apps/06-regions-mini-go/go.mod b/apps/07-regions-mini-go/go.mod similarity index 100% rename from apps/06-regions-mini-go/go.mod rename to apps/07-regions-mini-go/go.mod diff --git a/apps/06-regions-mini-go/main.go b/apps/07-regions-mini-go/main.go similarity index 52% rename from apps/06-regions-mini-go/main.go rename to apps/07-regions-mini-go/main.go index c50186f..b2f2ca8 100644 --- a/apps/06-regions-mini-go/main.go +++ b/apps/07-regions-mini-go/main.go @@ -14,8 +14,23 @@ const lens = "v1/genome/regions/chrM/1-16569" func main() { if len(os.Args) < 2 { - fmt.Printf("[package]\nname = \"regions-mini\"\nversion = \"0.1.0\"\n"+ - "datasets = [\"%s\"]\n", lens) + fmt.Printf("manifest = 1\n\n[app]\nname = \"One Region\"\nversion = \"0.1.0\"\n\n"+ + "[reads]\npaths = [\"%s\"]\n\n"+ + "[listing]\n"+ + "language = \"en\"\n"+ + "icon = \"📏\"\n"+ + "summary = \"Stream an interval's sequence without ever loading it whole.\"\n"+ + "category = \"developer\"\n"+ + "license = \"BSD-3-Clause\"\n"+ + "source = \"https://github.com/dark-bio/examples/tree/main/apps/07-regions-mini-go\"\n"+ + "keywords = [\"tutorial\", \"regions\", \"streaming\", \"mitochondria\"]\n"+ + "description = '''\n"+ + "Grant any interval as `v1/genome/regions//-`. This app takes the whole mitochondrial genome, `chrM:1-16569`, and streams its `sequence` in 8 KiB chunks to report its length and GC content. Memory use does not grow with the interval.\n\n"+ + "Repeats are soft masked in lowercase, so count both cases. A failed read stops the app.\n\n"+ + "An interval grant covers your variants inside it, which Powerhouse of the Cell lists.\n"+ + "'''\n\n"+ + "[listing.purposes]\n"+ + "\"v1/genome/regions/chrM/1-16569\" = \"The mitochondrial genome, to stream its sequence\"\n", lens) return } if err := run(os.Args[1]); err != nil { diff --git a/apps/06-regions-mini-python/README.md b/apps/07-regions-mini-python/README.md similarity index 60% rename from apps/06-regions-mini-python/README.md rename to apps/07-regions-mini-python/README.md index 60cade2..9a651e9 100644 --- a/apps/06-regions-mini-python/README.md +++ b/apps/07-regions-mini-python/README.md @@ -1,11 +1,11 @@ -# 06 - regions-mini (Python) +# 07 - regions-mini (Python) -[06-regions-mini-rust](../06-regions-mini-rust) written in Python. It reads the +[07-regions-mini-rust](../07-regions-mini-rust) written in Python. It reads the sequence in 8 KiB chunks of bytes, so the interval never has to fit in memory. [01-hello-python](../01-hello-python) covers what a Python module costs. ## Build and run ```sh -make run APP=06-regions-mini-python +make run APP=07-regions-mini-python ``` diff --git a/apps/07-regions-mini-python/main.py b/apps/07-regions-mini-python/main.py new file mode 100644 index 0000000..685455c --- /dev/null +++ b/apps/07-regions-mini-python/main.py @@ -0,0 +1,59 @@ +"""One interval's reference length and GC content through the regions/ lens.""" +import sys + +REGION = "chrM:1-16569" +LENS = "v1/genome/regions/chrM/1-16569" + + +def main(): + if len(sys.argv) < 2: + print(f"""manifest = 1 + +[app] +name = "One Region" +version = "0.1.0" + +[reads] +paths = ["{LENS}"] + +[listing] +language = "en" +icon = "📏" +summary = "Stream an interval's sequence without ever loading it whole." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/07-regions-mini-python" +keywords = ["tutorial", "regions", "streaming", "mitochondria"] +description = ''' +Grant any interval as `v1/genome/regions//-`. This app takes the whole mitochondrial genome, `chrM:1-16569`, and streams its `sequence` in 8 KiB chunks to report its length and GC content. Memory use does not grow with the interval. + +Repeats are soft masked in lowercase, so count both cases. A failed read stops the app. + +An interval grant covers your variants inside it, which Powerhouse of the Cell lists. +''' + +[listing.purposes] +"v1/genome/regions/chrM/1-16569" = "The mitochondrial genome, to stream its sequence" +""", end="") + return + + length = gc = 0 + # Forward-strand sequence has no newline; lowercase bases are soft-masked. + with open(f"{sys.argv[1]}/{LENS}/sequence", "rb") as sequence: + while chunk := sequence.read(8192): + length += len(chunk) + gc += sum(chunk.count(base) for base in (b"G", b"g", b"C", b"c")) + if length != 16569: + raise ValueError("Region sequence length does not match its span") + + print(f"## Region {REGION}\n") + print(f"- Reference length: {length} bp") + print(f"- GC content: {100.0 * gc / length:.1f}%") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError) as error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/apps/06-regions-mini-rust/Cargo.lock b/apps/07-regions-mini-rust/Cargo.lock similarity index 100% rename from apps/06-regions-mini-rust/Cargo.lock rename to apps/07-regions-mini-rust/Cargo.lock diff --git a/apps/06-regions-mini-rust/Cargo.toml b/apps/07-regions-mini-rust/Cargo.toml similarity index 100% rename from apps/06-regions-mini-rust/Cargo.toml rename to apps/07-regions-mini-rust/Cargo.toml diff --git a/apps/06-regions-mini-rust/README.md b/apps/07-regions-mini-rust/README.md similarity index 60% rename from apps/06-regions-mini-rust/README.md rename to apps/07-regions-mini-rust/README.md index 3a3354d..a502107 100644 --- a/apps/06-regions-mini-rust/README.md +++ b/apps/07-regions-mini-rust/README.md @@ -1,4 +1,4 @@ -# 06 - regions-mini (Rust) +# 07 - regions-mini (Rust) The app grants one interval, the whole mitochondrial genome at `v1/genome/regions/chrM/1-16569`, and streams its `sequence` in chunks to report @@ -6,13 +6,13 @@ its length and GC content, so memory use doesn't grow with the interval. A missing sequence or a failed read stops the app. The interval grant also covers the owner's `changes`, which -[06-powerhouse-of-the-cell](../06-powerhouse-of-the-cell) lists. +[07-powerhouse-of-the-cell](../07-powerhouse-of-the-cell) lists. -The same mini exists in [Go](../06-regions-mini-go), -[C](../06-regions-mini-c) and [Python](../06-regions-mini-python). +The same mini exists in [Go](../07-regions-mini-go), +[C](../07-regions-mini-c) and [Python](../07-regions-mini-python). ## Build and run ```sh -make run APP=06-regions-mini-rust +make run APP=07-regions-mini-rust ``` diff --git a/apps/06-regions-mini-rust/src/main.rs b/apps/07-regions-mini-rust/src/main.rs similarity index 59% rename from apps/06-regions-mini-rust/src/main.rs rename to apps/07-regions-mini-rust/src/main.rs index 10982d3..7d08db0 100644 --- a/apps/06-regions-mini-rust/src/main.rs +++ b/apps/07-regions-mini-rust/src/main.rs @@ -1,7 +1,7 @@ //! One interval through the regions/ lens (the mini version). //! //! Reads a region's reference sequence and reports its length and GC content. -//! The full report is ../06-powerhouse-of-the-cell. +//! The full report is ../07-powerhouse-of-the-cell. use std::fs::File; use std::io::Read; @@ -13,10 +13,34 @@ const LENS: &str = "v1/genome/regions/chrM/1-16569"; fn main() { let Some(dir) = std::env::args().nth(1) else { print!( - "[package]\n\ - name = \"regions-mini\"\n\ - version = \"0.1.0\"\n\ - datasets = [\"v1/genome/regions/chrM/1-16569\"]\n" + r#"manifest = 1 + +[app] +name = "One Region" +version = "0.1.0" + +[reads] +paths = ["v1/genome/regions/chrM/1-16569"] + +[listing] +language = "en" +icon = "📏" +summary = "Stream an interval's sequence without ever loading it whole." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/07-regions-mini-rust" +keywords = ["tutorial", "regions", "streaming", "mitochondria"] +description = ''' +Grant any interval as `v1/genome/regions//-`. This app takes the whole mitochondrial genome, `chrM:1-16569`, and streams its `sequence` in 8 KiB chunks to report its length and GC content. Memory use does not grow with the interval. + +Repeats are soft masked in lowercase, so count both cases. A failed read stops the app. + +An interval grant covers your variants inside it, which Powerhouse of the Cell lists. +''' + +[listing.purposes] +"v1/genome/regions/chrM/1-16569" = "The mitochondrial genome, to stream its sequence" +"# ); return; }; diff --git a/apps/07-vcf-mini-python/main.py b/apps/07-vcf-mini-python/main.py deleted file mode 100644 index 731ec5e..0000000 --- a/apps/07-vcf-mini-python/main.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Count headers and records in the decompressed VCF view.""" -import sys - -LENS = "v1/genome/snp-indel" - - -def main(): - if len(sys.argv) < 2: - print('[package]\nname = "vcf-mini"\nversion = "0.1.0"\n' - f'datasets = ["{LENS}"]') - return - - headers = records = 0 - first = None - # Keep one line at a time so a whole-genome call file need not fit in memory. - with open(f"{sys.argv[1]}/{LENS}/vcf", encoding="utf-8", newline="\n") as file: - for line in file: - if line.endswith("\n"): - line = line[:-1].removesuffix("\r") - if line.startswith("#"): - headers += 1 - elif line: - records += 1 - if first is None: - first = " ".join(line.split("\t")[:5]) - - print("## Variant file\n") - print(f"- Header lines: {headers}") - print(f"- Variant records: {records}") - if first is not None: - print(f"- First record: `{first}`") - - -if __name__ == "__main__": - try: - main() - except (OSError, UnicodeError) as error: - print(error, file=sys.stderr) - sys.exit(1) diff --git a/apps/07-vcf-mini-c/README.md b/apps/08-vcf-mini-c/README.md similarity index 72% rename from apps/07-vcf-mini-c/README.md rename to apps/08-vcf-mini-c/README.md index 8a5a4c0..7d0804e 100644 --- a/apps/07-vcf-mini-c/README.md +++ b/apps/08-vcf-mini-c/README.md @@ -1,6 +1,6 @@ -# 07 - vcf-mini (C) +# 08 - vcf-mini (C) -[07-vcf-mini-rust](../07-vcf-mini-rust) written in C. It reads the call file +[08-vcf-mini-rust](../08-vcf-mini-rust) written in C. It reads the call file with `getline`, which accepts long records while keeping only one line in memory, over a 64 KiB `setvbuf` buffer. The default buffer is small enough to make a whole-genome scan an order of magnitude slower. @@ -9,5 +9,5 @@ make a whole-genome scan an order of magnitude slower. ## Build and run ```sh -make run APP=07-vcf-mini-c +make run APP=08-vcf-mini-c ``` diff --git a/apps/07-vcf-mini-c/main.c b/apps/08-vcf-mini-c/main.c similarity index 64% rename from apps/07-vcf-mini-c/main.c rename to apps/08-vcf-mini-c/main.c index ebce858..c2265b3 100644 --- a/apps/07-vcf-mini-c/main.c +++ b/apps/08-vcf-mini-c/main.c @@ -9,8 +9,23 @@ int main(int argc, char *argv[]) { if (argc < 2) { - printf("[package]\nname = \"vcf-mini\"\nversion = \"0.1.0\"\n" - "datasets = [\"" LENS "\"]\n"); + printf("manifest = 1\n\n[app]\nname = \"The Call File\"\nversion = \"0.1.0\"\n\n" + "[reads]\npaths = [\"" LENS "\"]\n\n" + "[listing]\n" + "language = \"en\"\n" + "icon = \"📄\"\n" + "summary = \"Count the records of a whole genome one line at a time.\"\n" + "category = \"developer\"\n" + "license = \"BSD-3-Clause\"\n" + "source = \"https://github.com/dark-bio/examples/tree/main/apps/08-vcf-mini-c\"\n" + "keywords = [\"tutorial\", \"VCF\", \"call file\", \"streaming\"]\n" + "description = '''\n" + "Grant `v1/genome/snp-indel` and open its `vcf`, the decompressed call file. A whole genome file runs to gigabytes, far past an app's 128 MiB of memory, so this app streams it one line at a time and counts headers and records.\n\n" + "Every buffer refill leaves the sandbox, which adds up at that scale. The Rust, Go and C versions read through a 64 KiB buffer, and the Rust and C ones also reuse one line instead of allocating one per record.\n\n" + "This is the broadest grant there is. VCF Roll Call parses the same file with a library.\n" + "'''\n\n" + "[listing.purposes]\n" + "\"v1/genome/snp-indel\" = \"Your whole call file, to count its records\"\n"); return 0; } size_t size = strlen(argv[1]) + sizeof(LENS) + sizeof("/vcf") + 1; diff --git a/apps/07-vcf-mini-go/README.md b/apps/08-vcf-mini-go/README.md similarity index 62% rename from apps/07-vcf-mini-go/README.md rename to apps/08-vcf-mini-go/README.md index 9ad20bf..603131c 100644 --- a/apps/07-vcf-mini-go/README.md +++ b/apps/08-vcf-mini-go/README.md @@ -1,11 +1,11 @@ -# 07 - vcf-mini (Go) +# 08 - vcf-mini (Go) -[07-vcf-mini-rust](../07-vcf-mini-rust) written in Go. It reads the call file +[08-vcf-mini-rust](../08-vcf-mini-rust) written in Go. It reads the call file with `bufio.Reader.ReadString`, which accepts long records while keeping only one line in memory, over a 64 KiB buffer from `bufio.NewReaderSize`. ## Build and run ```sh -make run APP=07-vcf-mini-go +make run APP=08-vcf-mini-go ``` diff --git a/apps/07-vcf-mini-go/go.mod b/apps/08-vcf-mini-go/go.mod similarity index 100% rename from apps/07-vcf-mini-go/go.mod rename to apps/08-vcf-mini-go/go.mod diff --git a/apps/07-vcf-mini-go/main.go b/apps/08-vcf-mini-go/main.go similarity index 55% rename from apps/07-vcf-mini-go/main.go rename to apps/08-vcf-mini-go/main.go index dab47aa..9e6e8a5 100644 --- a/apps/07-vcf-mini-go/main.go +++ b/apps/08-vcf-mini-go/main.go @@ -14,8 +14,23 @@ const lens = "v1/genome/snp-indel" func main() { if len(os.Args) < 2 { - fmt.Printf("[package]\nname = \"vcf-mini\"\nversion = \"0.1.0\"\n"+ - "datasets = [\"%s\"]\n", lens) + fmt.Printf("manifest = 1\n\n[app]\nname = \"The Call File\"\nversion = \"0.1.0\"\n\n"+ + "[reads]\npaths = [\"%s\"]\n\n"+ + "[listing]\n"+ + "language = \"en\"\n"+ + "icon = \"📄\"\n"+ + "summary = \"Count the records of a whole genome one line at a time.\"\n"+ + "category = \"developer\"\n"+ + "license = \"BSD-3-Clause\"\n"+ + "source = \"https://github.com/dark-bio/examples/tree/main/apps/08-vcf-mini-go\"\n"+ + "keywords = [\"tutorial\", \"VCF\", \"call file\", \"streaming\"]\n"+ + "description = '''\n"+ + "Grant `v1/genome/snp-indel` and open its `vcf`, the decompressed call file. A whole genome file runs to gigabytes, far past an app's 128 MiB of memory, so this app streams it one line at a time and counts headers and records.\n\n"+ + "Every buffer refill leaves the sandbox, which adds up at that scale. The Rust, Go and C versions read through a 64 KiB buffer, and the Rust and C ones also reuse one line instead of allocating one per record.\n\n"+ + "This is the broadest grant there is. VCF Roll Call parses the same file with a library.\n"+ + "'''\n\n"+ + "[listing.purposes]\n"+ + "\"v1/genome/snp-indel\" = \"Your whole call file, to count its records\"\n", lens) return } if err := run(os.Args[1]); err != nil { diff --git a/apps/07-vcf-mini-python/README.md b/apps/08-vcf-mini-python/README.md similarity index 62% rename from apps/07-vcf-mini-python/README.md rename to apps/08-vcf-mini-python/README.md index 638328a..39b8ed8 100644 --- a/apps/07-vcf-mini-python/README.md +++ b/apps/08-vcf-mini-python/README.md @@ -1,11 +1,11 @@ -# 07 - vcf-mini (Python) +# 08 - vcf-mini (Python) -[07-vcf-mini-rust](../07-vcf-mini-rust) written in Python. Iterating the open +[08-vcf-mini-rust](../08-vcf-mini-rust) written in Python. Iterating the open file yields one line at a time, so a whole-genome call file never has to fit in memory. [01-hello-python](../01-hello-python) covers what a Python module costs. ## Build and run ```sh -make run APP=07-vcf-mini-python +make run APP=08-vcf-mini-python ``` diff --git a/apps/08-vcf-mini-python/main.py b/apps/08-vcf-mini-python/main.py new file mode 100644 index 0000000..91b55b4 --- /dev/null +++ b/apps/08-vcf-mini-python/main.py @@ -0,0 +1,65 @@ +"""Count headers and records in the decompressed VCF view.""" +import sys + +LENS = "v1/genome/snp-indel" + + +def main(): + if len(sys.argv) < 2: + print(f"""manifest = 1 + +[app] +name = "The Call File" +version = "0.1.0" + +[reads] +paths = ["{LENS}"] + +[listing] +language = "en" +icon = "📄" +summary = "Count the records of a whole genome one line at a time." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/08-vcf-mini-python" +keywords = ["tutorial", "VCF", "call file", "streaming"] +description = ''' +Grant `v1/genome/snp-indel` and open its `vcf`, the decompressed call file. A whole genome file runs to gigabytes, far past an app's 128 MiB of memory, so this app streams it one line at a time and counts headers and records. + +Every buffer refill leaves the sandbox, which adds up at that scale. The Rust, Go and C versions read through a 64 KiB buffer, and the Rust and C ones also reuse one line instead of allocating one per record. + +This is the broadest grant there is. VCF Roll Call parses the same file with a library. +''' + +[listing.purposes] +"v1/genome/snp-indel" = "Your whole call file, to count its records" +""", end="") + return + + headers = records = 0 + first = None + # Keep one line at a time so a whole-genome call file need not fit in memory. + with open(f"{sys.argv[1]}/{LENS}/vcf", encoding="utf-8", newline="\n") as file: + for line in file: + if line.endswith("\n"): + line = line[:-1].removesuffix("\r") + if line.startswith("#"): + headers += 1 + elif line: + records += 1 + if first is None: + first = " ".join(line.split("\t")[:5]) + + print("## Variant file\n") + print(f"- Header lines: {headers}") + print(f"- Variant records: {records}") + if first is not None: + print(f"- First record: `{first}`") + + +if __name__ == "__main__": + try: + main() + except (OSError, UnicodeError) as error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/apps/07-vcf-mini-rust/Cargo.lock b/apps/08-vcf-mini-rust/Cargo.lock similarity index 100% rename from apps/07-vcf-mini-rust/Cargo.lock rename to apps/08-vcf-mini-rust/Cargo.lock diff --git a/apps/07-vcf-mini-rust/Cargo.toml b/apps/08-vcf-mini-rust/Cargo.toml similarity index 100% rename from apps/07-vcf-mini-rust/Cargo.toml rename to apps/08-vcf-mini-rust/Cargo.toml diff --git a/apps/07-vcf-mini-rust/README.md b/apps/08-vcf-mini-rust/README.md similarity index 63% rename from apps/07-vcf-mini-rust/README.md rename to apps/08-vcf-mini-rust/README.md index 3d69f23..b4ba358 100644 --- a/apps/07-vcf-mini-rust/README.md +++ b/apps/08-vcf-mini-rust/README.md @@ -1,20 +1,20 @@ -# 07 - vcf-mini (Rust) +# 08 - vcf-mini (Rust) The app grants the whole call file, `v1/genome/snp-indel`, and streams `vcf` one line at a time to count its header lines and records. A whole-genome call file -runs to gigabytes, far past an app's 100 MiB of memory, so it's never loaded +runs to gigabytes, far past an app's 128 MiB of memory, so it's never loaded whole. A failed read stops the app instead of reporting zero records. -[07-vcf-roll-call](../07-vcf-roll-call) parses the same file with a library. +[08-vcf-roll-call](../08-vcf-roll-call) parses the same file with a library. It reads through a 64 KiB buffer into one reused line. Both matter over millions of records, since every refill leaves the sandbox and a fresh line per record would allocate for each one. -The same mini exists in [Go](../07-vcf-mini-go), [C](../07-vcf-mini-c) and -[Python](../07-vcf-mini-python). +The same mini exists in [Go](../08-vcf-mini-go), [C](../08-vcf-mini-c) and +[Python](../08-vcf-mini-python). ## Build and run ```sh -make run APP=07-vcf-mini-rust +make run APP=08-vcf-mini-rust ``` diff --git a/apps/07-vcf-mini-rust/src/main.rs b/apps/08-vcf-mini-rust/src/main.rs similarity index 63% rename from apps/07-vcf-mini-rust/src/main.rs rename to apps/08-vcf-mini-rust/src/main.rs index b9e4369..f57dab6 100644 --- a/apps/07-vcf-mini-rust/src/main.rs +++ b/apps/08-vcf-mini-rust/src/main.rs @@ -2,7 +2,7 @@ //! //! Opens the decompressed `vcf` view and counts records, with no genomics //! library at all: the view is plain text, so a line scan is enough. The full -//! scan with noodles-vcf is ../07-vcf-roll-call. +//! scan with noodles-vcf is ../08-vcf-roll-call. use std::fs::File; use std::io::{BufRead, BufReader}; @@ -13,10 +13,34 @@ const LENS: &str = "v1/genome/snp-indel"; fn main() { let Some(dir) = std::env::args().nth(1) else { print!( - "[package]\n\ - name = \"vcf-mini\"\n\ - version = \"0.1.0\"\n\ - datasets = [\"v1/genome/snp-indel\"]\n" + r#"manifest = 1 + +[app] +name = "The Call File" +version = "0.1.0" + +[reads] +paths = ["v1/genome/snp-indel"] + +[listing] +language = "en" +icon = "📄" +summary = "Count the records of a whole genome one line at a time." +category = "developer" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/08-vcf-mini-rust" +keywords = ["tutorial", "VCF", "call file", "streaming"] +description = ''' +Grant `v1/genome/snp-indel` and open its `vcf`, the decompressed call file. A whole genome file runs to gigabytes, far past an app's 128 MiB of memory, so this app streams it one line at a time and counts headers and records. + +Every buffer refill leaves the sandbox, which adds up at that scale. The Rust, Go and C versions read through a 64 KiB buffer, and the Rust and C ones also reuse one line instead of allocating one per record. + +This is the broadest grant there is. VCF Roll Call parses the same file with a library. +''' + +[listing.purposes] +"v1/genome/snp-indel" = "Your whole call file, to count its records" +"# ); return; }; diff --git a/apps/07-vcf-roll-call/Cargo.lock b/apps/08-vcf-roll-call/Cargo.lock similarity index 100% rename from apps/07-vcf-roll-call/Cargo.lock rename to apps/08-vcf-roll-call/Cargo.lock diff --git a/apps/07-vcf-roll-call/Cargo.toml b/apps/08-vcf-roll-call/Cargo.toml similarity index 100% rename from apps/07-vcf-roll-call/Cargo.toml rename to apps/08-vcf-roll-call/Cargo.toml diff --git a/apps/07-vcf-roll-call/README.md b/apps/08-vcf-roll-call/README.md similarity index 89% rename from apps/07-vcf-roll-call/README.md rename to apps/08-vcf-roll-call/README.md index 79cad52..6aca21c 100644 --- a/apps/07-vcf-roll-call/README.md +++ b/apps/08-vcf-roll-call/README.md @@ -1,4 +1,4 @@ -# 07 - vcf-roll-call +# 08 - vcf-roll-call The app streams the whole call file with [`noodles-vcf`](https://crates.io/crates/noodles-vcf) and reports on its records, @@ -10,5 +10,5 @@ much of a genome was sequenced. ## Build and run ```sh -make run APP=07-vcf-roll-call +make run APP=08-vcf-roll-call ``` diff --git a/apps/07-vcf-roll-call/src/main.rs b/apps/08-vcf-roll-call/src/main.rs similarity index 96% rename from apps/07-vcf-roll-call/src/main.rs rename to apps/08-vcf-roll-call/src/main.rs index 445da98..567e526 100644 --- a/apps/07-vcf-roll-call/src/main.rs +++ b/apps/08-vcf-roll-call/src/main.rs @@ -27,10 +27,34 @@ const METRIC_MAX: usize = 512; fn main() -> Result<(), Box> { let Some(dir) = std::env::args().nth(1) else { print!( - "[package]\n\ - name = \"vcf-roll-call\"\n\ - version = \"0.2.0\"\n\ - datasets = [\"v1/genome/snp-indel\"]\n" + r#"manifest = 1 + +[app] +name = "VCF Roll Call" +version = "0.2.0" + +[reads] +paths = ["v1/genome/snp-indel"] + +[listing] +language = "en" +icon = "📋" +summary = "An inventory of your call file, from record counts to depth and quality." +category = "tools" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/08-vcf-roll-call" +keywords = ["VCF", "call file", "statistics", "data quality", "noodles"] +description = ''' +Most apps ask for a variant, a gene or a region. This one asks for the call file itself and streams it end to end with the noodles VCF parser. The report describes the file rather than you. It counts records per chromosome, classifies genotype calls, checks whether depth and quality fields are present, measures missingness and totals the span covered by reference blocks. + +A malformed record is counted and skipped. A failed read stops the app, so a damaged file cannot pass as a partial success. + +This is the broadest grant an app can ask for. Your phone shows it as the whole call file, and that is exactly what the app can read. +''' + +[listing.purposes] +"v1/genome/snp-indel" = "Your whole call file, to describe its contents" +"# ); return Ok(()); }; diff --git a/apps/08-motif-finder/Cargo.lock b/apps/09-motif-finder/Cargo.lock similarity index 100% rename from apps/08-motif-finder/Cargo.lock rename to apps/09-motif-finder/Cargo.lock diff --git a/apps/08-motif-finder/Cargo.toml b/apps/09-motif-finder/Cargo.toml similarity index 100% rename from apps/08-motif-finder/Cargo.toml rename to apps/09-motif-finder/Cargo.toml diff --git a/apps/08-motif-finder/README.md b/apps/09-motif-finder/README.md similarity index 90% rename from apps/08-motif-finder/README.md rename to apps/09-motif-finder/README.md index 09caba1..b226930 100644 --- a/apps/08-motif-finder/README.md +++ b/apps/09-motif-finder/README.md @@ -1,4 +1,4 @@ -# 08 - motif-finder +# 09 - motif-finder The app streams a gene's reference `sequence` and counts where five restriction enzymes would cut it. It keeps a four-base window across chunks, so a site @@ -12,5 +12,5 @@ the code happens to read. ## Build and run ```sh -make run APP=08-motif-finder +make run APP=09-motif-finder ``` diff --git a/apps/08-motif-finder/src/main.rs b/apps/09-motif-finder/src/main.rs similarity index 77% rename from apps/08-motif-finder/src/main.rs rename to apps/09-motif-finder/src/main.rs index accc5f2..3ed5ba0 100644 --- a/apps/08-motif-finder/src/main.rs +++ b/apps/09-motif-finder/src/main.rs @@ -22,10 +22,32 @@ const ENZYMES: &[(&str, &str)] = &[ fn main() { let Some(dir) = std::env::args().nth(1) else { print!( - "[package]\n\ - name = \"motif-finder\"\n\ - version = \"0.2.0\"\n\ - datasets = [\"v1/genome/genes/TAS2R38\"]\n" + r#"manifest = 1 + +[app] +name = "Motif Finder" +version = "0.2.0" + +[reads] +paths = ["v1/genome/genes/TAS2R38"] + +[listing] +language = "en" +icon = "✂️" +summary = "Where five classic restriction enzymes would cut the TAS2R38 reference sequence." +category = "tools" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/09-motif-finder" +keywords = ["restriction enzymes", "motifs", "TAS2R38", "reference genome"] +description = ''' +A restriction enzyme cuts DNA at a short fixed motif. This app streams the reference sequence of _TAS2R38_ and counts the sites of five classic enzymes, AluI, HaeIII, RsaI, TaqI and Sau3AI. It keeps a four base window across chunks so a site spanning two reads still counts, and uppercases bases because repeats are soft masked. + +The app reads only public reference sequence. Its grant is still the gene, and a gene grant also covers your variants in it. You approve the grant, not what the code happens to read, and the listing shows the grant. +''' + +[listing.purposes] +"v1/genome/genes/TAS2R38" = "The gene whose reference sequence the app scans for cut sites" +"# ); return; }; diff --git a/apps/09-fortune-cookie/Cargo.lock b/apps/10-fortune-cookie/Cargo.lock similarity index 100% rename from apps/09-fortune-cookie/Cargo.lock rename to apps/10-fortune-cookie/Cargo.lock diff --git a/apps/09-fortune-cookie/Cargo.toml b/apps/10-fortune-cookie/Cargo.toml similarity index 100% rename from apps/09-fortune-cookie/Cargo.toml rename to apps/10-fortune-cookie/Cargo.toml diff --git a/apps/09-fortune-cookie/README.md b/apps/10-fortune-cookie/README.md similarity index 69% rename from apps/09-fortune-cookie/README.md rename to apps/10-fortune-cookie/README.md index deb7d0a..5ad1f9e 100644 --- a/apps/09-fortune-cookie/README.md +++ b/apps/10-fortune-cookie/README.md @@ -1,4 +1,4 @@ -# 09 - fortune-cookie +# 10 - fortune-cookie An Ark's sandbox hands apps zero random bytes and clocks that only count, so a random-looking fortune has to come from the input. This app hashes the genotypes @@ -9,7 +9,7 @@ any other read error stops the app. ## Build and run ```sh -make run APP=09-fortune-cookie -make run APP=09-fortune-cookie FIXTURES=fixtures/no-call -make run APP=09-fortune-cookie FIXTURES=fixtures/unanswered +make run APP=10-fortune-cookie +make run APP=10-fortune-cookie FIXTURES=fixtures/no-call +make run APP=10-fortune-cookie FIXTURES=fixtures/unanswered ``` diff --git a/apps/09-fortune-cookie/src/main.rs b/apps/10-fortune-cookie/src/main.rs similarity index 64% rename from apps/09-fortune-cookie/src/main.rs rename to apps/10-fortune-cookie/src/main.rs index 5bf4715..6c60428 100644 --- a/apps/09-fortune-cookie/src/main.rs +++ b/apps/10-fortune-cookie/src/main.rs @@ -1,11 +1,10 @@ //! Why the sandbox is deterministic. //! -//! The runtime gives an app no randomness: the RNG returns zeros and the clock +//! The runtime gives an app no randomness. The RNG returns zeros and the clock //! is a counter, not the time of day. So an app cannot roll dice. To make a //! choice that feels random it has to derive it from its input. This one folds a //! few of your genotypes into a number and picks a fortune from it. The same -//! genome gives the same fortune on every run, which is the whole point: nothing -//! here varies that is not in the data. +//! genome gives the same fortune every time it runs. Only the data varies. use std::path::Path; use std::{fs, io}; @@ -26,13 +25,34 @@ const FORTUNES: &[&str] = &[ fn main() { let Some(dir) = std::env::args().nth(1) else { print!( - "[package]\n\ - name = \"fortune-cookie\"\n\ - version = \"0.2.0\"\n\ - datasets = [\ - \"v1/genome/rsids/rs72921001\", \ - \"v1/genome/rsids/rs671\", \ - \"v1/genome/rsids/rs1229984\"]\n" + r#"manifest = 1 + +[app] +name = "Fortune Cookie" +version = "0.2.0" + +[reads] +paths = ["v1/genome/rsids/rs72921001", "v1/genome/rsids/rs671", "v1/genome/rsids/rs1229984"] + +[listing] +language = "en" +icon = "🥠" +summary = "A fortune derived from three of your genotypes. The same one every time." +category = "traits" +license = "BSD-3-Clause" +source = "https://github.com/dark-bio/examples/tree/main/apps/10-fortune-cookie" +keywords = ["fortune", "deterministic", "sandbox", "hashing"] +description = ''' +The Ark's sandbox hands apps zero random bytes and clocks that only count. There is no randomness to draw a fortune from, so this app hashes the genotypes of three variants into one. Run it again on the same data and you get the same fortune. + +An absent genotype adds nothing to the hash and is noted in the report. Any other read error stops the app. It is a demonstration of a deterministic sandbox, wrapped in a cookie. +''' + +[listing.purposes] +"v1/genome/rsids/rs72921001" = "One of three genotypes hashed into your fortune" +"v1/genome/rsids/rs671" = "One of three genotypes hashed into your fortune" +"v1/genome/rsids/rs1229984" = "One of three genotypes hashed into your fortune" +"# ); return; }; diff --git a/docs/01-app-model.md b/docs/01-app-model.md index a7ccf82..ce6b753 100644 --- a/docs/01-app-model.md +++ b/docs/01-app-model.md @@ -9,20 +9,20 @@ everything below follows from that. ## Two passes -The Ark runs every app twice, and tells the two passes apart by the first -command-line argument. +The Ark runs every app twice, and the first command-line argument distinguishes +the two passes. - **The manifest pass** runs the module with no arguments. The app prints a short TOML manifest naming itself and the data it wants, then exits. Nothing is mounted yet, so it can't read anything. [02-manifest.md](02-manifest.md) covers the format. -- **The run pass** runs the module again once the owner has approved it, with +- **The report pass** runs the module again once the owner has approved it, with the data directory as the first argument. The app reads the files it was granted and prints its report. An app picks its pass by counting arguments: -| Language | Manifest pass | Run pass | +| Language | Manifest pass | Report pass | | :-- | :-- | :-- | | Rust | `std::env::args().nth(1).is_none()` | otherwise | | Go | `len(os.Args) < 2` | otherwise | @@ -35,18 +35,27 @@ The report is Markdown, since that is how it is shown, and ## Before the owner is asked When an app is scheduled, the Ark checks it before any prompt reaches the owner's -phone, and refuses it if a check fails. - -- A module with a WebAssembly start section is refused, since a run begins at - `_start`. -- A name or version that is empty, too long, or holds control characters, line - separators or text direction controls is refused. -- A dataset path that is misspelled or can't be granted is refused, and so is - one whose data isn't on the Ark. +phone, in this order, and refuses it at the first check that fails. The refusal +names the rule that broke. + +1. **The module.** A module with a WebAssembly start section is refused, since + an app begins at `_start`. So is a component, or a module with more than one + linear memory or table. A module that starts with more memory or table space + than the limits below is refused, and so is one that imports anything but + WASI Preview 1 functions. The module has to export `_start` as a function + without parameters or results. +2. **The manifest pass.** A pass that traps, runs out of time, exits with a + status other than 0 or prints more than 64 KiB is refused. +3. **The manifest.** A manifest that breaks a rule of + [02-manifest.md](02-manifest.md) is refused. +4. **The paths.** A path that is misspelled or can't be granted is refused, and + so is a required one whose data isn't on the Ark. The refusal names the + missing data. An optional path whose data is missing goes to the owner as + unavailable instead. ## The sandbox -The run pass has no network, no writable storage, and no access to anything +The report pass has no network, no writable storage, and no access to anything beyond its grants, which are mounted read-only. The sandbox is also deterministic, so the same app over the same data prints the @@ -58,39 +67,47 @@ same report every time: unrelated to real time. An app that needs a random-looking choice derives it from its input. -[09-fortune-cookie](../apps/09-fortune-cookie) does exactly that. +[10-fortune-cookie](../apps/10-fortune-cookie) does exactly that. ## Limits -| | Manifest pass | Run pass | +| | Manifest pass | Report pass | | :-- | :-- | :-- | -| Memory | 16.125 MiB | 100 MiB | -| Standard output | 1 KiB | 1 MiB | -| Standard error | 1 KiB | 1 MiB | +| Memory | 32 MiB | 128 MiB | +| Table | 65,536 elements | 65,536 elements | +| Standard output | 64 KiB | 1 MiB | +| Standard error | 1 KiB, never returned | 1 MiB | | Time | 250 ms | none, but cancellable | | Data | none | granted paths, read-only | -Output past a cap is dropped. The module itself can be up to 256 MiB. +A manifest pass that prints more than 64 KiB is refused, and a report pass that +prints more than 1 MiB to either stream fails. A module can be up to 256 MiB. ## The develop flag -By default the Ark returns an app's standard output only when the app exits -successfully, and never returns its standard error, so a failed run returns -nothing. +After the report pass, the owner reviews what the app printed before anything +leaves the Ark. By default that is the report, the app's standard output, and +only when the app succeeds. A failure reaches the review too, with nothing but +the fact that it failed, and standard error never does. -Setting `develop = true` in the manifest returns standard output even on failure, -and standard error every time. It is a debugging switch, and the owner sees a -developer mode warning when approving such an app. Leave it out of apps you ship. +Setting `develop = true` in the manifest sends standard error to the review as +well, and standard output even when the app fails. It is a debugging switch, +and the owner sees a developer warning when approving such an app. Leave it out +of apps you ship. -## A run on an Ark +## Running on an Ark 1. A developer sends the module to the Ark, for example with `ark app run app.wasm`. -2. The Ark runs the manifest pass, then checks the module, its name and version, - and every dataset path. -3. The owner sees the app's name, version and requested paths on their phone, - and approves or declines. -4. On approval, the Ark mounts the granted paths read-only and runs the app. -5. The result is returned to the owner, who decides whether to release it. +2. The Ark checks the module, runs the manifest pass, then checks the manifest + and every path it grants. +3. The owner sees the app's name, version and requested paths on their phone. + They switch on the optional grants they choose, answer the app's questions, + and approve or decline. +4. On approval, the Ark mounts the granted paths and the answers read-only and + runs the report pass. +5. The owner reviews what the app printed, and releases it to the session that + started the task, such as the `ark` command, or keeps it. Nothing leaves the + Ark before that. Every step is recorded in a signed, tamper-evident journal on the device. diff --git a/docs/02-manifest.md b/docs/02-manifest.md index fcd10c1..658ddfe 100644 --- a/docs/02-manifest.md +++ b/docs/02-manifest.md @@ -1,33 +1,84 @@ # The manifest -The manifest is the TOML an app prints in its manifest pass. It names the app and -lists the data the app wants. The Ark reads it to decide what to check, what to -show the owner and what to mount. +The manifest is the TOML an app prints in its manifest pass. It names the app, +lists the data the app reads, asks the owner any questions the app needs +answered, and carries the text a listing shows. The Ark reads it to decide what +to check, what to show the owner and what to mount. ```toml -[package] -name = "cilantro" -version = "0.1.0" -datasets = ["v1/genome/rsids/rs72921001"] +manifest = 1 + +[app] +name = "Cilantro Taste Test" +version = "0.4.0" + +[reads] +paths = ["v1/genome/rsids/rs72921001", "v1/genome/reference"] + +[listing] +language = "en" +icon = "🌿" +summary = "Does cilantro taste like soap to you? One variant near OR6A2 has a say." +category = "traits" +license = "BSD-3-Clause" ``` -## Fields +[04-cilantro-soapiness](../apps/04-cilantro-soapiness) prints this manifest, +with a longer listing. + +## Format + +A manifest is a TOML 1.1.0 document. It opens with `manifest = 1`, the version +of this format, followed by the tables below. The Ark reads `[app]`, `[reads]`, +`[inputs]` and `[output]` strictly, and refuses a manifest with a key it doesn't +know in any of them. It skips `[listing]` without reading it, and refuses any +other table or top-level key, and any other `manifest` value. The whole manifest +has to fit in the manifest pass's 64 KiB of output. -All fields live under `[package]`. +Text that the owner or a listing shows, such as the name, a prompt or a choice, +is display text. It holds at least one character, and no control characters, +line or paragraph separators, or text direction controls. Emoji are fine, +joined ones included. Long text reads best in TOML literal strings, `'...'` and +`'''...'''`, since they need no escaping. -- **`name`** is the app's name, shown to the owner, from 1 to 64 characters. -- **`version`** is shown beside the name, from 1 to 32 characters. -- **`datasets`** lists the paths the app wants. It may be empty. +### `[app]` + +- **`name`** is the app's name as the owner sees it, on the phone, in the + journal and on a listing, as display text of 1 to 64 characters. +- **`version`** is shown beside the name. It is a + [Semantic Versioning 2.0.0](https://semver.org/) version of at most 32 + characters, such as `0.4.0` or `1.0.0-beta.1`, with no leading `v` and no + build metadata after a `+`. - **`develop`** is optional and `false` by default. See [01-app-model.md](01-app-model.md). -Names and versions can't hold control characters, line or paragraph separators, -or text direction controls. The whole manifest has to fit in the manifest pass's -1 KiB of output, which holds a panel of a dozen paths with room to spare. +### `[reads]` + +- **`paths`** lists the directories the app needs. +- **`optional`** lists directories the owner may decline, which + [Optional grants](#optional-grants) covers. + +An app that reads nothing leaves the table out. + +### `[inputs.]` + +Each table asks the owner one question, which +[Asking the owner](#asking-the-owner) covers. + +### `[output]` + +- **`report`** is the report's media type, `text/markdown` by default. The + format also defines `application/json`, which the current Ark firmware + refuses, so every example leaves the table out and prints Markdown. + +### `[listing]` + +The text Ark Hub shows when it lists the app, which +[Listing an app](#listing-an-app) covers. ## Granting data -Each dataset is a directory under the Ark's data root. Granting it gives the app +Each path is a directory under the Ark's data root. Granting it gives the app that directory and everything beneath it, read-only. At run time the grants sit under the data directory the app receives as its first argument, so with `/` as that argument, a grant of `v1/genome/rsids/rs72921001` is read at @@ -45,20 +96,114 @@ fails. paths, empty, `.` or `..` segments, a trailing `/`, other spellings such as `chr01` or `rs0334`, files, `changes` directories, the data root and `v1/` itself are all refused. -- **Its data must be on the Ark.** A well-formed path is refused when it points - into an empty slot, names a gene the annotations don't carry or an rsID dbSNP - doesn't carry, or names a position past the end of its chromosome. +- **A path is listed once.** A path listed twice, in one list or across both, is + refused, and so is one inside another listed path's directory, which already + grants it. A manifest lists at most 1,024 paths across both lists. +- **Its data must be on the Ark.** A well-formed path in `paths` is refused when + it points into an empty slot, names a gene the annotations don't carry or an + rsID dbSNP doesn't carry, or names a position past the end of its chromosome. A `changes` directory can't be granted. Grant its gene or interval instead. +## Optional grants + +A path in `optional` is one the owner may decline. The phone lists it with a +switch that starts off, so the owner turns on only what they choose to share. +It suits data that adds to a report without being needed for its answer. + +- Public data can't be optional, since there is nothing to decline, and the Ark + refuses it there. +- An optional path whose data isn't on the Ark doesn't refuse the app. The + phone shows it as unavailable instead. +- A declined path and one whose data the Ark lacks are both left unmounted, so + the app can't know which it was. It reads the missing directory like any + absent answer, as [04-reading-data.md](04-reading-data.md#least-privilege) + shows. + +[02-permissions](../apps/02-permissions) asks for one optional variant. + ## Choosing grants The owner reads the list of paths before approving, so ask for the narrowest ones that do the job. One variant reads very differently from the whole call file. [04-reading-data.md](04-reading-data.md) walks through the options, from -narrowest to broadest. +narrowest to broadest. Data that only enriches a report belongs in `optional`, +so an owner who would rather not share it can still run the app. Some grants reach further than they look. A gene or interval grant includes its `changes`, which hold the owner's variants, even when the app only reads the sequence. Only `v1/genome/reference` and `v1/genome/annotations` hold nothing but public data. + +## Asking the owner + +An app can ask the owner questions, which they answer on the phone as part of +the approval. Each `[inputs.]` table declares one: + +```toml +[inputs.flavor] +type = "choice" +prompt = "Which flavor should the scoop be?" +choices = ["Vanilla", "Chocolate", "Strawberry", "Pistachio"] +default = "Vanilla" +``` + +- **`type`** is `"choice"` or `"text"`. +- **`prompt`** is the question, display text of at most 80 characters. +- **`choices`**, for a choice, lists 1 to 32 distinct answers, each display + text of at most 64 characters. +- **`multiple`**, for a choice, lets the owner pick any number of them. It is + `false` by default. +- **`max`**, for a text, is the longest answer accepted, from 1 to 256 + characters. +- **`default`** is what the phone fills in first. It is one of the choices, a + list of them in the order of `choices` for a multiple choice, or a text that + fits `max`. Without one, a choice starts unpicked and a text starts empty. + +A name is 1 to 32 lowercase ASCII letters, digits, `-` and `_`, and starts with +a letter. A manifest declares at most 16 inputs. The Ark refuses a declaration +that breaks one of these rules, and checks every answer against its +declaration. + +The answers reach the app as files under `inputs/` in the data directory, one +per input and named after it, in UTF-8 with no trailing newline. A choice's file +holds the picked choice. A multiple choice's file holds the picks one per line, +in the order of `choices`, and is empty when nothing was picked. A text's file +holds the text, which is never empty. +[03-sundae-order](../apps/03-sundae-order) reads all three. + +Inputs never change what an app reads. An app whose reads depend on an answer +grants everything the answer can select. The Ark's journal records which inputs +the owner filled in, never the answers. + +## Listing an app + +`[listing]` holds what Ark Hub shows when it lists the app. The Ark never reads +it and it never reaches the phone, so it changes nothing about what an app can +do. Ark Hub needs the first five keys before it lists an app, and every example +carries a full listing. + +| Key | To be listed | Rules | +| :-- | :-- | :-- | +| `language` | required | The language of the listing and of the report, a BCP 47 tag such as `en` | +| `icon` | required | Exactly one emoji | +| `summary` | required | Display text of at most 80 characters | +| `category` | required | `traits`, `tools` or `developer` | +| `license` | required | An SPDX expression, such as `BSD-3-Clause` | +| `source` | optional | An `https` URL where the app's source lives | +| `description` | optional | Restricted Markdown of at most 8 KiB | +| `keywords` | optional | At most 5, each display text of at most 20 characters | +| `purposes` | optional | For each grant, why the app reads it | + +A description allows paragraphs, emphasis, inline code, fenced code blocks and +lists, and no headings, links, images, HTML, tables or footnotes. A purpose is +display text of at most 80 characters, keyed by a path from `[reads]`: + +```toml +[listing.purposes] +"v1/genome/rsids/rs72921001" = "The variant near OR6A2 tied to tasting cilantro as soap" +``` + +Purposes appear on the listing only. The phone describes every grant in the +Ark's own words, never the app's. A module holds one language, so an app in +another language is a separate app. diff --git a/docs/03-data-paths.md b/docs/03-data-paths.md index 7accd44..5e13031 100644 --- a/docs/03-data-paths.md +++ b/docs/03-data-paths.md @@ -95,7 +95,7 @@ value instead of a file format to parse. - **Sequences stream.** A `sequence` file holds exactly end - start + 1 bases, so its size is its length, and byte offset i is position start + i. Repeats are soft-masked in lowercase. Read long sequences as streams, since an app has - 100 MiB of memory. + 128 MiB of memory. - **The stored datasets are big.** `snp-indel/vcf` and `reference/fa` are decompressed on read and run to gigabytes, so read them line by line. - **`available`** means the data a path needs is on the Ark, not that every gene diff --git a/docs/04-reading-data.md b/docs/04-reading-data.md index 43af61c..b7bbc46 100644 --- a/docs/04-reading-data.md +++ b/docs/04-reading-data.md @@ -1,9 +1,9 @@ # Reading data An app reads its grants as ordinary files. This page covers the three things -every app has to get right, which are telling absence from failure, reading -sequences, and parsing genotypes. Then it walks the grants from narrowest to -broadest, each with an example to run. +every app has to get right, which are distinguishing absence from failure, +reading sequences, and parsing genotypes. Then it walks the grants from +narrowest to broadest, each with an example to run. ## Absence and failure @@ -36,7 +36,7 @@ reference sites. An app must never read absence as a result. A gene's or an interval's `sequence` holds exactly end - start + 1 bases, so the file's size is the span's length and byte offset i is position start + i. Repeats are soft-masked in lowercase, so uppercase bases before matching them. -A long sequence can outgrow an app's 100 MiB of memory, so read it in chunks +A long sequence can outgrow an app's 128 MiB of memory, so read it in chunks through a buffered reader rather than whole. ## Parsing genotypes @@ -54,7 +54,7 @@ app meets most are `A/G` and `A|G`, a haploid `G`, a missing allele such as At a known SNP, splitting on `/` and `|` is enough. An app that walks `changes` can meet symbolic alleles such as `` and breakends, which contain those characters themselves, so it has to split only outside them. -[05-bitter-meter](../apps/05-bitter-meter) shows how. +[06-bitter-meter](../apps/06-bitter-meter) shows how. ## From narrowest to broadest @@ -67,17 +67,18 @@ Grant one `rsids/` directory and read its `genotype`, `chromosome`, `position` and `reference`. ```toml -datasets = ["v1/genome/rsids/rs72921001"] +[reads] +paths = ["v1/genome/rsids/rs72921001"] ``` -[03-cilantro-mini-rust](../apps/03-cilantro-mini-rust), with its Go, C and +[04-cilantro-mini-rust](../apps/04-cilantro-mini-rust), with its Go, C and Python siblings, shows the bare read. -[03-cilantro-soapiness](../apps/03-cilantro-soapiness) turns it into a report. +[04-cilantro-soapiness](../apps/04-cilantro-soapiness) turns it into a report. ### A panel of variants Grant one `rsids/` directory per variant. The owner sees each site by name. -[04-drunk-o-type](../apps/04-drunk-o-type) reads eleven. +[05-drunk-o-type](../apps/05-drunk-o-type) reads eleven. ### One position @@ -91,20 +92,20 @@ Grant `genes/` to read its `chromosome`, `start`, `end`, `strand`, `biotype` and `sequence`, and its `changes`, one directory per position where a record starting inside the gene holds an ALT allele. A gene can have no `changes` directory at all, and a very long or very variable one fails with -"file too large". [05-genes-mini-rust](../apps/05-genes-mini-rust) shows the bare read, -and [05-bitter-meter](../apps/05-bitter-meter) turns TAS2R38 into a report. +"file too large". [06-genes-mini-rust](../apps/06-genes-mini-rust) shows the bare read, +and [06-bitter-meter](../apps/06-bitter-meter) turns TAS2R38 into a report. ### An interval Grant `regions//-` for any stretch of a chromosome, with the same `sequence` and `changes` as a gene. When `changes` fails with "file too large", split the interval and list each half. -[06-regions-mini-rust](../apps/06-regions-mini-rust) shows the bare read, and -[06-powerhouse-of-the-cell](../apps/06-powerhouse-of-the-cell) reads the whole +[07-regions-mini-rust](../apps/07-regions-mini-rust) shows the bare read, and +[07-powerhouse-of-the-cell](../apps/07-powerhouse-of-the-cell) reads the whole mitochondrial genome as one interval. A gene or interval grant includes its `changes`, even if the app never lists -them. [08-motif-finder](../apps/08-motif-finder) reads only a gene's sequence, +them. [09-motif-finder](../apps/09-motif-finder) reads only a gene's sequence, yet its grant still covers the gene's variants. ### The whole call file @@ -113,13 +114,20 @@ Grant `snp-indel` to read the `vcf` itself, when no lens answers the question. It is the broadest request there is, and the file runs to gigabytes, so read it line by line, through a large buffer. Every refill leaves the sandbox, so a small default buffer can cost ten times the scan time. -[07-vcf-mini-rust](../apps/07-vcf-mini-rust) counts records with a plain line -scan, and [07-vcf-roll-call](../apps/07-vcf-roll-call) parses the whole file +[08-vcf-mini-rust](../apps/08-vcf-mini-rust) counts records with a plain line +scan, and [08-vcf-roll-call](../apps/08-vcf-roll-call) parses the whole file with `noodles-vcf`. ## Least privilege The Ark mounts only what a manifest grants. Granting one rsID gives that directory, not the call file behind it or its neighbours. -[02-permissions](../apps/02-permissions) reads a granted path, then shows that an -undeclared one can't be opened. + +An optional grant the owner declined, or whose data the Ark lacks, isn't +mounted, so its directory is missing. An app reads that like any absent answer, +and it can't distinguish a decline from missing data. So an app checks for the +directory before reading an optional grant, and reports its absence as a +finding, never as an error. + +[02-permissions](../apps/02-permissions) reads a granted path and an optional +one, then shows that an undeclared path can't be opened. diff --git a/docs/05-running.md b/docs/05-running.md index 16d3bb8..6523e84 100644 --- a/docs/05-running.md +++ b/docs/05-running.md @@ -53,8 +53,8 @@ imports go in `--include` when calling `tools/python_build.py` directly. ```sh make build # build every app into build/.wasm make run # build and run every app -make run APP=03-cilantro-mini-rust -make run APP=03-cilantro-mini-rust FIXTURES=fixtures/no-call +make run APP=04-cilantro-mini-rust +make run APP=04-cilantro-mini-rust FIXTURES=fixtures/no-call ``` `make run` does what an Ark does, through `tools/run.sh`. It runs the module @@ -63,9 +63,28 @@ from the fixture root, read-only, and runs the module again with `/` as its first argument. An undeclared path is never mounted, which is how [02-permissions](../apps/02-permissions) can show a blocked read on a laptop. +An optional path is mounted when the fixture root holds it, as if the owner +switched it on, and left out when the root lacks it, the way an Ark leaves out +data it doesn't have. `OPTIONAL=off` leaves every optional path out, as if the +owner declined them all: + +```sh +OPTIONAL=off make run APP=02-permissions +``` + +An app that declares inputs gets its answers from files in its folder's +`inputs/` directory, one per input, which `make run` mounts at `/inputs`. +`INPUTS` points it at another directory of answer files. The answers are taken +as given, so a local run never checks them against their declarations. + +```sh +INPUTS=my-answers make run APP=03-sundae-order +``` + An app kept outside this repository runs against these fixtures the same way, since `tools/run.sh` takes any module, as in -`sh tools/run.sh path/to/app.wasm fixtures`. Build it with the `wasm-opt` +`sh tools/run.sh path/to/app.wasm fixtures`, with a directory of answers as a +third argument when the app declares inputs. Build it with the `wasm-opt` feature flags in `tools/build.sh`, because an Ark's runtime accepts only those. ### Small modules @@ -100,13 +119,14 @@ missing grants there. ### What a laptop doesn't reproduce - **The checks before approval.** Locally nothing refuses a start section, a bad - name, or a path that can't be granted or isn't on the Ark. + name, a path that can't be granted or isn't on the Ark, or an answer that + doesn't fit its input. - **The limits.** Memory, output and the manifest pass's time are unbounded. - **The deterministic sandbox.** A stock `wasmtime` gives real randomness and real clocks, so don't depend on either, as - [09-fortune-cookie](../apps/09-fortune-cookie) explains. -- **Output gating.** You always see standard output and standard error, as if - `develop` were set. + [10-fortune-cookie](../apps/10-fortune-cookie) explains. +- **The owner's review.** You always see standard output and standard error at + once, as if `develop` were set and the owner released every report. - **The generated tree.** Plain directories list everything, and no read fails with an I/O error or "file too large". - **Startup cost.** A laptop starts a module far faster than an Ark does, so a @@ -131,11 +151,13 @@ prints. Then check what the Ark holds and run the app: ```sh ark status # trust, firmware, pairing and lock state ark data paths # the paths apps can read, and what is available -ark app run build/03-cilantro-mini-rust.wasm > report.md +ark app run build/04-cilantro-mini-rust.wasm > report.md ``` `ark app run` uploads the module and waits while the owner approves it on their -phone. It then writes the report to standard output. A refused app comes back -with the reason. `ark help apps` covers manifests and grants, and +phone, with any optional grants and answers. When the app ends, the owner +reviews its report, and the command writes it to standard output once the owner +releases it. A refused app comes back with the reason, and a report the owner +keeps fails the command. `ark help apps` covers manifests and grants, and `ark help datasets` covers the data commands. AI agents read `ark help agents` first, and `ark-emulator help agents` for the emulator. diff --git a/docs/06-reports.md b/docs/06-reports.md index 29142f3..647a5fd 100644 --- a/docs/06-reports.md +++ b/docs/06-reports.md @@ -1,12 +1,12 @@ # Reports -A report is what an app prints in its run pass, and it is the only thing an app -can send out of the sandbox. The owner reads it first, on their phone, and -decides whether it goes any further. Whoever ran the app then gets its exact -bytes, in a terminal from `ark app run`, or rendered as Markdown. So a report is -read three ways, on a phone, as raw text and as a rendered page, and it has to -work in all of them. This page is a guideline for that, not a format the Ark -checks. The Ark returns whatever the app printed, up to 1 MiB. +A report is what an app prints in its report pass, and it is the only thing an +app can send out of the sandbox. The owner reads it first, on their phone, and +decides whether it goes any further. Once the owner releases it, the session +that started the task gets its exact bytes, in a terminal from `ark app run` or +rendered as Markdown in Ark Hub. So a report is read three ways, on a phone, as +raw text and as a rendered page, and it has to work in all of them. This page is a guideline for that. The Ark itself checks +only that a report is text, as [Form](#form) says. ## What a report has to do @@ -19,19 +19,19 @@ granted beside the report, so the report carries none of them. And the report is readable to the last byte. A report is only as trustworthy as it is readable. -Two things follow from the sandbox. The run is deterministic, so a report -carries no date and no run id. The same app over the same data prints the same +Two things follow from the sandbox. It is deterministic, so a report carries +no date and no task id. The same app over the same data prints the same report, which is what lets last year's be diffed against this year's, and an app that lists a directory sorts the listing before printing it, since listing -order isn't promised. And the output is capped, so a report summarises. It -lists the evidence for its finding and counts the rest, since output past the -cap is dropped without a mark. +order isn't promised. And the output is capped at 1 MiB, and an app that prints +more fails, so a report summarizes. It lists the evidence for its finding and +counts the rest. -An app that exits non-zero returns nothing. Without `develop = true` the Ark -withholds standard output on failure and never returns standard error, so the -owner is left with a blank screen after approving the run. A failure exit is -for data that couldn't be read. Everything an app can state, including that -it has no answer, is a report. +An app that exits non-zero fails, and the owner reviews a failure too. Without +`develop = true` the review shows nothing but the fact that it failed, since the +Ark withholds standard output on failure and never shows standard error. So a +failure exit is for data that couldn't be read. Everything an app can state, +including that it has no answer, is a report. ## The shape @@ -61,7 +61,7 @@ report, so an app called `drunk-o-type` prints a report titled "How Your Body Handles a Drink". The same shape holds one variant, a panel, a gene, a whole call file, and the summary a study asked for. -[03-cilantro-soapiness](../apps/03-cilantro-soapiness) is a light app, and +[04-cilantro-soapiness](../apps/04-cilantro-soapiness) is a light app, and its report in this shape: ```markdown @@ -121,7 +121,7 @@ never read as two reference alleles: **No answer.** No genotype is available at rs72921001, so this app has no result. An absent genotype is not a reference call. A call file that records only variants has no record at a reference site, and none at a site it didn't -cover, and the two can't be told apart here. +cover, and the two can't be distinguished here. ``` **Evidence** holds the values the finding rests on, one row per variant, gene @@ -145,8 +145,8 @@ established in. An app may carry allele frequencies or effect sizes from the literature, and cites where they came from like any other claim. **Limitations** says what the app didn't read and what the finding doesn't -establish, as facts rather than reassurance. One variant rarely tells a whole -story, and this section says how much this one tells. +establish, as facts rather than reassurance. One variant is rarely the whole +story, and this section says how far this one goes. **Sources** lists the studies Method named, in that order, in full, author, year, title and journal, with a URL where the work has a stable one. Links @@ -189,6 +189,12 @@ reference population. Title Case for the title, sentence case for headings. ## Form +A report is text. It has to be valid UTF-8, without control characters other +than line feed and tab, line or paragraph separators, or text direction +controls. Emoji are fine, joined ones included. The Ark drops a report that +breaks this, and the app counts as failed. A develop build's standard error is +held to the same rule. + Plain Markdown, and a small subset of it, because the subset is what keeps a report readable everywhere. One `#`, the `##` sections above, and `###` to group the rows of an Evidence section. Paragraphs, lists and @@ -198,7 +204,7 @@ is narrower still. A paragraph is one line of output, since every renderer wraps it. A text figure in a code fence when it earns its place. Nothing else. No images, no HTML, no footnotes and no encoded content, since the owner can't -read them. No dates and no run ids, which the sandbox can't give and the +read them. No dates and no task ids, which the sandbox can't give and the journal already has. Standard error is for the developer and never part of the report. diff --git a/fixtures/README.md b/fixtures/README.md index 7a9b87d..45f1123 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -13,11 +13,11 @@ made up, while the coordinates, reference alleles and sequences are public data. | `fixtures/no-call` | the rsID apps | a no-call, `./.`, at every panel variant | | `fixtures/unanswered` | the rsID apps | coordinates and reference alleles but no genotypes, as at reference sites in a variants-only call file | -The rsID apps are `02-permissions`, the cilantro apps, `04-drunk-o-type` and -`09-fortune-cookie`. Pick a root with `FIXTURES`: +The rsID apps are `02-permissions`, the cilantro apps, `05-drunk-o-type` and +`10-fortune-cookie`. Pick a root with `FIXTURES`: ```sh -make run APP=03-cilantro-mini-rust FIXTURES=fixtures/no-call +make run APP=04-cilantro-mini-rust FIXTURES=fixtures/no-call ``` The two smaller roots hold only the variant data those apps need, plus the diff --git a/tools/check.sh b/tools/check.sh index 6b78947..58a0cd7 100644 --- a/tools/check.sh +++ b/tools/check.sh @@ -1,7 +1,9 @@ #!/bin/sh # Check the examples the way CI does. Every documentation link resolves, every -# app builds, every app runs against each fixture root that holds its data, and -# the language versions of one demo all print the same report. +# app builds, and every app runs against each fixture root that holds its data. +# Apps with optional grants also run with those grants declined. Language ports +# print the same manifest apart from source URLs, and the same report except +# for the hello apps. # # Usage: tools/check.sh [app ...] set -eu @@ -18,6 +20,155 @@ fail() { failures=$((failures + 1)) } +# Compare one inline manifest with the facts its declarations supply. +test_manifest_facts() { + awk -f tools/manifest.awk > "$work/manifest-facts" + printf '%s\n' "$2" > "$work/manifest-expected" + diff "$work/manifest-expected" "$work/manifest-facts" || fail "manifest reader $1" +} + +# Check table, dotted and inline declarations while descriptions stay opaque. +check_manifest_reader() { + reader_failures=$failures + + # Read both grant lists and all three spellings of an input table name + test_manifest_facts tables 'paths v1/genome/rsids/rs72921001 +paths v1/genome/reference +optional v1/genome/rsids/rs671 +input flavor +input toppings +input message' <<'TOML' +manifest = 1 +[reads] # A comment can contain [inputs.fake] +paths = [ + 'v1/genome/rsids/rs72921001', # A comment can contain ] and "quotes" + "v1/genome/reference", +] +optional = ['v1/genome/rsids/rs671'] +[inputs.flavor] +type = 'text' +prompt = 'Flavor' +max = 32 +[ inputs . "toppings" ] +type = 'text' +prompt = 'Toppings' +max = 32 +[inputs.'message'] +type = 'text' +prompt = 'Message' +max = 32 +TOML + + # Dotted declarations keep source order and emit each input name once + test_manifest_facts dotted 'paths v1/genome/rsids/rs72921001 +optional v1/genome/rsids/rs671 +input z +input a' <<'TOML' +manifest = 1 +reads.paths = ['v1/genome/rsids/rs72921001'] +reads.optional = ["v1/genome/rsids/rs671"] +inputs."z".type = 'text' +inputs.'a'.type = 'text' +inputs.a.prompt = 'A' +inputs.z.prompt = 'Z' +inputs.a.max = 1 +inputs.z.max = 1 +TOML + + # Entries in the inputs table accept inline values and dotted fields + test_manifest_facts input_entries 'input flavor +input toppings +input message' <<'TOML' +manifest = 1 +[inputs] +"fl\u0061vor" = { type = 'text', prompt = 'Flavor', max = 32 } +'toppings'.type = 'text' +message.type = 'text' +message.prompt = 'Message' +toppings.prompt = 'Toppings' +message.max = 32 +toppings.max = 32 +TOML + + # Top-level inline tables accept comments, newlines and trailing commas + test_manifest_facts inline 'paths v1/genome/reference +optional v1/genome/rsids/rs671 +input z +input a' <<'TOML' +manifest = 1 +reads = { paths = ["v1/genome/\x72eference"], optional = [ + 'v1/genome/rsids/rs671', +] } +inputs = { + 'z' = {type = 'text', prompt = 'Z', max = 1}, # The first question + "a" = { + type = 'text', + prompt = 'A', + max = 1, + }, +} +TOML + + # Fenced declarations inside a literal description remain string content + test_manifest_facts literal_description 'paths v1/genome/reference +input real' <<'TOML' +manifest = 1 +[listing] +description = ''' +```toml +[reads] +paths = ['v1/genome/rsids/rs1'] +optional = ["v1/genome/rsids/rs2"] +[inputs.x] +type = 'text' +inputs.fake.type = 'text' +``` +''' +[reads] +paths = ['v1/genome/reference'] +[inputs.real] +type = 'text' +prompt = 'Real' +max = 1 +TOML + + # Escaped quotes keep a basic description open across fenced declarations + test_manifest_facts basic_description 'optional v1/genome/rsids/rs671 +input real' <<'TOML' +manifest = 1 +listing.description = """ +The text contains \"\"\" and two quotes "". +```toml +[reads] +paths = ['v1/genome/rsids/rs1'] +[inputs.x] +type = "text" +reads.optional = ['v1/genome/rsids/rs2'] +``` +""" +reads.optional = ['v1/genome/rsids/rs671'] +inputs.real = {type = 'text', prompt = 'Real', max = 1} +TOML + + # Unsupported names and string values never become partial facts + test_manifest_facts unsupported 'paths v1/genome/reference +input real' <<'TOML' +manifest = 1 +reads.paths = ["v1/genome/rsids/rs\n1", 'v1/genome/reference'] +"inputs.fake" = {type = 'text', prompt = 'Fake', max = 1} +[inputs."bad.name"] +type = 'text' +[inputs.real] +type = 'text' +prompt = 'Real' +max = 1 +[listing] +inputs.fake = {type = 'text', prompt = 'Fake', max = 1} +TOML + + [ "$reader_failures" -ne "$failures" ] || echo "checked the manifest reader" +} + # A markdown link is `](target)`, and only the relative ones name a file here. check_links() { for page in README.md fixtures/README.md docs/*.md apps/*/README.md; do @@ -39,27 +190,46 @@ fixture_roots() { done } -# An app whose data a root lacks stops before its run pass, the way an Ark -# refuses a grant it cannot satisfy. Anything else is a failure. +# Run an app, accepting a stop when its required data is missing. +# An optional third argument of off declines optional grants in a separate run. run_app() { - app=$1 root=$2 output=$work/output-$1-$(basename "$2") - if sh tools/run.sh "${BUILD:-build}/$app.wasm" "$root" > "$output" 2>&1; then - sed -n '/^== run pass ==$/,$p' "$output" > "$work/report-$app-$(basename "$root")" - elif ! grep -q "no declared dataset" "$output"; then - fail "$app on $root" + # Store declined-grant runs separately so language comparisons keep defaults + app=$1 root=$2 output=$work/output-$1-$(basename "$2")${3:+-optional-off} + + # Accept a successful report or the expected stop for a missing required grant + if OPTIONAL="${3:-${OPTIONAL:-}}" sh tools/run.sh "${BUILD:-build}/$app.wasm" "$root" "apps/$app/inputs" > "$output" 2>&1; then + sed -n '/^== report pass ==$/,$p' "$output" > "$work/report-$app-$(basename "$root")${3:+-optional-off}" + elif [ "$?" -ne 3 ]; then + fail "$app on $root${3:+ (OPTIONAL=$3)}" cat "$output" fi } +# Build every selected app and run its grants on each fixture root. build_and_run() { for app in $apps; do + # Build once for all fixture roots and grant choices if ! sh tools/build.sh "$app" > "$work/build-$app" 2>&1; then fail "$app does not build" cat "$work/build-$app" continue fi + + # Capture each built module's manifest without its port-specific source URL + if ! "${WASMTIME:-wasmtime}" "${BUILD:-build}/$app.wasm" > "$work/manifest-$app" 2> "$work/manifest-error-$app"; then + fail "$app does not print its manifest" + cat "$work/manifest-error-$app" + continue + fi + sed '/^source = /d' "$work/manifest-$app" > "$work/manifest-compared-$app" + awk -f tools/manifest.awk "$work/manifest-$app" > "$work/facts-$app" + + # Keep the default report for language comparisons and check declined grants for root in $(fixture_roots); do run_app "$app" "$root" + if grep -q '^optional ' "$work/facts-$app"; then + run_app "$app" "$root" off + fi done done echo "built and ran $(echo "$apps" | wc -w | tr -d ' ') apps" @@ -87,9 +257,26 @@ compare_languages() { echo "compared language versions" } +# Compare each port's manifest with Rust when both were built in this run. +compare_manifests() { + for app in $apps; do + family=$(printf '%s' "$app" | sed -E 's/-(c|go|python|rust)$//') + reference=$family-rust + [ "$family" != "$app" ] || continue + { [ "$app" != "$reference" ] && [ -d "apps/$reference" ]; } || continue + mine=$work/manifest-compared-$app + theirs=$work/manifest-compared-$reference + { [ -e "$mine" ] && [ -e "$theirs" ]; } || continue + diff "$theirs" "$mine" > /dev/null || fail "$app manifest differs from $reference" + done + echo "compared language manifests" +} + check_links +check_manifest_reader build_and_run compare_languages +compare_manifests [ "$failures" -eq 0 ] || { printf '%d check(s) failed\n' "$failures"; exit 1; } echo "all checks passed" diff --git a/tools/manifest.awk b/tools/manifest.awk new file mode 100644 index 0000000..3376d6b --- /dev/null +++ b/tools/manifest.awk @@ -0,0 +1,231 @@ +# Read grant paths and input names from an app's TOML manifest. +# Strings keep their contents separate from keys, including fenced examples. +# Unsupported names and strings are omitted. This is not a manifest validator. +# +# Usage: awk -f tools/manifest.awk manifest.toml + +# Read a quoted string, decoding printable ASCII escapes used in paths and keys. +function read_string(quote, triple, count, char, escape, digits, number, i) { + triple = substr(source, position, 3) == quote quote quote + kind = triple ? "multiline" : "string" + position += triple ? 3 : 1 + token = "" + readable = 1 + if (triple && substr(source, position, 1) == "\n") position++ + + # Consume the whole string even when its value cannot be represented + while (position <= length(source)) { + char = substr(source, position++, 1) + if (char == quote) { + count = 1 + if (triple) { + while (substr(source, position, 1) == quote && count < 5) { + count++ + position++ + } + } + if (!triple || count >= 3) { + for (i = 3; i < count; i++) token = token quote + if (!readable) token = "\001" + return + } + for (i = 0; i < count; i++) token = token quote + } else if (char == "\\" && quote == "\"") { + escape = substr(source, position++, 1) + if (triple && escape ~ /[ \t\n]/) { + while (substr(source, position, 1) ~ /[ \t\n]/) position++ + } else if (escape == "\"" || escape == "\\") { + token = token escape + } else if (escape == "x" || escape == "u" || escape == "U") { + digits = escape == "x" ? 2 : (escape == "u" ? 4 : 8) + number = 0 + for (i = 0; i < digits; i++) { + char = tolower(substr(source, position++, 1)) + if (char !~ /^[0-9a-f]$/) readable = 0 + number = number * 16 + index("0123456789abcdef", char) - 1 + } + if (number >= 32 && number <= 126) { + token = token sprintf("%c", number) + } else { + readable = 0 + } + } else { + readable = 0 + } + } else { + if (!triple && char == "\n") readable = 0 + token = token char + } + } + kind = "end" + token = "" +} + +# Advance to one token, keeping newlines and dropping whitespace and comments. +function scan( char) { + token = "" + kind = "end" + while (position <= length(source)) { + char = substr(source, position, 1) + if (char == "#") { + while (position <= length(source) && + substr(source, position, 1) != "\n") position++ + } else if (char ~ /[ \t]/) { + position++ + } else { + break + } + } + if (position > length(source)) return + + # Quoted values consume their own punctuation and embedded newlines + if (char == "\"" || char == "'") { + read_string(char) + return + } + if (index("[]=,.{}\n", char)) { + kind = token = char + position++ + return + } + kind = "bare" + while (position <= length(source)) { + char = substr(source, position, 1) + if (char ~ /[ \t\n]/ || index("[]=,.{}#\"'", char)) break + token = token char + position++ + } +} + +# Read a dotted key, keeping quoted dots separate from table separators. +function read_key( path, part) { + path = "" + while (kind == "bare" || kind == "string") { + part = token ~ /^[A-Za-z0-9_-]+$/ ? token : "?" + path = path == "" ? part : path SUBSEP part + scan() + if (kind != ".") return path + scan() + } + ok = 0 + return "?" +} + +# Skip newlines within arrays and TOML 1.1 inline tables. +function skip_newlines() { + while (kind == "\n") scan() +} + +# Return an input fact for a table path with a supported input name. +function input_fact(path, parts, count) { + count = split(path, parts, SUBSEP) + if (count >= 2 && parts[1] == "inputs" && + parts[2] ~ /^[a-z][a-z0-9_-]*$/ && length(parts[2]) <= 32) { + return "input " parts[2] "\n" + } + return "" +} + +# Read a value and return facts from its arrays, dotted keys and inline tables. +function read_value(path, facts, grant, key, prefix, parts, count) { + facts = "" + count = split(path, parts, SUBSEP) + prefix = count > 2 || kind == "{" ? input_fact(path) : "" + + # Grant arrays accept readable strings and skip values of other types + if (kind == "[") { + if (path == "reads" SUBSEP "paths") grant = "paths" + if (path == "reads" SUBSEP "optional") grant = "optional" + scan() + skip_newlines() + while (kind != "]") { + if (grant != "" && (kind == "string" || kind == "multiline") && + token != "" && token !~ /[[:space:][:cntrl:]]/) { + facts = facts grant " " token "\n" + } + read_value("?") + skip_newlines() + if (!ok) return "" + if (kind == "]") break + if (kind != ",") { ok = 0; return "" } + scan() + skip_newlines() + } + scan() + } else if (kind == "{") { + scan() + skip_newlines() + while (kind != "}") { + key = read_key() + if (!ok || kind != "=") { ok = 0; return "" } + scan() + facts = facts read_value(path SUBSEP key) + skip_newlines() + if (!ok) return "" + if (kind == "}") break + if (kind != ",") { ok = 0; return "" } + scan() + skip_newlines() + } + scan() + } else if (kind == "string" || kind == "multiline") { + scan() + } else if (kind == "bare") { + # Scalars such as numbers and dates contribute no nested facts + while (kind == "bare" || kind == ".") scan() + } else { + ok = 0 + return "" + } + return prefix facts +} + +# Print facts in source order, keeping each input name once. +function emit(facts, lines, count, i) { + count = split(facts, lines, "\n") + for (i = 1; i <= count; i++) { + if (lines[i] == "") continue + if (lines[i] ~ /^input / && seen[lines[i]]++) continue + print lines[i] + } +} + +# Normalize CRLF before strings and comments consume their own lines +{ + sub(/\r$/, "") + source = source $0 "\n" +} + +END { + # Keep the current table until a complete header replaces it + position = 1 + ok = 1 + table = "" + scan() + while (kind != "end" && ok) { + skip_newlines() + if (kind == "end") break + if (kind == "[") { + scan() + array_table = kind == "[" + if (array_table) scan() + table = read_key() + if (!ok || kind != "]") break + scan() + if (array_table) { + if (kind != "]") break + scan() + table = "?" + } + if (kind != "\n" && kind != "end") break + emit(input_fact(table)) + } else { + key = read_key() + if (!ok || kind != "=") break + scan() + facts = read_value(table == "" ? key : table SUBSEP key) + if (!ok || (kind != "\n" && kind != "end")) break + emit(facts) + } + } +} diff --git a/tools/python_embed.c b/tools/python_embed.c index 1bb8534..43dc560 100644 --- a/tools/python_embed.c +++ b/tools/python_embed.c @@ -24,6 +24,12 @@ static PyStatus configure(PyConfig *config, int argc, char **argv) { if (PyStatus_Exception(status)) { return status; } + + // Manifests and reports use UTF-8 even when WASI supplies an ASCII locale + status = PyConfig_SetString(config, &config->stdio_encoding, L"utf-8"); + if (PyStatus_Exception(status)) { + return status; + } return PyConfig_SetString(config, &config->program_name, L"ark-python"); } diff --git a/tools/run.sh b/tools/run.sh index 8a210d2..4c0f9aa 100644 --- a/tools/run.sh +++ b/tools/run.sh @@ -1,38 +1,75 @@ #!/bin/sh -# Run an Ark app the way the device does: read its manifest, mount only the -# datasets it declares (read-only), then run it with the data directory as its -# first argument. Invoked by `make run`. See docs/05-running.md. +# Run an Ark app with its required datasets and available optional datasets +# mounted read-only. OPTIONAL=off leaves every optional dataset unmounted. +# Declared inputs come from answer files, copied read-only without validation. +# Start the report pass with the data directory as its first argument. +# A missing required dataset exits with status 3; app failures exit with 1. +# Invoked by `make run`. See docs/05-running.md. # -# Usage: tools/run.sh +# Usage: tools/run.sh [inputs-dir] set -eu wasm="$1" fixtures="$2" +inputs="${3:-}" wasmtime="${WASMTIME:-wasmtime}" -# The manifest pass runs with no arguments, so capture the TOML it prints. -manifest="$("$wasmtime" "$wasm")" +# Capture the TOML from the manifest pass, which runs with no arguments +manifest="$("$wasmtime" "$wasm")" || exit 1 echo "== manifest pass ==" printf '%s\n' "$manifest" -# Mount only the datasets the manifest declares, each at its own path, so the app -# sees exactly what it asked for and nothing else, like the device does. -# wasmtime lets a guest write to every directory it mounts, so the datasets are -# copied into a scratch tree without write permission and mounted from there. +# wasmtime lets a guest write to every directory it mounts, so grants and +# inputs are mounted from copies in a scratch tree without write permission stage="$(mktemp -d)" trap 'chmod -R u+w "$stage"; rm -rf "$stage"' EXIT + +# Read grants and input names through the shared manifest reader +printf '%s\n' "$manifest" | awk -f "$(dirname "$0")/manifest.awk" > "$stage/facts" +sed -n '/^paths /p; /^optional /p' "$stage/facts" > "$stage/grants" +sed -n 's/^input //p' "$stage/facts" > "$stage/input-names" + +# Mount each accepted grant at its own path from a copy without write permission set -- -for dataset in $(printf '%s\n' "$manifest" | grep -oE '"v1/[^"]*"' | tr -d '"'); do +while read -r grant dataset; do + if [ "$grant" = optional ]; then + if [ "${OPTIONAL:-}" = off ]; then + printf 'optional dataset left unmounted, OPTIONAL=off: %s\n' "$dataset" >&2 + continue + fi + if [ ! -d "$fixtures/$dataset" ]; then + printf 'optional dataset left unmounted, not in this fixture root: %s\n' "$dataset" >&2 + continue + fi + fi if [ ! -d "$fixtures/$dataset" ]; then printf 'fixture root has no declared dataset: %s\n' "$dataset" >&2 - exit 1 + exit 3 fi mkdir -p "$stage/$(dirname "$dataset")" cp -R "$fixtures/$dataset" "$stage/$dataset" set -- "$@" --dir "$stage/$dataset::/$dataset" -done +done < "$stage/grants" + +# Copy only declared answers and mount them together under the data root +if [ -s "$stage/input-names" ]; then + if [ ! -d "$inputs" ]; then + printf 'declared inputs require an answers directory as the third argument: %s\n' "${inputs:-none provided}" >&2 + exit 1 + fi + mkdir -p "$stage/inputs" + while read -r name; do + if [ ! -f "$inputs/$name" ]; then + printf 'answers directory has no file for declared input: %s\n' "$name" >&2 + exit 1 + fi + cp "$inputs/$name" "$stage/inputs/$name" + done < "$stage/input-names" + set -- "$@" --dir "$stage/inputs::/inputs" +fi chmod -R a-w "$stage" +# Start the report pass with the accepted grants, inputs and the data root echo -echo "== run pass ==" -"$wasmtime" run "$@" "$wasm" / +echo "== report pass ==" +"$wasmtime" run "$@" "$wasm" / || exit 1