From 078609f188d7158c85d6b783678468ae9e998c47 Mon Sep 17 00:00:00 2001 From: Quentin Date: Thu, 8 Oct 2026 22:10:37 +0200 Subject: [PATCH] Count an infra start from the request, call a booting container starting, let a start finish once accepted, and name what an export sets - A copy being started counts from when the start was asked (the setup run or the apply command), the earlier of that and its row's stamp, and the duration is read on the install's clock (ApplyProgress and unanswered calls carry as_of_unix), so a reader's clock drift no longer shifts it. - A running container whose readiness check fails within its startup window (initial delay plus period times failure threshold, from the probe) reads as starting, not as the raw error; past it the raw error shows as before. - ctx.infra(..).request_start() returns once weft accepts the start; start() is that plus the wait until running. StartInstanceInfra gains waitUntilRunning (default on), so a route can start and then reply 202 with the copy already listed as provisioning. Docs and Tangle teach start, then reply. - weft target export names each secret it set (never a value), and the next-step hint adds --front-env front/.env.prod when that file exists, or says what the flag is for. --- .../start_instance_infra/metadata.json | 11 +- catalog/instances/start_instance_infra/mod.rs | 11 +- .../instances/start_instance_infra/tests.rs | 36 +++- crates/weft-cli/src/commands/frontend.rs | 11 +- crates/weft-cli/src/commands/infra.rs | 10 +- crates/weft-cli/src/commands/status.rs | 7 +- crates/weft-cli/src/commands/target.rs | 21 +- crates/weft-core/src/context/program_calls.rs | 52 +++-- crates/weft-core/src/infra/wire.rs | 51 +++-- crates/weft-core/src/program.rs | 8 +- crates/weft-core/src/projects.rs | 7 +- crates/weft-dispatcher/src/api/infra.rs | 60 +++--- crates/weft-dispatcher/src/api/project.rs | 37 ++-- crates/weft-dispatcher/src/infra_node.rs | 184 ++++++++++++++++-- crates/weft-dispatcher/tests/db_lifecycle.rs | 62 ++++++ crates/weft-platform-local/src/docker.rs | 24 +++ crates/weft-platform-local/src/infra_host.rs | 57 +++++- crates/weft-task-store/src/unanswered.rs | 10 +- docs/src/running/instances.md | 9 +- extension-vscode/package.json | 2 +- packages/weft-graph/src/protocol.ts | 5 +- packages/weft-graph/src/status.test.ts | 35 +++- packages/weft-graph/src/status.ts | 16 +- .../.claude/skills/weft-instances/SKILL.md | 20 +- .../.claude/skills/weft-members/SKILL.md | 9 +- .../.cline/skills/weft-instances/SKILL.md | 20 +- .../cline/.cline/skills/weft-members/SKILL.md | 9 +- .../.agents/skills/weft-instances/SKILL.md | 20 +- .../.agents/skills/weft-members/SKILL.md | 9 +- .../.cursor/skills/weft-instances/SKILL.md | 20 +- .../.cursor/skills/weft-members/SKILL.md | 9 +- .../.windsurf/skills/weft-instances/SKILL.md | 20 +- .../.windsurf/skills/weft-members/SKILL.md | 9 +- .../.agents/skills/weft-instances/SKILL.md | 20 +- .../.agents/skills/weft-members/SKILL.md | 9 +- .../.gemini/skills/weft-instances/SKILL.md | 20 +- .../.gemini/skills/weft-members/SKILL.md | 9 +- .../.github/skills/weft-instances/SKILL.md | 20 +- .../.github/skills/weft-members/SKILL.md | 9 +- .../.junie/skills/weft-instances/SKILL.md | 20 +- .../junie/.junie/skills/weft-members/SKILL.md | 9 +- .../.kilo/skills/weft-instances/SKILL.md | 20 +- .../.kilo/skills/weft-members/SKILL.md | 9 +- .../.opencode/skills/weft-instances/SKILL.md | 20 +- .../.opencode/skills/weft-members/SKILL.md | 9 +- 45 files changed, 810 insertions(+), 235 deletions(-) diff --git a/catalog/instances/start_instance_infra/metadata.json b/catalog/instances/start_instance_infra/metadata.json index 3deec13a..36bc7283 100644 --- a/catalog/instances/start_instance_infra/metadata.json +++ b/catalog/instances/start_instance_infra/metadata.json @@ -1,7 +1,7 @@ { "type": "StartInstanceInfra", "label": "Start instance infra", - "description": "Bring up one instance's copy of a `@per_instance` infra node; `done` fires once it answers, so the instance's triggers reading it can be activated next. The run waits without holding a worker; a copy that fails to come up fails the node with the reason; an already-running copy is left as is.", + "description": "Bring up one instance's copy of a `@per_instance` infra node; `done` fires once it answers, so the instance's triggers reading it can be activated next. The run waits without holding a worker; a copy that fails to come up fails the node with the reason; an already-running copy is left as is. If you want to answer a caller fast (a route replying 202), turn `waitUntilRunning` off and reply after this node: it finishes as soon as weft accepts the start, and a status read right after shows the copy as `provisioning`.", "tags": [ "instances", "infra", @@ -21,13 +21,20 @@ "type": "String", "required": true, "description": "The instance id your program gives it (`user-42`)." + }, + { + "name": "waitUntilRunning", + "type": "Boolean", + "default": true, + "label": "Wait until running", + "description": "On: `done` fires once the copy runs, and a copy that fails to come up fails the node. Off: `done` fires once weft accepts the start (a start refused for a passing reason, such as a build in progress, is asked again until it is), and whether the copy then comes up is for a status read to tell." } ], "outputs": [ { "name": "done", "type": "Boolean", - "description": "True once the copy is running." + "description": "True once the copy is running, or once weft accepted the start when `waitUntilRunning` is off." } ], "validate": [ diff --git a/catalog/instances/start_instance_infra/mod.rs b/catalog/instances/start_instance_infra/mod.rs index befa85b4..52039684 100644 --- a/catalog/instances/start_instance_infra/mod.rs +++ b/catalog/instances/start_instance_infra/mod.rs @@ -1,5 +1,6 @@ //! StartInstanceInfra: bring up one instance's copy of a `@per_instance` -//! infra node, and fire `done` once the copy answers. +//! infra node, and fire `done` once the copy answers, or, with +//! `waitUntilRunning` off, once weft accepted the start. use async_trait::async_trait; @@ -23,7 +24,13 @@ impl Node for StartInstanceInfraNode { async fn run(&self, ctx: ExecutionContext) -> WeftResult<()> { let node: String = ctx.inputs.get("node")?; - ctx.infra(node).instance(instance(&ctx)?).start().await?; + let wait_until_running: bool = ctx.inputs.get("waitUntilRunning")?; + let copy = ctx.infra(node).instance(instance(&ctx)?); + if wait_until_running { + copy.start().await?; + } else { + copy.request_start().await?; + } ctx.pulse_downstream(NodeOutput::new().set("done", true)).await } } diff --git a/catalog/instances/start_instance_infra/tests.rs b/catalog/instances/start_instance_infra/tests.rs index 5e175b8d..841358de 100644 --- a/catalog/instances/start_instance_infra/tests.rs +++ b/catalog/instances/start_instance_infra/tests.rs @@ -2,8 +2,9 @@ //! copy of that node and `done` waits for the copy to run, a copy that //! fails fails the node, a copy somebody stops (or that never came to be) //! while the node waits fails it instead of being started again, a start -//! refused for a passing reason is asked again, and a blank instance is -//! refused before anything is asked. +//! refused for a passing reason is asked again, a node told not to wait +//! finishes once the start is accepted without looking at the copy, and a +//! blank instance is refused before anything is asked. use serde_json::json; @@ -20,6 +21,8 @@ pub fn tests() -> Vec { NodeTest::fake("a_copy_stopped_while_waiting_fails_the_node", stopped_while_waiting), NodeTest::fake("a_start_that_ends_without_a_copy_fails_the_node", no_copy), NodeTest::fake("a_start_refused_for_now_is_asked_again", refused_for_now), + NodeTest::fake("without_waiting_finishes_once_the_start_is_accepted", without_waiting), + NodeTest::fake("without_waiting_a_start_refused_for_now_is_asked_again", without_waiting_refused_for_now), NodeTest::fake("a_blank_instance_is_refused", blank_instance), ] } @@ -101,3 +104,32 @@ async fn refused_for_now(rig: FakeRig) -> WeftResult<()> { assert_eq!(outcome.outputs["done"], json!(true)); Ok(()) } + +async fn without_waiting(rig: FakeRig) -> WeftResult<()> { + let outcome = rig + .run(&StartInstanceInfraNode, json!({ "node": "bridge", "instance": "user-42", "waitUntilRunning": false })) + .await + .ok()?; + let calls = rig.program_calls(); + assert_eq!(calls.len(), 1, "the start only, no look at the copy: {calls:?}"); + assert!(matches!(&calls[0], (ProgramCall::InfraStart { .. }, StopSelf::Keep)), "{calls:?}"); + assert!(rig.awaited_signals().is_empty()); + assert_eq!(outcome.outputs["done"], json!(true)); + Ok(()) +} + +async fn without_waiting_refused_for_now(rig: FakeRig) -> WeftResult<()> { + rig.answer_program_call("weft.infra.start", json!({ "answer": "waiting", "reason": "project is building" })); + rig.answer_program_call("weft.infra.start", json!({ "answer": "already_starting" })); + rig.signal(json!({})); + let outcome = rig + .run(&StartInstanceInfraNode, json!({ "node": "bridge", "instance": "user-42", "waitUntilRunning": false })) + .await + .ok()?; + let calls = rig.program_calls(); + assert_eq!(calls.len(), 2, "asked again after the wait, then done without a look: {calls:?}"); + assert!(calls.iter().all(|(call, _)| matches!(call, ProgramCall::InfraStart { .. })), "{calls:?}"); + assert_eq!(rig.awaited_signals().len(), 1); + assert_eq!(outcome.outputs["done"], json!(true)); + Ok(()) +} diff --git a/crates/weft-cli/src/commands/frontend.rs b/crates/weft-cli/src/commands/frontend.rs index fd95e47d..7155fbf6 100644 --- a/crates/weft-cli/src/commands/frontend.rs +++ b/crates/weft-cli/src/commands/frontend.rs @@ -154,11 +154,20 @@ fn hand_over(ctx: &Ctx, project: &str, install_url: &str, f: &Frontend, token: O if let Some(url) = &f.url { println!("visitors reach it at {url}"); } + let project = ctx.project()?; println!( "next, in {}: {} hands its deploy workflow the service and a token", repo.name, - super::target::export_command(ctx.project()?, ctx.on()) + super::target::export_command(project, ctx.on()) ); + if !project.root.join(super::target::FRONT_ENV_FILE).is_file() { + println!( + "if the frontend needs settings of its own (its database address, a sign-in secret), write them to \ + {} and add `--front-env {}`", + super::target::FRONT_ENV_FILE, + super::target::FRONT_ENV_FILE + ); + } return Ok(()); } // A frontend running elsewhere reaches the install at its public diff --git a/crates/weft-cli/src/commands/infra.rs b/crates/weft-cli/src/commands/infra.rs index a320609b..2fb9967f 100644 --- a/crates/weft-cli/src/commands/infra.rs +++ b/crates/weft-cli/src/commands/infra.rs @@ -624,9 +624,8 @@ async fn infra_status(ctx: &Ctx) -> Result<()> { eprintln!("warning: {note}"); } // Printed apart from the copies: a start stuck this way has none yet. - let now = crate::progress::now_unix() as i64; for entry in &status.unanswered { - eprintln!("warning: {}", entry.describe(now)); + eprintln!("warning: {}", entry.describe()); } Ok(()) } @@ -644,15 +643,14 @@ async fn changing_now(client: &crate::client::DispatcherClient, project_id: &str }; // A call of weft's own that keeps failing comes first: it is why // nothing below moves. - let now = crate::progress::now_unix() as i64; - let unanswered = status.unanswered.iter().map(|u| u.describe(now)); + let unanswered = status.unanswered.iter().map(|u| u.describe()); let copies = status.nodes.iter().filter_map(|n| { let node = match &n.instance { Some(instance) => format!("{} (instance {instance})", n.node), None => n.node.clone(), }; if let Some(progress) = &n.progress { - return Some(format!("{node}: {} {}", n.status, progress.describe_now())); + return Some(format!("{node}: {} {}", n.status, progress.describe(progress.as_of_unix))); } let changing = matches!( InfraNodeStatus::parse(&n.status), @@ -695,7 +693,7 @@ fn print_status(name: &str, id: &str, status: &InfraStatus) { let url = n.endpoint_url.as_deref().unwrap_or("(no endpoint)"); println!(" {node} [{}] -> {url}", n.status); if let Some(progress) = &n.progress { - println!(" {}", progress.describe_now()); + println!(" {}", progress.describe(progress.as_of_unix)); } // A public endpoint's outside address: what to hand to whoever // calls in (the node declared only its own path). diff --git a/crates/weft-cli/src/commands/status.rs b/crates/weft-cli/src/commands/status.rs index 1d95c202..bc2c984e 100644 --- a/crates/weft-cli/src/commands/status.rs +++ b/crates/weft-cli/src/commands/status.rs @@ -176,7 +176,7 @@ pub async fn run(ctx: Ctx) -> Result<()> { st => { println!(" {node}: {st} ({})", entry.endpoint_url.as_deref().unwrap_or("-")); if let Some(progress) = &entry.progress { - println!(" {}", progress.describe_now()); + println!(" {}", progress.describe(progress.as_of_unix)); } } } @@ -210,7 +210,7 @@ pub async fn run(ctx: Ctx) -> Result<()> { for entry in &data.instance_infra { println!(" {} (instance {}): {}", entry.node, entry.instance, entry.status); if let Some(progress) = &entry.progress { - println!(" {}", progress.describe_now()); + println!(" {}", progress.describe(progress.as_of_unix)); } } } @@ -237,9 +237,8 @@ pub async fn run(ctx: Ctx) -> Result<()> { // not move. if !data.unanswered.is_empty() { println!(" waiting on weft:"); - let now = crate::progress::now_unix() as i64; for entry in &data.unanswered { - println!(" {}", entry.describe(now)); + println!(" {}", entry.describe()); } } // What each trigger started lately: the one trace a run kept diff --git a/crates/weft-cli/src/commands/target.rs b/crates/weft-cli/src/commands/target.rs index 8266e2ef..e9f18789 100644 --- a/crates/weft-cli/src/commands/target.rs +++ b/crates/weft-cli/src/commands/target.rs @@ -376,9 +376,11 @@ async fn export( for (k, v) in &settings.variables { println!(" variable {k}={v}"); } - // Only how many: nothing that comes out of a secret reaches the - // terminal, its name included. - println!(" secrets {} set", settings.secrets.len()); + // The names only: they are weft's own fixed words, so a person can check + // what the workflow received; a value never reaches the terminal. + for (k, _) in &settings.secrets { + println!(" secret {k}"); + } println!("run its deploy workflow from the Actions tab"); if let Some(frontend) = deployed_before(&minted, frontend) { println!("frontend '{frontend}' keeps its old token working until that run deploys the new one and retires it"); @@ -702,9 +704,15 @@ fn committed_target_matches(committed: &str, name: &str, url: &str) -> Result<() } } +/// Where a project keeps its frontend server's own settings for a cloud +/// target, the file the next export's `--front-env` hands the deploy +/// workflow: in `front/`, which the workflow builds. +pub const FRONT_ENV_FILE: &str = "front/.env.prod"; + /// The `weft target export ... --github` a person runs next, spelled with /// a target they really have: the one this command was given, else the -/// project's only cloud target, else the choice between them. +/// project's only cloud target, else the choice between them. With +/// [`FRONT_ENV_FILE`] in the project, it hands that file over too. pub fn export_command(project: &weft_compiler::project::Project, on: Option<&str>) -> String { let name = match on { Some(on) => on.to_string(), @@ -723,7 +731,10 @@ pub fn export_command(project: &weft_compiler::project::Project, on: Option<&str } } }; - format!("`weft target export {name} --github`") + match project.root.join(FRONT_ENV_FILE).is_file() { + true => format!("`weft target export {name} --github --front-env {FRONT_ENV_FILE}`"), + false => format!("`weft target export {name} --github`"), + } } /// A target name is what people type after `--on`, and a TOML key. diff --git a/crates/weft-core/src/context/program_calls.rs b/crates/weft-core/src/context/program_calls.rs index 51406bde..3d4ad8e0 100644 --- a/crates/weft-core/src/context/program_calls.rs +++ b/crates/weft-core/src/context/program_calls.rs @@ -146,7 +146,8 @@ impl InfraCalls<'_> { } /// Bring the copy up, and return once it answers, so what comes - /// next (activating the triggers that read it, say) can use it. + /// next (activating the triggers that read it, say) can use it: a + /// [`Self::request_start`], then a wait for the copy to run. /// Starting a copy that runs, or is already starting, does not start /// it twice. A copy that fails to come up fails this call with the /// reason, and so does one somebody stops (or terminates) while this @@ -158,25 +159,12 @@ impl InfraCalls<'_> { /// next to their infra never waits on the run that asked for it. pub async fn start(self) -> WeftResult<()> { let instance = instance_id(&self.instance)?; + self.ask_until_accepted(&instance).await?; let node = self.node; - let start = || ProgramCall::InfraStart { node: node.clone(), instance: instance.clone() }; - // Until a start is queued (one refused for a passing reason, a - // build in progress or a trigger of the copy mid-activation, is - // asked again at each look), what the copy reads is no answer to - // this start. - let mut queued = false; loop { - if !queued { - let answer: InfraStartAnswer = self.ctx.program_call(start(), StopSelf::Keep).await?; - queued = answer.queued(); - if !queued { - self.ctx.await_signal(START_LOOK_AGAIN).await?; - continue; - } - } - // A queued start reads `provisioning` from the moment the call - // above answered (the runtime counts the start under way, not - // only what the copy's row says), so every other answer is + // An accepted start reads `provisioning` from the moment the + // runtime accepted it (the runtime counts the start under way, + // not only what the copy's row says), so every other answer is // where the copy really stands. let status = ProgramCall::InfraStatus { node: node.clone(), instance: instance.clone() }; let copy: Option = self.ctx.program_call(status, StopSelf::Keep).await?; @@ -205,6 +193,34 @@ impl InfraCalls<'_> { } } + /// Ask for the copy to come up, and return once weft accepted the + /// start, without waiting for the copy to run. From then on a status + /// read ([`Self::status`], [`Self::copies`]) shows the copy as + /// `provisioning`, so a program answering a caller fast (a route + /// replying 202) replies after this and the caller's next read finds + /// the copy. A start refused for a passing reason (a build in + /// progress, a trigger of the copy mid-activation) is asked again + /// every few seconds until it is accepted; a copy already running or + /// starting is not started twice. Whether the copy then comes up is + /// for a later `status` read to tell: nothing here waits for it. + pub async fn request_start(self) -> WeftResult<()> { + let instance = instance_id(&self.instance)?; + self.ask_until_accepted(&instance).await + } + + /// Ask for the start until weft accepts it, parking on a timer + /// between refusals for a passing reason. + async fn ask_until_accepted(&self, instance: &Option) -> WeftResult<()> { + loop { + let call = ProgramCall::InfraStart { node: self.node.clone(), instance: instance.clone() }; + let answer: InfraStartAnswer = self.ctx.program_call(call, StopSelf::Keep).await?; + if answer.accepted() { + return Ok(()); + } + self.ctx.await_signal(START_LOOK_AGAIN).await?; + } + } + /// Scale the copy down, keeping its disk. `spec` says what happens to /// the triggers reading it and the runs using it; `stop_self` whether /// this run goes too when it uses the copy. Answers whether there was diff --git a/crates/weft-core/src/infra/wire.rs b/crates/weft-core/src/infra/wire.rs index 51333ef9..8f11300c 100644 --- a/crates/weft-core/src/infra/wire.rs +++ b/crates/weft-core/src/infra/wire.rs @@ -337,23 +337,24 @@ pub struct InfraStatusEntry { /// How far a start of an infra copy got: since when it runs, and what it /// waits on right now, in the host's words ("its machine's agent does not /// answer yet: ..."). `waiting` is absent before the host reports -/// anything (the machine is still being made). +/// anything (the machine is still being made). `since_unix` is when the +/// start was asked, and `as_of_unix` the server's clock when it answered: +/// both on one clock, so how long it has run never depends on the +/// reader's own. // SYNC: ApplyProgress <-> packages/weft-graph/src/protocol.ts ApplyProgress #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ApplyProgress { #[serde(rename = "sinceUnix")] pub since_unix: i64, + #[serde(rename = "asOfUnix")] + pub as_of_unix: i64, #[serde(default, skip_serializing_if = "Option::is_none")] pub waiting: Option, } impl ApplyProgress { - /// The progress a row records, when it records one. - pub fn of(since_unix: Option, waiting: Option) -> Option { - since_unix.map(|since_unix| Self { since_unix, waiting }) - } - - /// As a person reads it, `now_unix` being the time it is read: + /// As a person reads it, `now_unix` being the time it is read, on the + /// clock `since_unix` is on (`as_of_unix` reads it as of the answer): /// "for 3m12s, waiting on: ...". // SYNC: describe <-> packages/weft-graph/src/status.ts describeProgress pub fn describe(&self, now_unix: i64) -> String { @@ -364,15 +365,6 @@ impl ApplyProgress { None => format!("for {elapsed}"), } } - - /// [`Self::describe`], read now. - pub fn describe_now(&self) -> String { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs() as i64) - .unwrap_or(self.since_unix); - self.describe(now) - } } #[cfg(test)] @@ -381,10 +373,29 @@ mod apply_progress_tests { #[test] fn progress_reads_as_how_long_and_on_what() { - let p = ApplyProgress { since_unix: 100, waiting: Some("db: its machine's agent does not answer yet".into()) }; - assert_eq!(p.describe(292), "for 3m12s, waiting on: db: its machine's agent does not answer yet"); - assert_eq!(ApplyProgress { since_unix: 100, waiting: None }.describe(130), "for 30s"); - assert_eq!(ApplyProgress::of(None, Some("x".into())), None, "no start, no progress"); + let p = ApplyProgress { since_unix: 100, as_of_unix: 292, waiting: Some("db: its machine's agent does not answer yet".into()) }; + assert_eq!(p.describe(p.as_of_unix), "for 3m12s, waiting on: db: its machine's agent does not answer yet"); + let bare = ApplyProgress { since_unix: 100, as_of_unix: 130, waiting: None }; + assert_eq!(bare.describe(bare.as_of_unix), "for 30s"); + } + + /// Read as of the answer, the duration is the server's alone: a + /// reader whose clock runs 20s behind the server's still reads 20s. + #[test] + fn progress_as_of_the_answer_ignores_the_readers_clock() { + let p = ApplyProgress { since_unix: 1_000, as_of_unix: 1_020, waiting: None }; + assert_eq!(p.describe(p.as_of_unix), "for 20s"); + assert_eq!(p.describe(990), "for 0s", "a time before the start reads as no time at all"); + } + + #[test] + fn progress_travels_as_camel_case() { + let p = ApplyProgress { since_unix: 1, as_of_unix: 2, waiting: None }; + assert_eq!(serde_json::to_value(&p).unwrap(), serde_json::json!({ "sinceUnix": 1, "asOfUnix": 2 })); + assert!( + serde_json::from_value::(serde_json::json!({ "sinceUnix": 1 })).is_err(), + "an answer without its clock is refused" + ); } } diff --git a/crates/weft-core/src/program.rs b/crates/weft-core/src/program.rs index 38b1fa24..2b50dfb5 100644 --- a/crates/weft-core/src/program.rs +++ b/crates/weft-core/src/program.rs @@ -189,7 +189,8 @@ pub struct WaitingFires { /// What `ProgramCall::InfraStart` answers: a start queued, one already /// under way or done, or none yet for a passing reason (a build in /// progress, a trigger of the copy mid-activation), which -/// `ctx.infra(..).start()` asks again at its next look. +/// `ctx.infra(..).start()` and `.request_start()` ask again at their next +/// look. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "answer", rename_all = "snake_case")] pub enum InfraStartAnswer { @@ -200,8 +201,9 @@ pub enum InfraStartAnswer { } impl InfraStartAnswer { - /// Whether a start is under way: every answer but `Waiting`. - pub fn queued(&self) -> bool { + /// Whether weft accepted the start (it is under way or done): every + /// answer but `Waiting`. + pub fn accepted(&self) -> bool { !matches!(self, InfraStartAnswer::Waiting { .. }) } } diff --git a/crates/weft-core/src/projects.rs b/crates/weft-core/src/projects.rs index e5f03858..20d5a51f 100644 --- a/crates/weft-core/src/projects.rs +++ b/crates/weft-core/src/projects.rs @@ -253,11 +253,16 @@ pub struct Unanswered { pub since_unix: i64, #[serde(rename = "lastUnix")] pub last_unix: i64, + /// The install's clock when this was read, which the times above are + /// on, so how long ago reads right whatever clock the reader has. + #[serde(rename = "asOfUnix")] + pub as_of_unix: i64, } impl Unanswered { /// One line for a person: what is not answering, since when, and why. - pub fn describe(&self, now_unix: i64) -> String { + pub fn describe(&self) -> String { + let now_unix = self.as_of_unix; let span = |secs: i64| match secs.max(0) { secs @ 0..120 => format!("{secs}s"), secs => format!("{} min", secs / 60), diff --git a/crates/weft-dispatcher/src/api/infra.rs b/crates/weft-dispatcher/src/api/infra.rs index e5bc098b..bab46184 100644 --- a/crates/weft-dispatcher/src/api/infra.rs +++ b/crates/weft-dispatcher/src/api/infra.rs @@ -1487,7 +1487,8 @@ pub(crate) async fn live_endpoint_url( /// Every copy as `weft status` and a program's `ctx.infra(..).status()` /// show it (`infra_node::observe`): a start or a stop under way reads as -/// such before the supervisor reaches the copy. +/// such before the supervisor reaches the copy, and a start counts its +/// progress from when it was asked for. async fn read_infra_entries( state: &DispatcherState, project_id: uuid::Uuid, @@ -1501,23 +1502,38 @@ async fn read_infra_entries( let copies = infra_node::observe(&state.pg_pool, project_id, &project) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("infra copies: {e:#}")))?; - let mut entries: Vec = - copies.rows.into_iter().map(|row| row_to_entry(row, state.external_base_url())).collect(); - entries.extend(copies.starting.into_iter().map(|(node, instance)| InfraStatusEntry { - node, - instance, - status: infra_node::InfraNodeStatus::Provisioning.as_str().to_string(), - endpoint_url: None, - public_urls: Default::default(), - failure_stage: None, - failure_message: None, - notes: Vec::new(), - progress: None, - })); - Ok(entries) + let starting: Vec = copies + .starting + .iter() + .map(|(node, instance)| InfraStatusEntry { + node: node.clone(), + instance: instance.clone(), + status: infra_node::InfraNodeStatus::Provisioning.as_str().to_string(), + endpoint_url: None, + public_urls: Default::default(), + failure_stage: None, + failure_message: None, + notes: Vec::new(), + progress: copies.progress_of(node, instance.as_ref()), + }) + .collect(); + let progress: Vec<_> = copies.rows.iter().map(|row| copies.progress_of(&row.node_id, row.instance.as_ref())).collect(); + Ok(copies + .rows + .into_iter() + .zip(progress) + .map(|(row, progress)| row_to_entry(row, progress, state.external_base_url())) + .chain(starting) + .collect()) } -fn row_to_entry(row: InfraNodeRow, front_door: &str) -> InfraStatusEntry { +/// One row as an entry, with the progress [`infra_node::ObservedCopies::progress_of`] +/// gives its copy. +fn row_to_entry( + row: InfraNodeRow, + progress: Option, + front_door: &str, +) -> InfraStatusEntry { InfraStatusEntry { public_urls: row .public_paths @@ -1534,20 +1550,10 @@ fn row_to_entry(row: InfraNodeRow, front_door: &str) -> InfraStatusEntry { failure_stage: row.failure_stage.map(|f| f.as_str().to_string()), failure_message: row.failure_message, notes: row.notes, - progress: provisioning_progress(row.status, row.provisioning_since_unix, row.waiting), + progress, } } -/// The progress a row records, shown only while it provisions: the -/// columns hold the last apply's until the next one rewrites them. -pub(crate) fn provisioning_progress( - status: InfraNodeStatus, - since_unix: Option, - waiting: Option, -) -> Option { - (status == InfraNodeStatus::Provisioning).then(|| weft_core::infra::wire::ApplyProgress::of(since_unix, waiting)).flatten() -} - /// Enqueue a lifecycle command for `project_id`, under its tenant. The /// command's row announces itself (`INFRA_COMMAND_CHANNEL`), which wakes /// the supervisor wherever it runs (`weft_runtime::role_waker` for one at diff --git a/crates/weft-dispatcher/src/api/project.rs b/crates/weft-dispatcher/src/api/project.rs index 6469776e..78da325a 100644 --- a/crates/weft-dispatcher/src/api/project.rs +++ b/crates/weft-dispatcher/src/api/project.rs @@ -1268,6 +1268,7 @@ pub(crate) async fn unanswered_for(state: &DispatcherState, project_id: uuid::Uu error: f.error, since_unix: f.since_ms / 1000, last_unix: f.last_ms / 1000, + as_of_unix: f.as_of_ms / 1000, }) .collect()) } @@ -1598,7 +1599,7 @@ fn infra_entries( node: row.node_id.clone(), instance: row.instance.clone()?, status: row.status.as_str().to_string(), - progress: crate::api::infra::provisioning_progress(row.status, row.provisioning_since_unix, row.waiting.clone()), + progress: copies.progress_of(&row.node_id, row.instance.as_ref()), }) }) .chain(copies.starting.iter().filter_map(|(node, instance)| { @@ -1606,7 +1607,7 @@ fn infra_entries( node: node.clone(), instance: instance.clone()?, status: crate::infra_node::InfraNodeStatus::Provisioning.as_str().to_string(), - progress: None, + progress: copies.progress_of(node, instance.as_ref()), }) })) .collect(); @@ -1640,17 +1641,16 @@ fn infra_entries( endpoint_url: row.install_endpoints.values().next().cloned(), failure_stage: row.failure_stage.map(|f| f.as_str().to_string()), failure_message: row.failure_message.clone(), - progress: crate::api::infra::provisioning_progress( - row.status, - row.provisioning_since_unix, - row.waiting.clone(), - ), + progress: copies.progress_of(&spelled, None), ..entry(row.status.as_str().to_string()) }), - None => infra.push(entry(match copies.status_of(&spelled, None) { - Some(status) => status.as_str().to_string(), - None => INFRA_NOT_STARTED.to_string(), - })), + None => infra.push(match copies.status_of(&spelled, None) { + Some(status) => ProjectInfraEntry { + progress: copies.progress_of(&spelled, None), + ..entry(status.as_str().to_string()) + }, + None => entry(INFRA_NOT_STARTED.to_string()), + }), } } (infra, instance_infra) @@ -5300,8 +5300,11 @@ mod infra_entries_tests { let copies = ObservedCopies { rows: vec![row("db", None, InfraNodeStatus::Running), row("bridge", Some("ada"), InfraNodeStatus::Stopped)], starting: vec![("queue".into(), None), ("bridge".into(), Some(InstanceId::new("bob").unwrap()))], + start_asked_unix: [(("queue".to_string(), None), 100), (("bridge".to_string(), Some(InstanceId::new("bob").unwrap())), 110)] + .into(), + as_of_unix: 130, }; - let (infra, instances) = infra_entries(&project, &copies); + let (infra, instance_entries) = infra_entries(&project, &copies); let listed: Vec<(&str, &str, Option)> = infra.iter().map(|e| (e.node.as_str(), e.status.as_str(), e.instance_copy_count)).collect(); assert_eq!( @@ -5314,8 +5317,16 @@ mod infra_entries_tests { ] ); let instances: Vec<(&str, &str, &str)> = - instances.iter().map(|c| (c.node.as_str(), c.instance.as_str(), c.status.as_str())).collect(); + instance_entries.iter().map(|c| (c.node.as_str(), c.instance.as_str(), c.status.as_str())).collect(); assert_eq!(instances, [("bridge", "ada", "stopped"), ("bridge", "bob", "provisioning")]); + // A copy being started counts from its request, row or no row, as + // of the clock the copies were read on. + let since = |progress: Option<&weft_core::infra::wire::ApplyProgress>| progress.map(|p| (p.since_unix, p.as_of_unix)); + let shared = |node: &str| infra.iter().find(|e| e.node == node).unwrap().progress.as_ref(); + assert_eq!(since(shared("queue")), Some((100, 130)), "a shared copy with no row"); + assert_eq!(since(shared("db")), None, "a running copy shows none"); + let bob = instance_entries.iter().find(|e| e.instance.as_str() == "bob").unwrap(); + assert_eq!(since(bob.progress.as_ref()), Some((110, 130)), "an instance's copy with no row"); } /// A program with no infra node lists none, which is the one case the diff --git a/crates/weft-dispatcher/src/infra_node.rs b/crates/weft-dispatcher/src/infra_node.rs index 8be0be4e..21ff79a6 100644 --- a/crates/weft-dispatcher/src/infra_node.rs +++ b/crates/weft-dispatcher/src/infra_node.rs @@ -277,6 +277,10 @@ pub struct PendingOp { /// stop of the copy cancels it, so its applies never land after the /// stop). pub setup: Option, + /// When it was asked for (unix seconds): a command's + /// `issued_at_unix`, a setup run's `started_at`. A start counts its + /// progress from it. + pub asked_unix: i64, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -319,6 +323,24 @@ impl PendingOps { } } + /// When the start under way for this copy was asked for, `None` when + /// no start is what is under way. Of the starts carried out after the + /// copy's last stop or terminate, the earliest: a setup run issues + /// its applies once it runs, so the run asked first. + pub fn start_asked_unix(&self, node: &str, instance: Option<&weft_core::instance::InstanceId>) -> Option { + if !self.starting(node, instance) { + return None; + } + let covering = || { + self.ops.iter().filter(|op| op.node.as_deref().is_none_or(|n| n == node) && op.copies.admits(instance)) + }; + let last_taken_down = covering().filter(|op| op.kind != PendingKind::Start).map(|op| op.order).max(); + covering() + .filter(|op| op.kind == PendingKind::Start && last_taken_down.is_none_or(|order| op.order > order)) + .map(|op| op.asked_unix) + .min() + } + /// Whether a start of this copy is under way. pub fn starting(&self, node: &str, instance: Option<&weft_core::instance::InstanceId>) -> bool { self.status_of(node, instance, None) == Some(InfraNodeStatus::Provisioning) @@ -371,6 +393,13 @@ pub struct ObservedCopies { /// Copies with no row yet that a start is bringing up, as `(place, /// instance)`: they read `provisioning`. pub starting: Vec<(String, Option)>, + /// For each copy a start is bringing up (`(place, instance)`), when + /// that start was asked for ([`PendingOps::start_asked_unix`]). + pub start_asked_unix: BTreeMap<(String, Option), i64>, + /// The database's clock when this was read (unix seconds): the clock + /// a row's and a command's stamps are on (a setup run's `started_at` + /// is on the clock of whoever began it). + pub as_of_unix: i64, } impl ObservedCopies { @@ -385,6 +414,31 @@ impl ObservedCopies { .any(|(n, m)| n == node && m.as_ref() == instance) .then_some(InfraNodeStatus::Provisioning) } + + /// How far the start of one copy got, while it reads `provisioning`: + /// counted from when the start was asked for, or from when the + /// supervisor began the apply under way (`provisioning_since_unix`) + /// when that came first (a start asked again while an earlier apply + /// still runs), read as of [`Self::as_of_unix`]. `None` for a copy + /// that is not starting, and for a provisioning row stamped by + /// nothing (one written before the column existed). + pub fn progress_of( + &self, + node: &str, + instance: Option<&weft_core::instance::InstanceId>, + ) -> Option { + if self.status_of(node, instance) != Some(InfraNodeStatus::Provisioning) { + return None; + } + let row = self.rows.iter().find(|r| r.node_id == node && r.instance.as_ref() == instance); + let asked = self.start_asked_unix.get(&(node.to_string(), instance.cloned())).copied(); + let since_unix = [row.and_then(|r| r.provisioning_since_unix), asked].into_iter().flatten().min()?; + Some(weft_core::infra::wire::ApplyProgress { + since_unix, + as_of_unix: self.as_of_unix, + waiting: row.and_then(|r| r.waiting.clone()), + }) + } } /// Read every copy of `project_id` with what is under way applied: the one @@ -403,6 +457,7 @@ pub async fn observe_with( project_id: uuid::Uuid, project: &weft_core::ProjectDefinition, ) -> Result<(ObservedCopies, PendingOps)> { + let (as_of_unix,): (i64,) = sqlx::query_as("SELECT EXTRACT(EPOCH FROM NOW())::BIGINT").fetch_one(pool).await?; let mut rows = list_for_project(pool, project_id).await?; let pending = pending_ops(pool, project_id, project).await?; for row in &mut rows { @@ -411,7 +466,13 @@ pub async fn observe_with( } } let starting = pending.starting_without_row(&rows); - Ok((ObservedCopies { rows, starting }, pending)) + let start_asked_unix = rows + .iter() + .map(|r| (r.node_id.clone(), r.instance.clone())) + .chain(starting.iter().cloned()) + .filter_map(|copy| pending.start_asked_unix(©.0, copy.1.as_ref()).map(|asked| (copy, asked))) + .collect(); + Ok((ObservedCopies { rows, starting, start_asked_unix, as_of_unix }, pending)) } /// Read what is under way for `project_id`'s copies (see [`PendingOps`]). @@ -422,17 +483,17 @@ pub async fn pending_ops( project: &weft_core::ProjectDefinition, ) -> Result { let mut ops = Vec::new(); - /// id, node_id, verb, instance_id, every_copy. - type CommandRow = (i64, Option, String, Option, bool); + /// id, node_id, verb, instance_id, every_copy, issued_at_unix. + type CommandRow = (i64, Option, String, Option, bool, i64); let commands: Vec = sqlx::query_as(&format!( - "SELECT id, node_id, verb, instance_id, every_copy FROM infra_lifecycle_command \ + "SELECT id, node_id, verb, instance_id, every_copy, issued_at_unix FROM infra_lifecycle_command \ WHERE project_id = $1 AND completed_at_unix IS NULL AND verb IN ({verbs})", verbs = weft_broker_client::lifecycle_command::SUPERVISOR_VERBS_SQL, )) .bind(project_id) .fetch_all(pool) .await?; - for (id, node, verb, instance, every) in commands { + for (id, node, verb, instance, every, issued_at_unix) in commands { let kind = match crate::infra_lifecycle_command::InfraLifecycleVerb::parse(&verb) { Some(crate::infra_lifecycle_command::InfraLifecycleVerb::Apply) => PendingKind::Start, Some(crate::infra_lifecycle_command::InfraLifecycleVerb::Stop) => PendingKind::Stop, @@ -441,21 +502,23 @@ pub async fn pending_ops( }; let copies = weft_core::instance::Copies::from_columns(instance, every) .map_err(|e| anyhow::anyhow!("infra_lifecycle_command {id}: {e}"))?; - ops.push(PendingOp { kind, node, copies, order: id, setup: None }); + ops.push(PendingOp { kind, node, copies, order: id, setup: None, asked_unix: issued_at_unix }); } // A setup run that has not ended brings up the infra places its // selection holds, for its instance. let infra: std::collections::BTreeSet = weft_core::project::infra_place_spellings(project); - let setups: Vec<(weft_core::ExecutionId, Option, Option>)> = - sqlx::query_as( - "SELECT r.execution_id, r.instance_id, s.selection FROM run r \ - LEFT JOIN run_selection s ON s.digest = r.selection \ - WHERE r.project_id = $1 AND r.phase = 'infra_setup' AND r.state <> 'ended'", - ) - .bind(project_id) - .fetch_all(pool) - .await?; - for (execution_id, instance, subgraph) in setups { + /// execution_id, instance_id, started_at, selection. + type SetupRow = + (weft_core::ExecutionId, Option, i64, Option>); + let setups: Vec = sqlx::query_as( + "SELECT r.execution_id, r.instance_id, r.started_at, s.selection FROM run r \ + LEFT JOIN run_selection s ON s.digest = r.selection \ + WHERE r.project_id = $1 AND r.phase = 'infra_setup' AND r.state <> 'ended'", + ) + .bind(project_id) + .fetch_all(pool) + .await?; + for (execution_id, instance, started_at, subgraph) in setups { let instance = instance .map(weft_core::instance::InstanceId::new) .transpose() @@ -472,6 +535,7 @@ pub async fn pending_ops( copies: weft_core::instance::Copies::of(instance.clone()), order: i64::MAX, setup: Some(execution_id), + asked_unix: started_at, }); } } @@ -623,7 +687,11 @@ mod tests { } fn op(kind: PendingKind, node: Option<&str>, copies: weft_core::instance::Copies, order: i64) -> PendingOp { - PendingOp { kind, node: node.map(str::to_string), copies, order, setup: None } + asked(kind, node, copies, order, 0) + } + + fn asked(kind: PendingKind, node: Option<&str>, copies: weft_core::instance::Copies, order: i64, asked_unix: i64) -> PendingOp { + PendingOp { kind, node: node.map(str::to_string), copies, order, setup: None, asked_unix } } fn ada() -> weft_core::instance::InstanceId { @@ -698,6 +766,88 @@ mod tests { assert_eq!(ops.starting_without_row(&[]), vec![("bridge".to_string(), Some(ada()))]); } + /// A start counts from when it was asked: the earliest start carried + /// out after the copy's last stop (a setup run asked before the apply + /// it issued), never one a later stop overrode, and nothing at all + /// while a stop is what is under way. + #[test] + fn a_start_is_asked_when_its_earliest_live_request_was() { + use weft_core::instance::Copies; + let instance = Copies::Instance(ada()); + let resumed = PendingOps::new(vec![ + asked(PendingKind::Start, Some("bridge"), instance.clone(), 3, 50), + asked(PendingKind::Stop, Some("bridge"), instance.clone(), 7, 60), + asked(PendingKind::Start, Some("bridge"), instance.clone(), 9, 80), + asked(PendingKind::Start, Some("bridge"), instance.clone(), i64::MAX, 70), + ]); + assert_eq!(resumed.start_asked_unix("bridge", Some(&ada())), Some(70), "the setup run, asked before its apply"); + assert_eq!(resumed.start_asked_unix("other", Some(&ada())), None, "nothing starts another place"); + let paused = PendingOps::new(vec![ + asked(PendingKind::Start, Some("bridge"), instance.clone(), 8, 50), + asked(PendingKind::Stop, Some("bridge"), instance, 9, 60), + ]); + assert_eq!(paused.start_asked_unix("bridge", Some(&ada())), None, "a stop is what is under way"); + } + + fn provisioning_row(since: Option, waiting: Option<&str>) -> InfraNodeRow { + InfraNodeRow { + project_id: uuid::Uuid::nil(), + node_id: "db".into(), + instance: None, + copy_id: String::new(), + status: InfraNodeStatus::Provisioning, + failure_stage: None, + failure_message: None, + applied_spec_hash: None, + applied_at_unix: None, + endpoints: Default::default(), + public_paths: Default::default(), + doors: Default::default(), + install_endpoints: Default::default(), + keep_disks: Vec::new(), + units: Default::default(), + notes: Vec::new(), + waiting: waiting.map(str::to_string), + provisioning_since_unix: since, + } + } + + /// A copy being started counts from the request, row or no row: with + /// no row yet it counts from the request alone, and over a row the + /// supervisor stamped later it still counts from the request (which + /// came first), keeping what the row says it waits on. Every answer + /// carries the clock it was read on. + #[test] + fn a_starting_copy_counts_from_the_request() { + let bob = weft_core::instance::InstanceId::new("bob").unwrap(); + let copies = ObservedCopies { + rows: vec![provisioning_row(Some(120), Some("its machine's agent does not answer yet"))], + starting: vec![("bridge".into(), Some(bob.clone()))], + start_asked_unix: [(("db".to_string(), None), 100), (("bridge".to_string(), Some(bob.clone())), 90)].into(), + as_of_unix: 130, + }; + let db = copies.progress_of("db", None).expect("the row provisions"); + assert_eq!((db.since_unix, db.as_of_unix), (100, 130)); + assert_eq!(db.waiting.as_deref(), Some("its machine's agent does not answer yet")); + let bridge = copies.progress_of("bridge", Some(&bob)).expect("a start with no row yet"); + assert_eq!((bridge.since_unix, bridge.as_of_unix, bridge.waiting), (90, 130, None)); + assert_eq!(copies.progress_of("cache", None), None, "nothing starts it"); + } + + /// A row the supervisor already stamped, with no request under way + /// any more, counts from its own stamp; a copy that is up shows none. + #[test] + fn a_provisioning_row_alone_counts_from_its_stamp() { + let copies = ObservedCopies { rows: vec![provisioning_row(Some(120), None)], as_of_unix: 130, ..Default::default() }; + assert_eq!(copies.progress_of("db", None).map(|p| p.since_unix), Some(120)); + let running = ObservedCopies { + rows: vec![InfraNodeRow { status: InfraNodeStatus::Running, ..provisioning_row(Some(120), None) }], + as_of_unix: 130, + ..Default::default() + }; + assert_eq!(running.progress_of("db", None), None); + } + #[test] fn failure_stage_as_str() { assert_eq!(FailureStage::Provision.as_str(), "provision"); diff --git a/crates/weft-dispatcher/tests/db_lifecycle.rs b/crates/weft-dispatcher/tests/db_lifecycle.rs index f30fe79b..8bd56c0b 100644 --- a/crates/weft-dispatcher/tests/db_lifecycle.rs +++ b/crates/weft-dispatcher/tests/db_lifecycle.rs @@ -1325,6 +1325,68 @@ async fn copies_read_with_the_commands_under_way(pool: PgPool) { assert_eq!(copies.status_of("bridge", None), None, "no shared copy, nothing starting one"); } +/// A copy being started counts from when the start was asked for, before +/// the supervisor writes any row: an instance's copy with no row yet, and +/// one restarted over its stopped row (which holds no stamp of its own), +/// both read as provisioning since their apply command was issued, on the +/// database's clock. +#[sqlx::test] +async fn a_starting_copy_counts_from_its_request(pool: PgPool) { + use weft_broker::lifecycle_writes::{issue_command, IssuedCommand}; + use weft_dispatcher::infra_lifecycle_command::InfraLifecycleVerb; + use weft_dispatcher::infra_node::observe; + let (_journal, projects) = setup(&pool).await; + let project = Uuid::new_v4(); + seed_project(&projects, project, "bin-A").await; + sqlx::query("INSERT INTO infra_node (project_id, node_id, instance_id, copy_id, status) VALUES ($1, 'bridge', 'ada', 'inst', 'stopped')") + .bind(project) + .execute(&pool) + .await + .expect("seed ada's stopped copy"); + let ada = InstanceId::new("ada").unwrap(); + let bob = InstanceId::new("bob").unwrap(); + let mut issued = Vec::new(); + for (instance, asked_unix) in [(&ada, 1_000_i64), (&bob, 2_000)] { + let spec = json!({}); + let apply = IssuedCommand { + tenant_id: TENANT, + project_id: project, + node_id: Some("bridge"), + copies: &weft_core::instance::Copies::Instance(instance.clone()), + verb: InfraLifecycleVerb::Apply, + spec_json: Some(&spec), + issued_by_replica: "worker-1", + }; + let id = issue_command(&pool, &apply).await.unwrap().expect("the project row is there"); + // Asked well in the past, so the count cannot pass by luck. + sqlx::query("UPDATE infra_lifecycle_command SET issued_at_unix = $2 WHERE id = $1") + .bind(id) + .bind(asked_unix) + .execute(&pool) + .await + .unwrap(); + issued.push(asked_unix); + } + + let copies = observe(&pool, project, &empty_project(project)).await.unwrap(); + let (db_now,): (i64,) = sqlx::query_as("SELECT EXTRACT(EPOCH FROM NOW())::BIGINT").fetch_one(&pool).await.unwrap(); + let restarted = copies.progress_of("bridge", Some(&ada)).expect("a restart over a stopped row shows progress"); + assert_eq!(restarted.since_unix, issued[0]); + let fresh = copies.progress_of("bridge", Some(&bob)).expect("a start with no row yet shows progress"); + assert_eq!(fresh.since_unix, issued[1]); + assert_eq!(fresh.waiting, None, "nothing has reported what it waits on yet"); + assert!(fresh.as_of_unix <= db_now && fresh.as_of_unix > issued[1], "read on the database's clock: {}", fresh.as_of_unix); + + // Once the supervisor stamps the row later, the request still counts. + sqlx::query("UPDATE infra_node SET status = 'provisioning', provisioning_since_unix = 5000 WHERE project_id = $1") + .bind(project) + .execute(&pool) + .await + .unwrap(); + let copies = observe(&pool, project, &empty_project(project)).await.unwrap(); + assert_eq!(copies.progress_of("bridge", Some(&ada)).map(|p| p.since_unix), Some(issued[0]), "the request came first"); +} + /// What holds the program's own verbs back is infra work on the copies /// the bar starts and stops: an instance's copy coming up holds nothing /// back, the shared copies' work does, and so does the project going. diff --git a/crates/weft-platform-local/src/docker.rs b/crates/weft-platform-local/src/docker.rs index 557bdf76..184ba4e4 100644 --- a/crates/weft-platform-local/src/docker.rs +++ b/crates/weft-platform-local/src/docker.rs @@ -128,6 +128,22 @@ pub async fn containers_using(docker: &dyn Docker, volume: &str) -> anyhow::Resu Ok(out.lines().map(str::trim).filter(|l| !l.is_empty()).map(str::to_string).collect()) } +/// When the container `name` last started (`State.StartedAt`, on the +/// daemon's clock): a restart moves it. `docker ps` prints only a +/// rounded "Up 2 minutes", so this asks `docker inspect`. +pub async fn started_at(docker: &dyn Docker, name: &str) -> anyhow::Result> { + let out = run(docker, vec!["inspect".into(), "--format".into(), "{{json .State.StartedAt}}".into(), name.into()]).await?; + parse_started_at(name, &out) +} + +fn parse_started_at(name: &str, printed: &str) -> anyhow::Result> { + let at: String = serde_json::from_str(printed.trim()) + .map_err(|e| anyhow::anyhow!("`docker inspect` printed no start time for {name} ({e}): {}", printed.trim()))?; + Ok(chrono::DateTime::parse_from_rfc3339(&at) + .map_err(|e| anyhow::anyhow!("`docker inspect` printed a start time for {name} that is not RFC 3339 ({e}): {at}"))? + .to_utc()) +} + /// One line of `docker ps --format '{{json .}}'`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ContainerRow { @@ -310,6 +326,14 @@ mod tests { assert!(ContainerRow::parse("not json").is_err()); } + #[test] + fn a_start_time_is_read_from_inspect_or_refused() { + let at = parse_started_at("c-pg", "\"2024-01-01T00:00:05.123456789Z\"\n").unwrap(); + assert_eq!(at, chrono::DateTime::parse_from_rfc3339("2024-01-01T00:00:05.123456789Z").unwrap().to_utc()); + assert!(parse_started_at("c-pg", "\"yesterday\"").is_err()); + assert!(parse_started_at("c-pg", "").is_err()); + } + #[test] fn labels_and_filters_are_spelled_as_docker_takes_them() { let mut l = BTreeMap::new(); diff --git a/crates/weft-platform-local/src/infra_host.rs b/crates/weft-platform-local/src/infra_host.rs index fd351b11..cc1ecb79 100644 --- a/crates/weft-platform-local/src/infra_host.rs +++ b/crates/weft-platform-local/src/infra_host.rs @@ -408,9 +408,10 @@ impl LocalInfraHost { if let Some(ready) = readiness { let answer = self.probe(&agent_url, &c.name, ready).await?; if !answer.ok { - return Ok(UnitRunState::NotReady { - why: format!("container {short}: {}", answer.why.unwrap_or_else(|| "not ready".into())), - }); + let started = docker::started_at(self.docker.as_ref(), &c.name).await?; + let now = daemon_now(self.docker.as_ref()).await?; + let why = answer.why.unwrap_or_else(|| "not ready".into()); + return Ok(readiness_failed(short, &why, now.signed_duration_since(started), ready)); } } } @@ -422,6 +423,28 @@ impl LocalInfraHost { /// after a restart of this process still knows them. const PROBES: &str = "weft.probes"; +/// How long after it starts a container may fail its readiness check +/// and still count as starting: the wait its probe asks before the first +/// check, then one period per failure it tolerates. +fn startup_window(probe: &weft_core::infra::Probe) -> chrono::TimeDelta { + let seconds = u64::from(probe.initial_delay_seconds) + u64::from(probe.period_seconds) * u64::from(probe.failure_threshold.max(1)); + // Two u32 settings multiplied can pass what a duration holds; a + // window that long is a container never declared stuck, so it is + // the longest duration rather than a panic. + i64::try_from(seconds).ok().and_then(chrono::TimeDelta::try_seconds).unwrap_or(chrono::TimeDelta::MAX) +} + +/// What a running container whose readiness check failed with `why` is +/// doing, `up` after it started: still starting inside its +/// [`startup_window`], otherwise not ready, with why. +fn readiness_failed(short: &str, why: &str, up: chrono::TimeDelta, probe: &weft_core::infra::Probe) -> UnitRunState { + if up < startup_window(probe) { + UnitRunState::Starting { step: format!("container {short} has not answered its health check yet") } + } else { + UnitRunState::NotReady { why: format!("container {short}: {why}") } + } +} + fn container_short(c: &ContainerRow) -> &str { c.label("weft.container").unwrap_or(&c.name) } @@ -1278,4 +1301,32 @@ mod tests { assert!(!raw.contains(',')); assert_eq!(decode_probes(&raw).unwrap(), (Some(p), None)); } + + #[test] + fn the_startup_window_is_the_probes_delay_then_a_period_per_tolerated_failure() { + let mut p = Probe::http("/health", 8099); + p.initial_delay_seconds = 5; + p.period_seconds = 10; + p.failure_threshold = 3; + assert_eq!(startup_window(&p), chrono::TimeDelta::seconds(35)); + p.failure_threshold = 0; + assert_eq!(startup_window(&p), chrono::TimeDelta::seconds(15), "a threshold of 0 still allows one failure, as liveness reads it"); + } + + #[test] + fn a_failing_readiness_check_is_starting_inside_the_window_and_not_ready_after() { + let mut p = Probe::http("/health", 8099); + p.initial_delay_seconds = 0; + p.period_seconds = 10; + p.failure_threshold = 3; + let why = "GET /health on 8099: error sending request"; + assert_eq!( + readiness_failed("credential", why, chrono::TimeDelta::seconds(2), &p), + UnitRunState::Starting { step: "container credential has not answered its health check yet".into() } + ); + assert_eq!( + readiness_failed("credential", why, chrono::TimeDelta::seconds(30), &p), + UnitRunState::NotReady { why: format!("container credential: {why}") } + ); + } } diff --git a/crates/weft-task-store/src/unanswered.rs b/crates/weft-task-store/src/unanswered.rs index 3b6679e3..235f629b 100644 --- a/crates/weft-task-store/src/unanswered.rs +++ b/crates/weft-task-store/src/unanswered.rs @@ -67,6 +67,9 @@ pub struct Failing { pub error: String, pub since_ms: i64, pub last_ms: i64, + /// The database's clock when this was read, which the times above are + /// on: how long ago reads right whatever clock the reader has. + pub as_of_ms: i64, } /// A try at `callee` failed with `error`. @@ -93,8 +96,8 @@ pub async fn answered(pool: &PgPool, callee: Callee<'_>) -> Result<()> { /// Every role still failing, and `project`'s workers if they are: what /// the project's work waits on. pub async fn failing_for(pool: &PgPool, project: uuid::Uuid) -> Result> { - let rows: Vec<(String, String, i64, i64)> = sqlx::query_as(&format!( - "SELECT callee, error, since_ms, last_ms FROM unanswered_call \ + let rows: Vec<(String, String, i64, i64, i64)> = sqlx::query_as(&format!( + "SELECT callee, error, since_ms, last_ms, {DB_NOW_MS} FROM unanswered_call \ WHERE (callee LIKE 'role:%' OR callee = $1) AND last_ms >= {DB_NOW_MS} - {RECENT_MS} \ ORDER BY callee" )) @@ -103,11 +106,12 @@ pub async fn failing_for(pool: &PgPool, project: uuid::Uuid) -> Result crates/weft-core/src/infra/wire.rs ApplyProgress export interface ApplyProgress { sinceUnix: number; + asOfUnix: number; /// What it waits on right now, in the host's words; absent before the /// host reports anything. waiting?: string; diff --git a/packages/weft-graph/src/status.test.ts b/packages/weft-graph/src/status.test.ts index d69dc1f8..7bb95556 100644 --- a/packages/weft-graph/src/status.test.ts +++ b/packages/weft-graph/src/status.test.ts @@ -1,5 +1,11 @@ -import { describe, it, expect } from 'vitest'; -import { parseRunning, parseStatusPayload, emptyActionAvailability } from './status'; +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { + describeProgress, + emptyActionAvailability, + parseRunning, + parseStatusPayload, + progressOnLocalClock, +} from './status'; describe('parseStatusPayload', () => { it('remaps snake_case wire fields to the camelCase snapshot', () => { @@ -67,3 +73,28 @@ describe('parseRunning', () => { })).toThrow(/cannot read/); }); }); + +describe('infra progress', () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it('counts the server\'s duration, then ticks on this machine\'s clock', () => { + // The server says 20s had passed; this machine's clock runs 500s ahead. + const local = progressOnLocalClock({ sinceUnix: 1_000, asOfUnix: 1_020, waiting: 'db' }, 1_520); + expect(local).toEqual({ sinceUnix: 1_500, asOfUnix: 1_520, waiting: 'db' }); + expect(describeProgress(local, 1_520)).toBe('for 20s, waiting on: db'); + expect(describeProgress(local, 1_565)).toBe('for 1m05s, waiting on: db'); + }); + + it('moves a status answer\'s progress onto this machine\'s clock as it arrives', () => { + vi.useFakeTimers(); + vi.setSystemTime(9_000_000); + const snap = parseStatusPayload({ + infra: [{ node: 'db', node_type: 'pg', status: 'provisioning', progress: { sinceUnix: 100, asOfUnix: 130 } }], + }); + const progress = snap.infraNodes[0].progress!; + expect(progress).toEqual({ sinceUnix: 8_970, asOfUnix: 9_000 }); + expect(describeProgress(progress, Date.now() / 1000)).toBe('for 30s'); + }); +}); diff --git a/packages/weft-graph/src/status.ts b/packages/weft-graph/src/status.ts index 1db48af4..ae006452 100644 --- a/packages/weft-graph/src/status.ts +++ b/packages/weft-graph/src/status.ts @@ -135,7 +135,7 @@ export function parseStatusPayload(raw: RawStatusPayload): ActionAvailability { ...(n.failureStage !== undefined ? { failureStage: n.failureStage } : {}), ...(n.failureMessage !== undefined ? { failureMessage: n.failureMessage } : {}), ...(n.instance_copy_count !== undefined ? { instanceCopyCount: n.instance_copy_count } : {}), - ...(n.progress !== undefined ? { progress: n.progress } : {}), + ...(n.progress !== undefined ? { progress: progressOnLocalClock(n.progress, Date.now() / 1000) } : {}), })); return { availableActions: (Array.isArray(raw.available_actions) @@ -208,8 +208,20 @@ export function backendFromSnapshot(snapshot: ActionAvailability): BackendSnapsh }; } +/** The server's progress, moved onto this machine's clock: both stamps + * shift by how far this clock reads from the server's when the answer + * arrived (`localNowUnix`). Read against `Date.now()` afterwards, the + * duration is the server's at the answer plus the time elapsed here + * since, so it keeps ticking between answers whatever the two clocks + * say. */ +export function progressOnLocalClock(progress: ApplyProgress, localNowUnix: number): ApplyProgress { + const shift = localNowUnix - progress.asOfUnix; + return { ...progress, sinceUnix: progress.sinceUnix + shift, asOfUnix: localNowUnix }; +} + /** How far a start of an infra copy got, as a person reads it: "for - * 3m12s, waiting on: ...". */ + * 3m12s, waiting on: ...". `nowUnix` is on the clock the progress's + * stamps are on (this machine's, once `parseStatusPayload` moved it). */ // SYNC: describeProgress <-> crates/weft-core/src/infra/wire.rs ApplyProgress::describe export function describeProgress(progress: ApplyProgress, nowUnix: number): string { const secs = Math.max(0, Math.floor(nowUnix - progress.sinceUnix)); diff --git a/tangle/claude-code/.claude/skills/weft-instances/SKILL.md b/tangle/claude-code/.claude/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/claude-code/.claude/skills/weft-instances/SKILL.md +++ b/tangle/claude-code/.claude/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/claude-code/.claude/skills/weft-members/SKILL.md b/tangle/claude-code/.claude/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/claude-code/.claude/skills/weft-members/SKILL.md +++ b/tangle/claude-code/.claude/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/cline/.cline/skills/weft-instances/SKILL.md b/tangle/cline/.cline/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/cline/.cline/skills/weft-instances/SKILL.md +++ b/tangle/cline/.cline/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/cline/.cline/skills/weft-members/SKILL.md b/tangle/cline/.cline/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/cline/.cline/skills/weft-members/SKILL.md +++ b/tangle/cline/.cline/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/codex/.agents/skills/weft-instances/SKILL.md b/tangle/codex/.agents/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/codex/.agents/skills/weft-instances/SKILL.md +++ b/tangle/codex/.agents/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/codex/.agents/skills/weft-members/SKILL.md b/tangle/codex/.agents/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/codex/.agents/skills/weft-members/SKILL.md +++ b/tangle/codex/.agents/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/cursor/.cursor/skills/weft-instances/SKILL.md b/tangle/cursor/.cursor/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/cursor/.cursor/skills/weft-instances/SKILL.md +++ b/tangle/cursor/.cursor/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/cursor/.cursor/skills/weft-members/SKILL.md b/tangle/cursor/.cursor/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/cursor/.cursor/skills/weft-members/SKILL.md +++ b/tangle/cursor/.cursor/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/devin-desktop/.windsurf/skills/weft-instances/SKILL.md b/tangle/devin-desktop/.windsurf/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/devin-desktop/.windsurf/skills/weft-instances/SKILL.md +++ b/tangle/devin-desktop/.windsurf/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/devin-desktop/.windsurf/skills/weft-members/SKILL.md b/tangle/devin-desktop/.windsurf/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/devin-desktop/.windsurf/skills/weft-members/SKILL.md +++ b/tangle/devin-desktop/.windsurf/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/fallback/.agents/skills/weft-instances/SKILL.md b/tangle/fallback/.agents/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/fallback/.agents/skills/weft-instances/SKILL.md +++ b/tangle/fallback/.agents/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/fallback/.agents/skills/weft-members/SKILL.md b/tangle/fallback/.agents/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/fallback/.agents/skills/weft-members/SKILL.md +++ b/tangle/fallback/.agents/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/gemini-cli/.gemini/skills/weft-instances/SKILL.md b/tangle/gemini-cli/.gemini/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/gemini-cli/.gemini/skills/weft-instances/SKILL.md +++ b/tangle/gemini-cli/.gemini/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/gemini-cli/.gemini/skills/weft-members/SKILL.md b/tangle/gemini-cli/.gemini/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/gemini-cli/.gemini/skills/weft-members/SKILL.md +++ b/tangle/gemini-cli/.gemini/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/github-copilot/.github/skills/weft-instances/SKILL.md b/tangle/github-copilot/.github/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/github-copilot/.github/skills/weft-instances/SKILL.md +++ b/tangle/github-copilot/.github/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/github-copilot/.github/skills/weft-members/SKILL.md b/tangle/github-copilot/.github/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/github-copilot/.github/skills/weft-members/SKILL.md +++ b/tangle/github-copilot/.github/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/junie/.junie/skills/weft-instances/SKILL.md b/tangle/junie/.junie/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/junie/.junie/skills/weft-instances/SKILL.md +++ b/tangle/junie/.junie/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/junie/.junie/skills/weft-members/SKILL.md b/tangle/junie/.junie/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/junie/.junie/skills/weft-members/SKILL.md +++ b/tangle/junie/.junie/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/kilo-code/.kilo/skills/weft-instances/SKILL.md b/tangle/kilo-code/.kilo/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/kilo-code/.kilo/skills/weft-instances/SKILL.md +++ b/tangle/kilo-code/.kilo/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/kilo-code/.kilo/skills/weft-members/SKILL.md b/tangle/kilo-code/.kilo/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/kilo-code/.kilo/skills/weft-members/SKILL.md +++ b/tangle/kilo-code/.kilo/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/opencode/.opencode/skills/weft-instances/SKILL.md b/tangle/opencode/.opencode/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/opencode/.opencode/skills/weft-instances/SKILL.md +++ b/tangle/opencode/.opencode/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/opencode/.opencode/skills/weft-members/SKILL.md b/tangle/opencode/.opencode/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/opencode/.opencode/skills/weft-members/SKILL.md +++ b/tangle/opencode/.opencode/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of