diff --git a/catalog/instances/start_instance_infra/metadata.json b/catalog/instances/start_instance_infra/metadata.json index 3deec13a..36bc7283 100644 --- a/catalog/instances/start_instance_infra/metadata.json +++ b/catalog/instances/start_instance_infra/metadata.json @@ -1,7 +1,7 @@ { "type": "StartInstanceInfra", "label": "Start instance infra", - "description": "Bring up one instance's copy of a `@per_instance` infra node; `done` fires once it answers, so the instance's triggers reading it can be activated next. The run waits without holding a worker; a copy that fails to come up fails the node with the reason; an already-running copy is left as is.", + "description": "Bring up one instance's copy of a `@per_instance` infra node; `done` fires once it answers, so the instance's triggers reading it can be activated next. The run waits without holding a worker; a copy that fails to come up fails the node with the reason; an already-running copy is left as is. If you want to answer a caller fast (a route replying 202), turn `waitUntilRunning` off and reply after this node: it finishes as soon as weft accepts the start, and a status read right after shows the copy as `provisioning`.", "tags": [ "instances", "infra", @@ -21,13 +21,20 @@ "type": "String", "required": true, "description": "The instance id your program gives it (`user-42`)." + }, + { + "name": "waitUntilRunning", + "type": "Boolean", + "default": true, + "label": "Wait until running", + "description": "On: `done` fires once the copy runs, and a copy that fails to come up fails the node. Off: `done` fires once weft accepts the start (a start refused for a passing reason, such as a build in progress, is asked again until it is), and whether the copy then comes up is for a status read to tell." } ], "outputs": [ { "name": "done", "type": "Boolean", - "description": "True once the copy is running." + "description": "True once the copy is running, or once weft accepted the start when `waitUntilRunning` is off." } ], "validate": [ diff --git a/catalog/instances/start_instance_infra/mod.rs b/catalog/instances/start_instance_infra/mod.rs index befa85b4..52039684 100644 --- a/catalog/instances/start_instance_infra/mod.rs +++ b/catalog/instances/start_instance_infra/mod.rs @@ -1,5 +1,6 @@ //! StartInstanceInfra: bring up one instance's copy of a `@per_instance` -//! infra node, and fire `done` once the copy answers. +//! infra node, and fire `done` once the copy answers, or, with +//! `waitUntilRunning` off, once weft accepted the start. use async_trait::async_trait; @@ -23,7 +24,13 @@ impl Node for StartInstanceInfraNode { async fn run(&self, ctx: ExecutionContext) -> WeftResult<()> { let node: String = ctx.inputs.get("node")?; - ctx.infra(node).instance(instance(&ctx)?).start().await?; + let wait_until_running: bool = ctx.inputs.get("waitUntilRunning")?; + let copy = ctx.infra(node).instance(instance(&ctx)?); + if wait_until_running { + copy.start().await?; + } else { + copy.request_start().await?; + } ctx.pulse_downstream(NodeOutput::new().set("done", true)).await } } diff --git a/catalog/instances/start_instance_infra/tests.rs b/catalog/instances/start_instance_infra/tests.rs index 5e175b8d..841358de 100644 --- a/catalog/instances/start_instance_infra/tests.rs +++ b/catalog/instances/start_instance_infra/tests.rs @@ -2,8 +2,9 @@ //! copy of that node and `done` waits for the copy to run, a copy that //! fails fails the node, a copy somebody stops (or that never came to be) //! while the node waits fails it instead of being started again, a start -//! refused for a passing reason is asked again, and a blank instance is -//! refused before anything is asked. +//! refused for a passing reason is asked again, a node told not to wait +//! finishes once the start is accepted without looking at the copy, and a +//! blank instance is refused before anything is asked. use serde_json::json; @@ -20,6 +21,8 @@ pub fn tests() -> Vec { NodeTest::fake("a_copy_stopped_while_waiting_fails_the_node", stopped_while_waiting), NodeTest::fake("a_start_that_ends_without_a_copy_fails_the_node", no_copy), NodeTest::fake("a_start_refused_for_now_is_asked_again", refused_for_now), + NodeTest::fake("without_waiting_finishes_once_the_start_is_accepted", without_waiting), + NodeTest::fake("without_waiting_a_start_refused_for_now_is_asked_again", without_waiting_refused_for_now), NodeTest::fake("a_blank_instance_is_refused", blank_instance), ] } @@ -101,3 +104,32 @@ async fn refused_for_now(rig: FakeRig) -> WeftResult<()> { assert_eq!(outcome.outputs["done"], json!(true)); Ok(()) } + +async fn without_waiting(rig: FakeRig) -> WeftResult<()> { + let outcome = rig + .run(&StartInstanceInfraNode, json!({ "node": "bridge", "instance": "user-42", "waitUntilRunning": false })) + .await + .ok()?; + let calls = rig.program_calls(); + assert_eq!(calls.len(), 1, "the start only, no look at the copy: {calls:?}"); + assert!(matches!(&calls[0], (ProgramCall::InfraStart { .. }, StopSelf::Keep)), "{calls:?}"); + assert!(rig.awaited_signals().is_empty()); + assert_eq!(outcome.outputs["done"], json!(true)); + Ok(()) +} + +async fn without_waiting_refused_for_now(rig: FakeRig) -> WeftResult<()> { + rig.answer_program_call("weft.infra.start", json!({ "answer": "waiting", "reason": "project is building" })); + rig.answer_program_call("weft.infra.start", json!({ "answer": "already_starting" })); + rig.signal(json!({})); + let outcome = rig + .run(&StartInstanceInfraNode, json!({ "node": "bridge", "instance": "user-42", "waitUntilRunning": false })) + .await + .ok()?; + let calls = rig.program_calls(); + assert_eq!(calls.len(), 2, "asked again after the wait, then done without a look: {calls:?}"); + assert!(calls.iter().all(|(call, _)| matches!(call, ProgramCall::InfraStart { .. })), "{calls:?}"); + assert_eq!(rig.awaited_signals().len(), 1); + assert_eq!(outcome.outputs["done"], json!(true)); + Ok(()) +} diff --git a/crates/weft-cli/src/commands/frontend.rs b/crates/weft-cli/src/commands/frontend.rs index fd95e47d..7155fbf6 100644 --- a/crates/weft-cli/src/commands/frontend.rs +++ b/crates/weft-cli/src/commands/frontend.rs @@ -154,11 +154,20 @@ fn hand_over(ctx: &Ctx, project: &str, install_url: &str, f: &Frontend, token: O if let Some(url) = &f.url { println!("visitors reach it at {url}"); } + let project = ctx.project()?; println!( "next, in {}: {} hands its deploy workflow the service and a token", repo.name, - super::target::export_command(ctx.project()?, ctx.on()) + super::target::export_command(project, ctx.on()) ); + if !project.root.join(super::target::FRONT_ENV_FILE).is_file() { + println!( + "if the frontend needs settings of its own (its database address, a sign-in secret), write them to \ + {} and add `--front-env {}`", + super::target::FRONT_ENV_FILE, + super::target::FRONT_ENV_FILE + ); + } return Ok(()); } // A frontend running elsewhere reaches the install at its public diff --git a/crates/weft-cli/src/commands/infra.rs b/crates/weft-cli/src/commands/infra.rs index a320609b..2fb9967f 100644 --- a/crates/weft-cli/src/commands/infra.rs +++ b/crates/weft-cli/src/commands/infra.rs @@ -624,9 +624,8 @@ async fn infra_status(ctx: &Ctx) -> Result<()> { eprintln!("warning: {note}"); } // Printed apart from the copies: a start stuck this way has none yet. - let now = crate::progress::now_unix() as i64; for entry in &status.unanswered { - eprintln!("warning: {}", entry.describe(now)); + eprintln!("warning: {}", entry.describe()); } Ok(()) } @@ -644,15 +643,14 @@ async fn changing_now(client: &crate::client::DispatcherClient, project_id: &str }; // A call of weft's own that keeps failing comes first: it is why // nothing below moves. - let now = crate::progress::now_unix() as i64; - let unanswered = status.unanswered.iter().map(|u| u.describe(now)); + let unanswered = status.unanswered.iter().map(|u| u.describe()); let copies = status.nodes.iter().filter_map(|n| { let node = match &n.instance { Some(instance) => format!("{} (instance {instance})", n.node), None => n.node.clone(), }; if let Some(progress) = &n.progress { - return Some(format!("{node}: {} {}", n.status, progress.describe_now())); + return Some(format!("{node}: {} {}", n.status, progress.describe(progress.as_of_unix))); } let changing = matches!( InfraNodeStatus::parse(&n.status), @@ -695,7 +693,7 @@ fn print_status(name: &str, id: &str, status: &InfraStatus) { let url = n.endpoint_url.as_deref().unwrap_or("(no endpoint)"); println!(" {node} [{}] -> {url}", n.status); if let Some(progress) = &n.progress { - println!(" {}", progress.describe_now()); + println!(" {}", progress.describe(progress.as_of_unix)); } // A public endpoint's outside address: what to hand to whoever // calls in (the node declared only its own path). diff --git a/crates/weft-cli/src/commands/status.rs b/crates/weft-cli/src/commands/status.rs index 1d95c202..bc2c984e 100644 --- a/crates/weft-cli/src/commands/status.rs +++ b/crates/weft-cli/src/commands/status.rs @@ -176,7 +176,7 @@ pub async fn run(ctx: Ctx) -> Result<()> { st => { println!(" {node}: {st} ({})", entry.endpoint_url.as_deref().unwrap_or("-")); if let Some(progress) = &entry.progress { - println!(" {}", progress.describe_now()); + println!(" {}", progress.describe(progress.as_of_unix)); } } } @@ -210,7 +210,7 @@ pub async fn run(ctx: Ctx) -> Result<()> { for entry in &data.instance_infra { println!(" {} (instance {}): {}", entry.node, entry.instance, entry.status); if let Some(progress) = &entry.progress { - println!(" {}", progress.describe_now()); + println!(" {}", progress.describe(progress.as_of_unix)); } } } @@ -237,9 +237,8 @@ pub async fn run(ctx: Ctx) -> Result<()> { // not move. if !data.unanswered.is_empty() { println!(" waiting on weft:"); - let now = crate::progress::now_unix() as i64; for entry in &data.unanswered { - println!(" {}", entry.describe(now)); + println!(" {}", entry.describe()); } } // What each trigger started lately: the one trace a run kept diff --git a/crates/weft-cli/src/commands/target.rs b/crates/weft-cli/src/commands/target.rs index 8266e2ef..e9f18789 100644 --- a/crates/weft-cli/src/commands/target.rs +++ b/crates/weft-cli/src/commands/target.rs @@ -376,9 +376,11 @@ async fn export( for (k, v) in &settings.variables { println!(" variable {k}={v}"); } - // Only how many: nothing that comes out of a secret reaches the - // terminal, its name included. - println!(" secrets {} set", settings.secrets.len()); + // The names only: they are weft's own fixed words, so a person can check + // what the workflow received; a value never reaches the terminal. + for (k, _) in &settings.secrets { + println!(" secret {k}"); + } println!("run its deploy workflow from the Actions tab"); if let Some(frontend) = deployed_before(&minted, frontend) { println!("frontend '{frontend}' keeps its old token working until that run deploys the new one and retires it"); @@ -702,9 +704,15 @@ fn committed_target_matches(committed: &str, name: &str, url: &str) -> Result<() } } +/// Where a project keeps its frontend server's own settings for a cloud +/// target, the file the next export's `--front-env` hands the deploy +/// workflow: in `front/`, which the workflow builds. +pub const FRONT_ENV_FILE: &str = "front/.env.prod"; + /// The `weft target export ... --github` a person runs next, spelled with /// a target they really have: the one this command was given, else the -/// project's only cloud target, else the choice between them. +/// project's only cloud target, else the choice between them. With +/// [`FRONT_ENV_FILE`] in the project, it hands that file over too. pub fn export_command(project: &weft_compiler::project::Project, on: Option<&str>) -> String { let name = match on { Some(on) => on.to_string(), @@ -723,7 +731,10 @@ pub fn export_command(project: &weft_compiler::project::Project, on: Option<&str } } }; - format!("`weft target export {name} --github`") + match project.root.join(FRONT_ENV_FILE).is_file() { + true => format!("`weft target export {name} --github --front-env {FRONT_ENV_FILE}`"), + false => format!("`weft target export {name} --github`"), + } } /// A target name is what people type after `--on`, and a TOML key. diff --git a/crates/weft-core/src/context/program_calls.rs b/crates/weft-core/src/context/program_calls.rs index 51406bde..3d4ad8e0 100644 --- a/crates/weft-core/src/context/program_calls.rs +++ b/crates/weft-core/src/context/program_calls.rs @@ -146,7 +146,8 @@ impl InfraCalls<'_> { } /// Bring the copy up, and return once it answers, so what comes - /// next (activating the triggers that read it, say) can use it. + /// next (activating the triggers that read it, say) can use it: a + /// [`Self::request_start`], then a wait for the copy to run. /// Starting a copy that runs, or is already starting, does not start /// it twice. A copy that fails to come up fails this call with the /// reason, and so does one somebody stops (or terminates) while this @@ -158,25 +159,12 @@ impl InfraCalls<'_> { /// next to their infra never waits on the run that asked for it. pub async fn start(self) -> WeftResult<()> { let instance = instance_id(&self.instance)?; + self.ask_until_accepted(&instance).await?; let node = self.node; - let start = || ProgramCall::InfraStart { node: node.clone(), instance: instance.clone() }; - // Until a start is queued (one refused for a passing reason, a - // build in progress or a trigger of the copy mid-activation, is - // asked again at each look), what the copy reads is no answer to - // this start. - let mut queued = false; loop { - if !queued { - let answer: InfraStartAnswer = self.ctx.program_call(start(), StopSelf::Keep).await?; - queued = answer.queued(); - if !queued { - self.ctx.await_signal(START_LOOK_AGAIN).await?; - continue; - } - } - // A queued start reads `provisioning` from the moment the call - // above answered (the runtime counts the start under way, not - // only what the copy's row says), so every other answer is + // An accepted start reads `provisioning` from the moment the + // runtime accepted it (the runtime counts the start under way, + // not only what the copy's row says), so every other answer is // where the copy really stands. let status = ProgramCall::InfraStatus { node: node.clone(), instance: instance.clone() }; let copy: Option = self.ctx.program_call(status, StopSelf::Keep).await?; @@ -205,6 +193,34 @@ impl InfraCalls<'_> { } } + /// Ask for the copy to come up, and return once weft accepted the + /// start, without waiting for the copy to run. From then on a status + /// read ([`Self::status`], [`Self::copies`]) shows the copy as + /// `provisioning`, so a program answering a caller fast (a route + /// replying 202) replies after this and the caller's next read finds + /// the copy. A start refused for a passing reason (a build in + /// progress, a trigger of the copy mid-activation) is asked again + /// every few seconds until it is accepted; a copy already running or + /// starting is not started twice. Whether the copy then comes up is + /// for a later `status` read to tell: nothing here waits for it. + pub async fn request_start(self) -> WeftResult<()> { + let instance = instance_id(&self.instance)?; + self.ask_until_accepted(&instance).await + } + + /// Ask for the start until weft accepts it, parking on a timer + /// between refusals for a passing reason. + async fn ask_until_accepted(&self, instance: &Option) -> WeftResult<()> { + loop { + let call = ProgramCall::InfraStart { node: self.node.clone(), instance: instance.clone() }; + let answer: InfraStartAnswer = self.ctx.program_call(call, StopSelf::Keep).await?; + if answer.accepted() { + return Ok(()); + } + self.ctx.await_signal(START_LOOK_AGAIN).await?; + } + } + /// Scale the copy down, keeping its disk. `spec` says what happens to /// the triggers reading it and the runs using it; `stop_self` whether /// this run goes too when it uses the copy. Answers whether there was diff --git a/crates/weft-core/src/infra/wire.rs b/crates/weft-core/src/infra/wire.rs index 51333ef9..8f11300c 100644 --- a/crates/weft-core/src/infra/wire.rs +++ b/crates/weft-core/src/infra/wire.rs @@ -337,23 +337,24 @@ pub struct InfraStatusEntry { /// How far a start of an infra copy got: since when it runs, and what it /// waits on right now, in the host's words ("its machine's agent does not /// answer yet: ..."). `waiting` is absent before the host reports -/// anything (the machine is still being made). +/// anything (the machine is still being made). `since_unix` is when the +/// start was asked, and `as_of_unix` the server's clock when it answered: +/// both on one clock, so how long it has run never depends on the +/// reader's own. // SYNC: ApplyProgress <-> packages/weft-graph/src/protocol.ts ApplyProgress #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ApplyProgress { #[serde(rename = "sinceUnix")] pub since_unix: i64, + #[serde(rename = "asOfUnix")] + pub as_of_unix: i64, #[serde(default, skip_serializing_if = "Option::is_none")] pub waiting: Option, } impl ApplyProgress { - /// The progress a row records, when it records one. - pub fn of(since_unix: Option, waiting: Option) -> Option { - since_unix.map(|since_unix| Self { since_unix, waiting }) - } - - /// As a person reads it, `now_unix` being the time it is read: + /// As a person reads it, `now_unix` being the time it is read, on the + /// clock `since_unix` is on (`as_of_unix` reads it as of the answer): /// "for 3m12s, waiting on: ...". // SYNC: describe <-> packages/weft-graph/src/status.ts describeProgress pub fn describe(&self, now_unix: i64) -> String { @@ -364,15 +365,6 @@ impl ApplyProgress { None => format!("for {elapsed}"), } } - - /// [`Self::describe`], read now. - pub fn describe_now(&self) -> String { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs() as i64) - .unwrap_or(self.since_unix); - self.describe(now) - } } #[cfg(test)] @@ -381,10 +373,29 @@ mod apply_progress_tests { #[test] fn progress_reads_as_how_long_and_on_what() { - let p = ApplyProgress { since_unix: 100, waiting: Some("db: its machine's agent does not answer yet".into()) }; - assert_eq!(p.describe(292), "for 3m12s, waiting on: db: its machine's agent does not answer yet"); - assert_eq!(ApplyProgress { since_unix: 100, waiting: None }.describe(130), "for 30s"); - assert_eq!(ApplyProgress::of(None, Some("x".into())), None, "no start, no progress"); + let p = ApplyProgress { since_unix: 100, as_of_unix: 292, waiting: Some("db: its machine's agent does not answer yet".into()) }; + assert_eq!(p.describe(p.as_of_unix), "for 3m12s, waiting on: db: its machine's agent does not answer yet"); + let bare = ApplyProgress { since_unix: 100, as_of_unix: 130, waiting: None }; + assert_eq!(bare.describe(bare.as_of_unix), "for 30s"); + } + + /// Read as of the answer, the duration is the server's alone: a + /// reader whose clock runs 20s behind the server's still reads 20s. + #[test] + fn progress_as_of_the_answer_ignores_the_readers_clock() { + let p = ApplyProgress { since_unix: 1_000, as_of_unix: 1_020, waiting: None }; + assert_eq!(p.describe(p.as_of_unix), "for 20s"); + assert_eq!(p.describe(990), "for 0s", "a time before the start reads as no time at all"); + } + + #[test] + fn progress_travels_as_camel_case() { + let p = ApplyProgress { since_unix: 1, as_of_unix: 2, waiting: None }; + assert_eq!(serde_json::to_value(&p).unwrap(), serde_json::json!({ "sinceUnix": 1, "asOfUnix": 2 })); + assert!( + serde_json::from_value::(serde_json::json!({ "sinceUnix": 1 })).is_err(), + "an answer without its clock is refused" + ); } } diff --git a/crates/weft-core/src/program.rs b/crates/weft-core/src/program.rs index 38b1fa24..2b50dfb5 100644 --- a/crates/weft-core/src/program.rs +++ b/crates/weft-core/src/program.rs @@ -189,7 +189,8 @@ pub struct WaitingFires { /// What `ProgramCall::InfraStart` answers: a start queued, one already /// under way or done, or none yet for a passing reason (a build in /// progress, a trigger of the copy mid-activation), which -/// `ctx.infra(..).start()` asks again at its next look. +/// `ctx.infra(..).start()` and `.request_start()` ask again at their next +/// look. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "answer", rename_all = "snake_case")] pub enum InfraStartAnswer { @@ -200,8 +201,9 @@ pub enum InfraStartAnswer { } impl InfraStartAnswer { - /// Whether a start is under way: every answer but `Waiting`. - pub fn queued(&self) -> bool { + /// Whether weft accepted the start (it is under way or done): every + /// answer but `Waiting`. + pub fn accepted(&self) -> bool { !matches!(self, InfraStartAnswer::Waiting { .. }) } } diff --git a/crates/weft-core/src/projects.rs b/crates/weft-core/src/projects.rs index e5f03858..20d5a51f 100644 --- a/crates/weft-core/src/projects.rs +++ b/crates/weft-core/src/projects.rs @@ -253,11 +253,16 @@ pub struct Unanswered { pub since_unix: i64, #[serde(rename = "lastUnix")] pub last_unix: i64, + /// The install's clock when this was read, which the times above are + /// on, so how long ago reads right whatever clock the reader has. + #[serde(rename = "asOfUnix")] + pub as_of_unix: i64, } impl Unanswered { /// One line for a person: what is not answering, since when, and why. - pub fn describe(&self, now_unix: i64) -> String { + pub fn describe(&self) -> String { + let now_unix = self.as_of_unix; let span = |secs: i64| match secs.max(0) { secs @ 0..120 => format!("{secs}s"), secs => format!("{} min", secs / 60), diff --git a/crates/weft-dispatcher/src/api/infra.rs b/crates/weft-dispatcher/src/api/infra.rs index e5bc098b..bab46184 100644 --- a/crates/weft-dispatcher/src/api/infra.rs +++ b/crates/weft-dispatcher/src/api/infra.rs @@ -1487,7 +1487,8 @@ pub(crate) async fn live_endpoint_url( /// Every copy as `weft status` and a program's `ctx.infra(..).status()` /// show it (`infra_node::observe`): a start or a stop under way reads as -/// such before the supervisor reaches the copy. +/// such before the supervisor reaches the copy, and a start counts its +/// progress from when it was asked for. async fn read_infra_entries( state: &DispatcherState, project_id: uuid::Uuid, @@ -1501,23 +1502,38 @@ async fn read_infra_entries( let copies = infra_node::observe(&state.pg_pool, project_id, &project) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("infra copies: {e:#}")))?; - let mut entries: Vec = - copies.rows.into_iter().map(|row| row_to_entry(row, state.external_base_url())).collect(); - entries.extend(copies.starting.into_iter().map(|(node, instance)| InfraStatusEntry { - node, - instance, - status: infra_node::InfraNodeStatus::Provisioning.as_str().to_string(), - endpoint_url: None, - public_urls: Default::default(), - failure_stage: None, - failure_message: None, - notes: Vec::new(), - progress: None, - })); - Ok(entries) + let starting: Vec = copies + .starting + .iter() + .map(|(node, instance)| InfraStatusEntry { + node: node.clone(), + instance: instance.clone(), + status: infra_node::InfraNodeStatus::Provisioning.as_str().to_string(), + endpoint_url: None, + public_urls: Default::default(), + failure_stage: None, + failure_message: None, + notes: Vec::new(), + progress: copies.progress_of(node, instance.as_ref()), + }) + .collect(); + let progress: Vec<_> = copies.rows.iter().map(|row| copies.progress_of(&row.node_id, row.instance.as_ref())).collect(); + Ok(copies + .rows + .into_iter() + .zip(progress) + .map(|(row, progress)| row_to_entry(row, progress, state.external_base_url())) + .chain(starting) + .collect()) } -fn row_to_entry(row: InfraNodeRow, front_door: &str) -> InfraStatusEntry { +/// One row as an entry, with the progress [`infra_node::ObservedCopies::progress_of`] +/// gives its copy. +fn row_to_entry( + row: InfraNodeRow, + progress: Option, + front_door: &str, +) -> InfraStatusEntry { InfraStatusEntry { public_urls: row .public_paths @@ -1534,20 +1550,10 @@ fn row_to_entry(row: InfraNodeRow, front_door: &str) -> InfraStatusEntry { failure_stage: row.failure_stage.map(|f| f.as_str().to_string()), failure_message: row.failure_message, notes: row.notes, - progress: provisioning_progress(row.status, row.provisioning_since_unix, row.waiting), + progress, } } -/// The progress a row records, shown only while it provisions: the -/// columns hold the last apply's until the next one rewrites them. -pub(crate) fn provisioning_progress( - status: InfraNodeStatus, - since_unix: Option, - waiting: Option, -) -> Option { - (status == InfraNodeStatus::Provisioning).then(|| weft_core::infra::wire::ApplyProgress::of(since_unix, waiting)).flatten() -} - /// Enqueue a lifecycle command for `project_id`, under its tenant. The /// command's row announces itself (`INFRA_COMMAND_CHANNEL`), which wakes /// the supervisor wherever it runs (`weft_runtime::role_waker` for one at diff --git a/crates/weft-dispatcher/src/api/project.rs b/crates/weft-dispatcher/src/api/project.rs index 6469776e..78da325a 100644 --- a/crates/weft-dispatcher/src/api/project.rs +++ b/crates/weft-dispatcher/src/api/project.rs @@ -1268,6 +1268,7 @@ pub(crate) async fn unanswered_for(state: &DispatcherState, project_id: uuid::Uu error: f.error, since_unix: f.since_ms / 1000, last_unix: f.last_ms / 1000, + as_of_unix: f.as_of_ms / 1000, }) .collect()) } @@ -1598,7 +1599,7 @@ fn infra_entries( node: row.node_id.clone(), instance: row.instance.clone()?, status: row.status.as_str().to_string(), - progress: crate::api::infra::provisioning_progress(row.status, row.provisioning_since_unix, row.waiting.clone()), + progress: copies.progress_of(&row.node_id, row.instance.as_ref()), }) }) .chain(copies.starting.iter().filter_map(|(node, instance)| { @@ -1606,7 +1607,7 @@ fn infra_entries( node: node.clone(), instance: instance.clone()?, status: crate::infra_node::InfraNodeStatus::Provisioning.as_str().to_string(), - progress: None, + progress: copies.progress_of(node, instance.as_ref()), }) })) .collect(); @@ -1640,17 +1641,16 @@ fn infra_entries( endpoint_url: row.install_endpoints.values().next().cloned(), failure_stage: row.failure_stage.map(|f| f.as_str().to_string()), failure_message: row.failure_message.clone(), - progress: crate::api::infra::provisioning_progress( - row.status, - row.provisioning_since_unix, - row.waiting.clone(), - ), + progress: copies.progress_of(&spelled, None), ..entry(row.status.as_str().to_string()) }), - None => infra.push(entry(match copies.status_of(&spelled, None) { - Some(status) => status.as_str().to_string(), - None => INFRA_NOT_STARTED.to_string(), - })), + None => infra.push(match copies.status_of(&spelled, None) { + Some(status) => ProjectInfraEntry { + progress: copies.progress_of(&spelled, None), + ..entry(status.as_str().to_string()) + }, + None => entry(INFRA_NOT_STARTED.to_string()), + }), } } (infra, instance_infra) @@ -5300,8 +5300,11 @@ mod infra_entries_tests { let copies = ObservedCopies { rows: vec![row("db", None, InfraNodeStatus::Running), row("bridge", Some("ada"), InfraNodeStatus::Stopped)], starting: vec![("queue".into(), None), ("bridge".into(), Some(InstanceId::new("bob").unwrap()))], + start_asked_unix: [(("queue".to_string(), None), 100), (("bridge".to_string(), Some(InstanceId::new("bob").unwrap())), 110)] + .into(), + as_of_unix: 130, }; - let (infra, instances) = infra_entries(&project, &copies); + let (infra, instance_entries) = infra_entries(&project, &copies); let listed: Vec<(&str, &str, Option)> = infra.iter().map(|e| (e.node.as_str(), e.status.as_str(), e.instance_copy_count)).collect(); assert_eq!( @@ -5314,8 +5317,16 @@ mod infra_entries_tests { ] ); let instances: Vec<(&str, &str, &str)> = - instances.iter().map(|c| (c.node.as_str(), c.instance.as_str(), c.status.as_str())).collect(); + instance_entries.iter().map(|c| (c.node.as_str(), c.instance.as_str(), c.status.as_str())).collect(); assert_eq!(instances, [("bridge", "ada", "stopped"), ("bridge", "bob", "provisioning")]); + // A copy being started counts from its request, row or no row, as + // of the clock the copies were read on. + let since = |progress: Option<&weft_core::infra::wire::ApplyProgress>| progress.map(|p| (p.since_unix, p.as_of_unix)); + let shared = |node: &str| infra.iter().find(|e| e.node == node).unwrap().progress.as_ref(); + assert_eq!(since(shared("queue")), Some((100, 130)), "a shared copy with no row"); + assert_eq!(since(shared("db")), None, "a running copy shows none"); + let bob = instance_entries.iter().find(|e| e.instance.as_str() == "bob").unwrap(); + assert_eq!(since(bob.progress.as_ref()), Some((110, 130)), "an instance's copy with no row"); } /// A program with no infra node lists none, which is the one case the diff --git a/crates/weft-dispatcher/src/infra_node.rs b/crates/weft-dispatcher/src/infra_node.rs index 8be0be4e..21ff79a6 100644 --- a/crates/weft-dispatcher/src/infra_node.rs +++ b/crates/weft-dispatcher/src/infra_node.rs @@ -277,6 +277,10 @@ pub struct PendingOp { /// stop of the copy cancels it, so its applies never land after the /// stop). pub setup: Option, + /// When it was asked for (unix seconds): a command's + /// `issued_at_unix`, a setup run's `started_at`. A start counts its + /// progress from it. + pub asked_unix: i64, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -319,6 +323,24 @@ impl PendingOps { } } + /// When the start under way for this copy was asked for, `None` when + /// no start is what is under way. Of the starts carried out after the + /// copy's last stop or terminate, the earliest: a setup run issues + /// its applies once it runs, so the run asked first. + pub fn start_asked_unix(&self, node: &str, instance: Option<&weft_core::instance::InstanceId>) -> Option { + if !self.starting(node, instance) { + return None; + } + let covering = || { + self.ops.iter().filter(|op| op.node.as_deref().is_none_or(|n| n == node) && op.copies.admits(instance)) + }; + let last_taken_down = covering().filter(|op| op.kind != PendingKind::Start).map(|op| op.order).max(); + covering() + .filter(|op| op.kind == PendingKind::Start && last_taken_down.is_none_or(|order| op.order > order)) + .map(|op| op.asked_unix) + .min() + } + /// Whether a start of this copy is under way. pub fn starting(&self, node: &str, instance: Option<&weft_core::instance::InstanceId>) -> bool { self.status_of(node, instance, None) == Some(InfraNodeStatus::Provisioning) @@ -371,6 +393,13 @@ pub struct ObservedCopies { /// Copies with no row yet that a start is bringing up, as `(place, /// instance)`: they read `provisioning`. pub starting: Vec<(String, Option)>, + /// For each copy a start is bringing up (`(place, instance)`), when + /// that start was asked for ([`PendingOps::start_asked_unix`]). + pub start_asked_unix: BTreeMap<(String, Option), i64>, + /// The database's clock when this was read (unix seconds): the clock + /// a row's and a command's stamps are on (a setup run's `started_at` + /// is on the clock of whoever began it). + pub as_of_unix: i64, } impl ObservedCopies { @@ -385,6 +414,31 @@ impl ObservedCopies { .any(|(n, m)| n == node && m.as_ref() == instance) .then_some(InfraNodeStatus::Provisioning) } + + /// How far the start of one copy got, while it reads `provisioning`: + /// counted from when the start was asked for, or from when the + /// supervisor began the apply under way (`provisioning_since_unix`) + /// when that came first (a start asked again while an earlier apply + /// still runs), read as of [`Self::as_of_unix`]. `None` for a copy + /// that is not starting, and for a provisioning row stamped by + /// nothing (one written before the column existed). + pub fn progress_of( + &self, + node: &str, + instance: Option<&weft_core::instance::InstanceId>, + ) -> Option { + if self.status_of(node, instance) != Some(InfraNodeStatus::Provisioning) { + return None; + } + let row = self.rows.iter().find(|r| r.node_id == node && r.instance.as_ref() == instance); + let asked = self.start_asked_unix.get(&(node.to_string(), instance.cloned())).copied(); + let since_unix = [row.and_then(|r| r.provisioning_since_unix), asked].into_iter().flatten().min()?; + Some(weft_core::infra::wire::ApplyProgress { + since_unix, + as_of_unix: self.as_of_unix, + waiting: row.and_then(|r| r.waiting.clone()), + }) + } } /// Read every copy of `project_id` with what is under way applied: the one @@ -403,6 +457,7 @@ pub async fn observe_with( project_id: uuid::Uuid, project: &weft_core::ProjectDefinition, ) -> Result<(ObservedCopies, PendingOps)> { + let (as_of_unix,): (i64,) = sqlx::query_as("SELECT EXTRACT(EPOCH FROM NOW())::BIGINT").fetch_one(pool).await?; let mut rows = list_for_project(pool, project_id).await?; let pending = pending_ops(pool, project_id, project).await?; for row in &mut rows { @@ -411,7 +466,13 @@ pub async fn observe_with( } } let starting = pending.starting_without_row(&rows); - Ok((ObservedCopies { rows, starting }, pending)) + let start_asked_unix = rows + .iter() + .map(|r| (r.node_id.clone(), r.instance.clone())) + .chain(starting.iter().cloned()) + .filter_map(|copy| pending.start_asked_unix(©.0, copy.1.as_ref()).map(|asked| (copy, asked))) + .collect(); + Ok((ObservedCopies { rows, starting, start_asked_unix, as_of_unix }, pending)) } /// Read what is under way for `project_id`'s copies (see [`PendingOps`]). @@ -422,17 +483,17 @@ pub async fn pending_ops( project: &weft_core::ProjectDefinition, ) -> Result { let mut ops = Vec::new(); - /// id, node_id, verb, instance_id, every_copy. - type CommandRow = (i64, Option, String, Option, bool); + /// id, node_id, verb, instance_id, every_copy, issued_at_unix. + type CommandRow = (i64, Option, String, Option, bool, i64); let commands: Vec = sqlx::query_as(&format!( - "SELECT id, node_id, verb, instance_id, every_copy FROM infra_lifecycle_command \ + "SELECT id, node_id, verb, instance_id, every_copy, issued_at_unix FROM infra_lifecycle_command \ WHERE project_id = $1 AND completed_at_unix IS NULL AND verb IN ({verbs})", verbs = weft_broker_client::lifecycle_command::SUPERVISOR_VERBS_SQL, )) .bind(project_id) .fetch_all(pool) .await?; - for (id, node, verb, instance, every) in commands { + for (id, node, verb, instance, every, issued_at_unix) in commands { let kind = match crate::infra_lifecycle_command::InfraLifecycleVerb::parse(&verb) { Some(crate::infra_lifecycle_command::InfraLifecycleVerb::Apply) => PendingKind::Start, Some(crate::infra_lifecycle_command::InfraLifecycleVerb::Stop) => PendingKind::Stop, @@ -441,21 +502,23 @@ pub async fn pending_ops( }; let copies = weft_core::instance::Copies::from_columns(instance, every) .map_err(|e| anyhow::anyhow!("infra_lifecycle_command {id}: {e}"))?; - ops.push(PendingOp { kind, node, copies, order: id, setup: None }); + ops.push(PendingOp { kind, node, copies, order: id, setup: None, asked_unix: issued_at_unix }); } // A setup run that has not ended brings up the infra places its // selection holds, for its instance. let infra: std::collections::BTreeSet = weft_core::project::infra_place_spellings(project); - let setups: Vec<(weft_core::ExecutionId, Option, Option>)> = - sqlx::query_as( - "SELECT r.execution_id, r.instance_id, s.selection FROM run r \ - LEFT JOIN run_selection s ON s.digest = r.selection \ - WHERE r.project_id = $1 AND r.phase = 'infra_setup' AND r.state <> 'ended'", - ) - .bind(project_id) - .fetch_all(pool) - .await?; - for (execution_id, instance, subgraph) in setups { + /// execution_id, instance_id, started_at, selection. + type SetupRow = + (weft_core::ExecutionId, Option, i64, Option>); + let setups: Vec = sqlx::query_as( + "SELECT r.execution_id, r.instance_id, r.started_at, s.selection FROM run r \ + LEFT JOIN run_selection s ON s.digest = r.selection \ + WHERE r.project_id = $1 AND r.phase = 'infra_setup' AND r.state <> 'ended'", + ) + .bind(project_id) + .fetch_all(pool) + .await?; + for (execution_id, instance, started_at, subgraph) in setups { let instance = instance .map(weft_core::instance::InstanceId::new) .transpose() @@ -472,6 +535,7 @@ pub async fn pending_ops( copies: weft_core::instance::Copies::of(instance.clone()), order: i64::MAX, setup: Some(execution_id), + asked_unix: started_at, }); } } @@ -623,7 +687,11 @@ mod tests { } fn op(kind: PendingKind, node: Option<&str>, copies: weft_core::instance::Copies, order: i64) -> PendingOp { - PendingOp { kind, node: node.map(str::to_string), copies, order, setup: None } + asked(kind, node, copies, order, 0) + } + + fn asked(kind: PendingKind, node: Option<&str>, copies: weft_core::instance::Copies, order: i64, asked_unix: i64) -> PendingOp { + PendingOp { kind, node: node.map(str::to_string), copies, order, setup: None, asked_unix } } fn ada() -> weft_core::instance::InstanceId { @@ -698,6 +766,88 @@ mod tests { assert_eq!(ops.starting_without_row(&[]), vec![("bridge".to_string(), Some(ada()))]); } + /// A start counts from when it was asked: the earliest start carried + /// out after the copy's last stop (a setup run asked before the apply + /// it issued), never one a later stop overrode, and nothing at all + /// while a stop is what is under way. + #[test] + fn a_start_is_asked_when_its_earliest_live_request_was() { + use weft_core::instance::Copies; + let instance = Copies::Instance(ada()); + let resumed = PendingOps::new(vec![ + asked(PendingKind::Start, Some("bridge"), instance.clone(), 3, 50), + asked(PendingKind::Stop, Some("bridge"), instance.clone(), 7, 60), + asked(PendingKind::Start, Some("bridge"), instance.clone(), 9, 80), + asked(PendingKind::Start, Some("bridge"), instance.clone(), i64::MAX, 70), + ]); + assert_eq!(resumed.start_asked_unix("bridge", Some(&ada())), Some(70), "the setup run, asked before its apply"); + assert_eq!(resumed.start_asked_unix("other", Some(&ada())), None, "nothing starts another place"); + let paused = PendingOps::new(vec![ + asked(PendingKind::Start, Some("bridge"), instance.clone(), 8, 50), + asked(PendingKind::Stop, Some("bridge"), instance, 9, 60), + ]); + assert_eq!(paused.start_asked_unix("bridge", Some(&ada())), None, "a stop is what is under way"); + } + + fn provisioning_row(since: Option, waiting: Option<&str>) -> InfraNodeRow { + InfraNodeRow { + project_id: uuid::Uuid::nil(), + node_id: "db".into(), + instance: None, + copy_id: String::new(), + status: InfraNodeStatus::Provisioning, + failure_stage: None, + failure_message: None, + applied_spec_hash: None, + applied_at_unix: None, + endpoints: Default::default(), + public_paths: Default::default(), + doors: Default::default(), + install_endpoints: Default::default(), + keep_disks: Vec::new(), + units: Default::default(), + notes: Vec::new(), + waiting: waiting.map(str::to_string), + provisioning_since_unix: since, + } + } + + /// A copy being started counts from the request, row or no row: with + /// no row yet it counts from the request alone, and over a row the + /// supervisor stamped later it still counts from the request (which + /// came first), keeping what the row says it waits on. Every answer + /// carries the clock it was read on. + #[test] + fn a_starting_copy_counts_from_the_request() { + let bob = weft_core::instance::InstanceId::new("bob").unwrap(); + let copies = ObservedCopies { + rows: vec![provisioning_row(Some(120), Some("its machine's agent does not answer yet"))], + starting: vec![("bridge".into(), Some(bob.clone()))], + start_asked_unix: [(("db".to_string(), None), 100), (("bridge".to_string(), Some(bob.clone())), 90)].into(), + as_of_unix: 130, + }; + let db = copies.progress_of("db", None).expect("the row provisions"); + assert_eq!((db.since_unix, db.as_of_unix), (100, 130)); + assert_eq!(db.waiting.as_deref(), Some("its machine's agent does not answer yet")); + let bridge = copies.progress_of("bridge", Some(&bob)).expect("a start with no row yet"); + assert_eq!((bridge.since_unix, bridge.as_of_unix, bridge.waiting), (90, 130, None)); + assert_eq!(copies.progress_of("cache", None), None, "nothing starts it"); + } + + /// A row the supervisor already stamped, with no request under way + /// any more, counts from its own stamp; a copy that is up shows none. + #[test] + fn a_provisioning_row_alone_counts_from_its_stamp() { + let copies = ObservedCopies { rows: vec![provisioning_row(Some(120), None)], as_of_unix: 130, ..Default::default() }; + assert_eq!(copies.progress_of("db", None).map(|p| p.since_unix), Some(120)); + let running = ObservedCopies { + rows: vec![InfraNodeRow { status: InfraNodeStatus::Running, ..provisioning_row(Some(120), None) }], + as_of_unix: 130, + ..Default::default() + }; + assert_eq!(running.progress_of("db", None), None); + } + #[test] fn failure_stage_as_str() { assert_eq!(FailureStage::Provision.as_str(), "provision"); diff --git a/crates/weft-dispatcher/tests/db_lifecycle.rs b/crates/weft-dispatcher/tests/db_lifecycle.rs index f30fe79b..8bd56c0b 100644 --- a/crates/weft-dispatcher/tests/db_lifecycle.rs +++ b/crates/weft-dispatcher/tests/db_lifecycle.rs @@ -1325,6 +1325,68 @@ async fn copies_read_with_the_commands_under_way(pool: PgPool) { assert_eq!(copies.status_of("bridge", None), None, "no shared copy, nothing starting one"); } +/// A copy being started counts from when the start was asked for, before +/// the supervisor writes any row: an instance's copy with no row yet, and +/// one restarted over its stopped row (which holds no stamp of its own), +/// both read as provisioning since their apply command was issued, on the +/// database's clock. +#[sqlx::test] +async fn a_starting_copy_counts_from_its_request(pool: PgPool) { + use weft_broker::lifecycle_writes::{issue_command, IssuedCommand}; + use weft_dispatcher::infra_lifecycle_command::InfraLifecycleVerb; + use weft_dispatcher::infra_node::observe; + let (_journal, projects) = setup(&pool).await; + let project = Uuid::new_v4(); + seed_project(&projects, project, "bin-A").await; + sqlx::query("INSERT INTO infra_node (project_id, node_id, instance_id, copy_id, status) VALUES ($1, 'bridge', 'ada', 'inst', 'stopped')") + .bind(project) + .execute(&pool) + .await + .expect("seed ada's stopped copy"); + let ada = InstanceId::new("ada").unwrap(); + let bob = InstanceId::new("bob").unwrap(); + let mut issued = Vec::new(); + for (instance, asked_unix) in [(&ada, 1_000_i64), (&bob, 2_000)] { + let spec = json!({}); + let apply = IssuedCommand { + tenant_id: TENANT, + project_id: project, + node_id: Some("bridge"), + copies: &weft_core::instance::Copies::Instance(instance.clone()), + verb: InfraLifecycleVerb::Apply, + spec_json: Some(&spec), + issued_by_replica: "worker-1", + }; + let id = issue_command(&pool, &apply).await.unwrap().expect("the project row is there"); + // Asked well in the past, so the count cannot pass by luck. + sqlx::query("UPDATE infra_lifecycle_command SET issued_at_unix = $2 WHERE id = $1") + .bind(id) + .bind(asked_unix) + .execute(&pool) + .await + .unwrap(); + issued.push(asked_unix); + } + + let copies = observe(&pool, project, &empty_project(project)).await.unwrap(); + let (db_now,): (i64,) = sqlx::query_as("SELECT EXTRACT(EPOCH FROM NOW())::BIGINT").fetch_one(&pool).await.unwrap(); + let restarted = copies.progress_of("bridge", Some(&ada)).expect("a restart over a stopped row shows progress"); + assert_eq!(restarted.since_unix, issued[0]); + let fresh = copies.progress_of("bridge", Some(&bob)).expect("a start with no row yet shows progress"); + assert_eq!(fresh.since_unix, issued[1]); + assert_eq!(fresh.waiting, None, "nothing has reported what it waits on yet"); + assert!(fresh.as_of_unix <= db_now && fresh.as_of_unix > issued[1], "read on the database's clock: {}", fresh.as_of_unix); + + // Once the supervisor stamps the row later, the request still counts. + sqlx::query("UPDATE infra_node SET status = 'provisioning', provisioning_since_unix = 5000 WHERE project_id = $1") + .bind(project) + .execute(&pool) + .await + .unwrap(); + let copies = observe(&pool, project, &empty_project(project)).await.unwrap(); + assert_eq!(copies.progress_of("bridge", Some(&ada)).map(|p| p.since_unix), Some(issued[0]), "the request came first"); +} + /// What holds the program's own verbs back is infra work on the copies /// the bar starts and stops: an instance's copy coming up holds nothing /// back, the shared copies' work does, and so does the project going. diff --git a/crates/weft-platform-local/src/docker.rs b/crates/weft-platform-local/src/docker.rs index 557bdf76..184ba4e4 100644 --- a/crates/weft-platform-local/src/docker.rs +++ b/crates/weft-platform-local/src/docker.rs @@ -128,6 +128,22 @@ pub async fn containers_using(docker: &dyn Docker, volume: &str) -> anyhow::Resu Ok(out.lines().map(str::trim).filter(|l| !l.is_empty()).map(str::to_string).collect()) } +/// When the container `name` last started (`State.StartedAt`, on the +/// daemon's clock): a restart moves it. `docker ps` prints only a +/// rounded "Up 2 minutes", so this asks `docker inspect`. +pub async fn started_at(docker: &dyn Docker, name: &str) -> anyhow::Result> { + let out = run(docker, vec!["inspect".into(), "--format".into(), "{{json .State.StartedAt}}".into(), name.into()]).await?; + parse_started_at(name, &out) +} + +fn parse_started_at(name: &str, printed: &str) -> anyhow::Result> { + let at: String = serde_json::from_str(printed.trim()) + .map_err(|e| anyhow::anyhow!("`docker inspect` printed no start time for {name} ({e}): {}", printed.trim()))?; + Ok(chrono::DateTime::parse_from_rfc3339(&at) + .map_err(|e| anyhow::anyhow!("`docker inspect` printed a start time for {name} that is not RFC 3339 ({e}): {at}"))? + .to_utc()) +} + /// One line of `docker ps --format '{{json .}}'`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ContainerRow { @@ -310,6 +326,14 @@ mod tests { assert!(ContainerRow::parse("not json").is_err()); } + #[test] + fn a_start_time_is_read_from_inspect_or_refused() { + let at = parse_started_at("c-pg", "\"2024-01-01T00:00:05.123456789Z\"\n").unwrap(); + assert_eq!(at, chrono::DateTime::parse_from_rfc3339("2024-01-01T00:00:05.123456789Z").unwrap().to_utc()); + assert!(parse_started_at("c-pg", "\"yesterday\"").is_err()); + assert!(parse_started_at("c-pg", "").is_err()); + } + #[test] fn labels_and_filters_are_spelled_as_docker_takes_them() { let mut l = BTreeMap::new(); diff --git a/crates/weft-platform-local/src/infra_host.rs b/crates/weft-platform-local/src/infra_host.rs index fd351b11..cc1ecb79 100644 --- a/crates/weft-platform-local/src/infra_host.rs +++ b/crates/weft-platform-local/src/infra_host.rs @@ -408,9 +408,10 @@ impl LocalInfraHost { if let Some(ready) = readiness { let answer = self.probe(&agent_url, &c.name, ready).await?; if !answer.ok { - return Ok(UnitRunState::NotReady { - why: format!("container {short}: {}", answer.why.unwrap_or_else(|| "not ready".into())), - }); + let started = docker::started_at(self.docker.as_ref(), &c.name).await?; + let now = daemon_now(self.docker.as_ref()).await?; + let why = answer.why.unwrap_or_else(|| "not ready".into()); + return Ok(readiness_failed(short, &why, now.signed_duration_since(started), ready)); } } } @@ -422,6 +423,28 @@ impl LocalInfraHost { /// after a restart of this process still knows them. const PROBES: &str = "weft.probes"; +/// How long after it starts a container may fail its readiness check +/// and still count as starting: the wait its probe asks before the first +/// check, then one period per failure it tolerates. +fn startup_window(probe: &weft_core::infra::Probe) -> chrono::TimeDelta { + let seconds = u64::from(probe.initial_delay_seconds) + u64::from(probe.period_seconds) * u64::from(probe.failure_threshold.max(1)); + // Two u32 settings multiplied can pass what a duration holds; a + // window that long is a container never declared stuck, so it is + // the longest duration rather than a panic. + i64::try_from(seconds).ok().and_then(chrono::TimeDelta::try_seconds).unwrap_or(chrono::TimeDelta::MAX) +} + +/// What a running container whose readiness check failed with `why` is +/// doing, `up` after it started: still starting inside its +/// [`startup_window`], otherwise not ready, with why. +fn readiness_failed(short: &str, why: &str, up: chrono::TimeDelta, probe: &weft_core::infra::Probe) -> UnitRunState { + if up < startup_window(probe) { + UnitRunState::Starting { step: format!("container {short} has not answered its health check yet") } + } else { + UnitRunState::NotReady { why: format!("container {short}: {why}") } + } +} + fn container_short(c: &ContainerRow) -> &str { c.label("weft.container").unwrap_or(&c.name) } @@ -1278,4 +1301,32 @@ mod tests { assert!(!raw.contains(',')); assert_eq!(decode_probes(&raw).unwrap(), (Some(p), None)); } + + #[test] + fn the_startup_window_is_the_probes_delay_then_a_period_per_tolerated_failure() { + let mut p = Probe::http("/health", 8099); + p.initial_delay_seconds = 5; + p.period_seconds = 10; + p.failure_threshold = 3; + assert_eq!(startup_window(&p), chrono::TimeDelta::seconds(35)); + p.failure_threshold = 0; + assert_eq!(startup_window(&p), chrono::TimeDelta::seconds(15), "a threshold of 0 still allows one failure, as liveness reads it"); + } + + #[test] + fn a_failing_readiness_check_is_starting_inside_the_window_and_not_ready_after() { + let mut p = Probe::http("/health", 8099); + p.initial_delay_seconds = 0; + p.period_seconds = 10; + p.failure_threshold = 3; + let why = "GET /health on 8099: error sending request"; + assert_eq!( + readiness_failed("credential", why, chrono::TimeDelta::seconds(2), &p), + UnitRunState::Starting { step: "container credential has not answered its health check yet".into() } + ); + assert_eq!( + readiness_failed("credential", why, chrono::TimeDelta::seconds(30), &p), + UnitRunState::NotReady { why: format!("container credential: {why}") } + ); + } } diff --git a/crates/weft-task-store/src/unanswered.rs b/crates/weft-task-store/src/unanswered.rs index 3b6679e3..235f629b 100644 --- a/crates/weft-task-store/src/unanswered.rs +++ b/crates/weft-task-store/src/unanswered.rs @@ -67,6 +67,9 @@ pub struct Failing { pub error: String, pub since_ms: i64, pub last_ms: i64, + /// The database's clock when this was read, which the times above are + /// on: how long ago reads right whatever clock the reader has. + pub as_of_ms: i64, } /// A try at `callee` failed with `error`. @@ -93,8 +96,8 @@ pub async fn answered(pool: &PgPool, callee: Callee<'_>) -> Result<()> { /// Every role still failing, and `project`'s workers if they are: what /// the project's work waits on. pub async fn failing_for(pool: &PgPool, project: uuid::Uuid) -> Result> { - let rows: Vec<(String, String, i64, i64)> = sqlx::query_as(&format!( - "SELECT callee, error, since_ms, last_ms FROM unanswered_call \ + let rows: Vec<(String, String, i64, i64, i64)> = sqlx::query_as(&format!( + "SELECT callee, error, since_ms, last_ms, {DB_NOW_MS} FROM unanswered_call \ WHERE (callee LIKE 'role:%' OR callee = $1) AND last_ms >= {DB_NOW_MS} - {RECENT_MS} \ ORDER BY callee" )) @@ -103,11 +106,12 @@ pub async fn failing_for(pool: &PgPool, project: uuid::Uuid) -> Result crates/weft-core/src/infra/wire.rs ApplyProgress export interface ApplyProgress { sinceUnix: number; + asOfUnix: number; /// What it waits on right now, in the host's words; absent before the /// host reports anything. waiting?: string; diff --git a/packages/weft-graph/src/status.test.ts b/packages/weft-graph/src/status.test.ts index d69dc1f8..7bb95556 100644 --- a/packages/weft-graph/src/status.test.ts +++ b/packages/weft-graph/src/status.test.ts @@ -1,5 +1,11 @@ -import { describe, it, expect } from 'vitest'; -import { parseRunning, parseStatusPayload, emptyActionAvailability } from './status'; +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { + describeProgress, + emptyActionAvailability, + parseRunning, + parseStatusPayload, + progressOnLocalClock, +} from './status'; describe('parseStatusPayload', () => { it('remaps snake_case wire fields to the camelCase snapshot', () => { @@ -67,3 +73,28 @@ describe('parseRunning', () => { })).toThrow(/cannot read/); }); }); + +describe('infra progress', () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it('counts the server\'s duration, then ticks on this machine\'s clock', () => { + // The server says 20s had passed; this machine's clock runs 500s ahead. + const local = progressOnLocalClock({ sinceUnix: 1_000, asOfUnix: 1_020, waiting: 'db' }, 1_520); + expect(local).toEqual({ sinceUnix: 1_500, asOfUnix: 1_520, waiting: 'db' }); + expect(describeProgress(local, 1_520)).toBe('for 20s, waiting on: db'); + expect(describeProgress(local, 1_565)).toBe('for 1m05s, waiting on: db'); + }); + + it('moves a status answer\'s progress onto this machine\'s clock as it arrives', () => { + vi.useFakeTimers(); + vi.setSystemTime(9_000_000); + const snap = parseStatusPayload({ + infra: [{ node: 'db', node_type: 'pg', status: 'provisioning', progress: { sinceUnix: 100, asOfUnix: 130 } }], + }); + const progress = snap.infraNodes[0].progress!; + expect(progress).toEqual({ sinceUnix: 8_970, asOfUnix: 9_000 }); + expect(describeProgress(progress, Date.now() / 1000)).toBe('for 30s'); + }); +}); diff --git a/packages/weft-graph/src/status.ts b/packages/weft-graph/src/status.ts index 1db48af4..ae006452 100644 --- a/packages/weft-graph/src/status.ts +++ b/packages/weft-graph/src/status.ts @@ -135,7 +135,7 @@ export function parseStatusPayload(raw: RawStatusPayload): ActionAvailability { ...(n.failureStage !== undefined ? { failureStage: n.failureStage } : {}), ...(n.failureMessage !== undefined ? { failureMessage: n.failureMessage } : {}), ...(n.instance_copy_count !== undefined ? { instanceCopyCount: n.instance_copy_count } : {}), - ...(n.progress !== undefined ? { progress: n.progress } : {}), + ...(n.progress !== undefined ? { progress: progressOnLocalClock(n.progress, Date.now() / 1000) } : {}), })); return { availableActions: (Array.isArray(raw.available_actions) @@ -208,8 +208,20 @@ export function backendFromSnapshot(snapshot: ActionAvailability): BackendSnapsh }; } +/** The server's progress, moved onto this machine's clock: both stamps + * shift by how far this clock reads from the server's when the answer + * arrived (`localNowUnix`). Read against `Date.now()` afterwards, the + * duration is the server's at the answer plus the time elapsed here + * since, so it keeps ticking between answers whatever the two clocks + * say. */ +export function progressOnLocalClock(progress: ApplyProgress, localNowUnix: number): ApplyProgress { + const shift = localNowUnix - progress.asOfUnix; + return { ...progress, sinceUnix: progress.sinceUnix + shift, asOfUnix: localNowUnix }; +} + /** How far a start of an infra copy got, as a person reads it: "for - * 3m12s, waiting on: ...". */ + * 3m12s, waiting on: ...". `nowUnix` is on the clock the progress's + * stamps are on (this machine's, once `parseStatusPayload` moved it). */ // SYNC: describeProgress <-> crates/weft-core/src/infra/wire.rs ApplyProgress::describe export function describeProgress(progress: ApplyProgress, nowUnix: number): string { const secs = Math.max(0, Math.floor(nowUnix - progress.sinceUnix)); diff --git a/tangle/claude-code/.claude/skills/weft-instances/SKILL.md b/tangle/claude-code/.claude/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/claude-code/.claude/skills/weft-instances/SKILL.md +++ b/tangle/claude-code/.claude/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/claude-code/.claude/skills/weft-members/SKILL.md b/tangle/claude-code/.claude/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/claude-code/.claude/skills/weft-members/SKILL.md +++ b/tangle/claude-code/.claude/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/cline/.cline/skills/weft-instances/SKILL.md b/tangle/cline/.cline/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/cline/.cline/skills/weft-instances/SKILL.md +++ b/tangle/cline/.cline/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/cline/.cline/skills/weft-members/SKILL.md b/tangle/cline/.cline/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/cline/.cline/skills/weft-members/SKILL.md +++ b/tangle/cline/.cline/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/codex/.agents/skills/weft-instances/SKILL.md b/tangle/codex/.agents/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/codex/.agents/skills/weft-instances/SKILL.md +++ b/tangle/codex/.agents/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/codex/.agents/skills/weft-members/SKILL.md b/tangle/codex/.agents/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/codex/.agents/skills/weft-members/SKILL.md +++ b/tangle/codex/.agents/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/cursor/.cursor/skills/weft-instances/SKILL.md b/tangle/cursor/.cursor/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/cursor/.cursor/skills/weft-instances/SKILL.md +++ b/tangle/cursor/.cursor/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/cursor/.cursor/skills/weft-members/SKILL.md b/tangle/cursor/.cursor/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/cursor/.cursor/skills/weft-members/SKILL.md +++ b/tangle/cursor/.cursor/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/devin-desktop/.windsurf/skills/weft-instances/SKILL.md b/tangle/devin-desktop/.windsurf/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/devin-desktop/.windsurf/skills/weft-instances/SKILL.md +++ b/tangle/devin-desktop/.windsurf/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/devin-desktop/.windsurf/skills/weft-members/SKILL.md b/tangle/devin-desktop/.windsurf/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/devin-desktop/.windsurf/skills/weft-members/SKILL.md +++ b/tangle/devin-desktop/.windsurf/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/fallback/.agents/skills/weft-instances/SKILL.md b/tangle/fallback/.agents/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/fallback/.agents/skills/weft-instances/SKILL.md +++ b/tangle/fallback/.agents/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/fallback/.agents/skills/weft-members/SKILL.md b/tangle/fallback/.agents/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/fallback/.agents/skills/weft-members/SKILL.md +++ b/tangle/fallback/.agents/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/gemini-cli/.gemini/skills/weft-instances/SKILL.md b/tangle/gemini-cli/.gemini/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/gemini-cli/.gemini/skills/weft-instances/SKILL.md +++ b/tangle/gemini-cli/.gemini/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/gemini-cli/.gemini/skills/weft-members/SKILL.md b/tangle/gemini-cli/.gemini/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/gemini-cli/.gemini/skills/weft-members/SKILL.md +++ b/tangle/gemini-cli/.gemini/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/github-copilot/.github/skills/weft-instances/SKILL.md b/tangle/github-copilot/.github/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/github-copilot/.github/skills/weft-instances/SKILL.md +++ b/tangle/github-copilot/.github/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/github-copilot/.github/skills/weft-members/SKILL.md b/tangle/github-copilot/.github/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/github-copilot/.github/skills/weft-members/SKILL.md +++ b/tangle/github-copilot/.github/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/junie/.junie/skills/weft-instances/SKILL.md b/tangle/junie/.junie/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/junie/.junie/skills/weft-instances/SKILL.md +++ b/tangle/junie/.junie/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/junie/.junie/skills/weft-members/SKILL.md b/tangle/junie/.junie/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/junie/.junie/skills/weft-members/SKILL.md +++ b/tangle/junie/.junie/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/kilo-code/.kilo/skills/weft-instances/SKILL.md b/tangle/kilo-code/.kilo/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/kilo-code/.kilo/skills/weft-instances/SKILL.md +++ b/tangle/kilo-code/.kilo/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/kilo-code/.kilo/skills/weft-members/SKILL.md b/tangle/kilo-code/.kilo/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/kilo-code/.kilo/skills/weft-members/SKILL.md +++ b/tangle/kilo-code/.kilo/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of diff --git a/tangle/opencode/.opencode/skills/weft-instances/SKILL.md b/tangle/opencode/.opencode/skills/weft-instances/SKILL.md index b460219b..7bf5ea3c 100644 --- a/tangle/opencode/.opencode/skills/weft-instances/SKILL.md +++ b/tangle/opencode/.opencode/skills/weft-instances/SKILL.md @@ -231,10 +231,14 @@ infra node, or not per instance, is refused then); a name arriving on a wire is checked only when the run reaches it. Creating an instance, from a gated route the backend calls: -`MintInstanceToken` (if a browser will act inside it), then `Reply` with the -token so the request ends at once, then `StartInstanceInfra` if the program -has a per-instance infra node (it fires `done` once the container runs), then -`ActivateInstanceTriggers`. A container can take minutes to come up, so the +`MintInstanceToken` (if a browser will act inside it), then, if the program +has a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` +off (it fires `done` as soon as weft accepts the start), then `Reply` with the +token so the request ends at once, then a second `StartInstanceInfra` left on +(it fires `done` once the container runs), then `ActivateInstanceTriggers`. +Never put the `Reply` before the first start: a caller that lists the +instance's containers right after the answer would find nothing yet, while a +start weft accepted already reads `provisioning`. A container can take minutes to come up, so the request never waits for it: whoever shows it reads the `status` of that instance's displays (the `weft-consumers` skill has the listing) or polls a status route. If a trigger needs a value with no fallback, turning it on is @@ -288,7 +292,7 @@ something they do not cover calls the ctx directly in a node of its own: | Call | What it does | |---|---| | `ctx.instance()` | Which instance this run is for (the `CurrentInstance` node fires `instance` or `nobody`) | -| `ctx.infra("bridge").instance(id).start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | +| `ctx.infra("bridge").instance(id).start()` / `.request_start()` / `.stop(spec, stop_self)` / `.terminate(spec, stop_self)` / `.wipe(spec, stop_self)` / `.status()` | One instance's container; `terminate` keeps the disks listed in `keepOnTerminate`, `wipe` deletes them too | | `ctx.infra("bridge").copies()` | Every copy of the node: the shared one and each instance's (the `ListInstanceInfra` node) | | `ctx.triggers().instance(id).activate()` / `.deactivate(spec, stop_self)` | An instance's triggers (`.only([..])` narrows) | | `ctx.values().instance(id).get()` / `.set(step, field, value)` / `.clear(step, field)` / `.apply()` / `.forget()` | What was given for an instance's `@instance_filled` fields: read (the `GetInstanceValues` node), change in one go (the `SetInstanceValues` node; `apply()` returns the triggers it set up again), or forget all | @@ -303,7 +307,11 @@ something they do not cover calls the ctx directly in a node of its own: (the run parks between looks and holds no worker), so you can activate the instance's triggers right after it. If the container never comes up, `start()` fails with the reason, and so it does if somebody stops the -container while it waits (it never starts it again behind a pause). Every +container while it waits (it never starts it again behind a pause). +`request_start()` returns as soon as weft accepts the start, without waiting +for the container (`StartInstanceInfra` with `waitUntilRunning` off); a start +refused for a passing reason, such as a build in progress, is asked again +until it is accepted. Every reader of a container's state (`weft status`, `weft infra status`, `.status()`, `InstanceInfraStatus`) gives one answer: `none` for a copy never started (or terminated), and a start or stop on its way reads `provisioning` diff --git a/tangle/opencode/.opencode/skills/weft-members/SKILL.md b/tangle/opencode/.opencode/skills/weft-members/SKILL.md index 9a0f252d..c0687387 100644 --- a/tangle/opencode/.opencode/skills/weft-members/SKILL.md +++ b/tangle/opencode/.opencode/skills/weft-members/SKILL.md @@ -66,9 +66,12 @@ into that group through the group's inputs. be turned on the moment the instance is created. 2. **An admin route** the frontend's server calls, gated by `ApiKeyAuth` on the `Route`'s `auth`, its key only in that server's environment. It - onboards a person: writes the row, `MintInstanceToken`, `Reply` with the - token so the request ends at once, then `StartInstanceInfra` if there is a - per-instance infra node, then `ActivateInstanceTriggers`. The same route + onboards a person: writes the row, `MintInstanceToken`, then, if there is + a per-instance infra node, `StartInstanceInfra` with `waitUntilRunning` off, + then `Reply` with the token so the request ends at once, then a second + `StartInstanceInfra` left on and `ActivateInstanceTriggers`. A `Reply` + before the start would leave the person's container out of a list read + right after it. The same route (or siblings on the same gate) lists a person's instances, mints a token for the one they open, and removes one with `WipeInstance`. 3. **A settings page** where each person fills in what the program asks of