From 58521aabbc5fc1170a336e13d9f451b974f59d81 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:43:38 +0000 Subject: [PATCH 01/24] Initial plan From b3d36a9e9d9abe76c03b4d59ba58c8c21fef6b53 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:52:27 +0000 Subject: [PATCH 02/24] feat(engine): add runtime Copilot model controls Co-authored-by: jamesadevine <4742697+jamesadevine@users.noreply.github.com> --- src/compile/agentic_pipeline.rs | 2 +- src/compile/extensions/ado_aw_marker.rs | 97 +++++++-- src/engine.rs | 268 ++++++++++++++++++++++-- 3 files changed, 332 insertions(+), 35 deletions(-) diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index 176fb2e0b..0bf517e20 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -299,7 +299,7 @@ fn build_engine_setup( "/tmp/awf-tools/agent-prompt.md", Some("/tmp/awf-tools/mcp-config.json"), )?; - let engine_run_detection = detection_engine.invocation_with_config( + let engine_run_detection = detection_engine.detection_invocation_with_config( detection_engine_config, ctx.front_matter, extension_declarations, diff --git a/src/compile/extensions/ado_aw_marker.rs b/src/compile/extensions/ado_aw_marker.rs index 47c249ad7..74c7cbab7 100644 --- a/src/compile/extensions/ado_aw_marker.rs +++ b/src/compile/extensions/ado_aw_marker.rs @@ -24,6 +24,7 @@ use super::{CompileContext, CompilerExtension, Declarations, ExtensionPhase}; use crate::compile::ir::condition::Condition; +use crate::compile::ir::env::EnvValue; use crate::compile::ir::step::{BashStep, Step}; use crate::compile::shell::{Binding, ShellScript}; use crate::shell_script; @@ -57,15 +58,65 @@ shell_script! { EMIT_AW_INFO { interpreter: Bash, bindings: [AGENT_TEMP], - externals: [], + externals: [ADO_AW_MODEL_AGENT_COPILOT, ADO_AW_MODEL_DETECTION_COPILOT, ADO_AW_DEFAULT_MODEL_COPILOT], fragments: [aw_info_json], body: r#" set -eo pipefail +ado_aw_runtime_model() { + local specific_var="$1" + local specific_value="${!specific_var-}" + local candidate + for candidate in "$specific_value" "${ADO_AW_DEFAULT_MODEL_COPILOT:-}"; do + if [ -z "$candidate" ] \ + || [ "$candidate" = "\$($specific_var)" ] \ + || [ "$candidate" = "\$(ADO_AW_DEFAULT_MODEL_COPILOT)" ]; then + continue + fi + case "$candidate" in + *[!A-Za-z0-9._:-]*) + echo "ERROR: runtime Copilot model from $specific_var/ADO_AW_DEFAULT_MODEL_COPILOT contains invalid characters. Only ASCII alphanumerics, ., _, :, and - are allowed." >&2 + exit 1 + ;; + esac + printf '%s' "$candidate" + return 0 + done +} + +ado_aw_append_model_field() { + local field="$1" + local value="$2" + local file="$3" + if [ -z "$value" ] || grep -q "\"$field\"" "$file"; then + return 0 + fi + local tmp + tmp="$(mktemp)" + sed "$ s/}$/,\"$field\":\"$value\"/" "$file" > "$tmp" + mv "$tmp" "$file" +} + mkdir -p "$AGENT_TEMP/staging" cat >"$AGENT_TEMP/staging/aw_info.json" <<'AW_INFO_EOF' # ado-aw:fragment aw_info_json AW_INFO_EOF + +ADO_AW_INFO_JSON="$AGENT_TEMP/staging/aw_info.json" +if ! grep -q '"model"' "$ADO_AW_INFO_JSON"; then + ADO_AW_AGENT_RUNTIME_MODEL="$(ado_aw_runtime_model ADO_AW_MODEL_AGENT_COPILOT)" + ado_aw_append_model_field \ + "model" \ + "$ADO_AW_AGENT_RUNTIME_MODEL" \ + "$ADO_AW_INFO_JSON" +fi +if ! grep -q '"detection_model"' "$ADO_AW_INFO_JSON"; then + ADO_AW_DETECTION_RUNTIME_MODEL="$(ado_aw_runtime_model ADO_AW_MODEL_DETECTION_COPILOT)" + ado_aw_append_model_field \ + "detection_model" \ + "$ADO_AW_DETECTION_RUNTIME_MODEL" \ + "$ADO_AW_INFO_JSON" +fi "#, } } @@ -211,6 +262,18 @@ fn aw_info_bash_step(metadata: &CompileMetadata) -> BashStep { .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) .fragment("aw_info_json", metadata.aw_info_json()) .into_step("Emit aw_info.json") + .with_env( + crate::engine::ADO_AW_MODEL_AGENT_COPILOT, + EnvValue::pipeline_var(crate::engine::ADO_AW_MODEL_AGENT_COPILOT), + ) + .with_env( + crate::engine::ADO_AW_MODEL_DETECTION_COPILOT, + EnvValue::pipeline_var(crate::engine::ADO_AW_MODEL_DETECTION_COPILOT), + ) + .with_env( + crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT, + EnvValue::pipeline_var(crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT), + ) .with_condition(Condition::Always) } @@ -330,7 +393,10 @@ impl CompileMetadata { ); } if let Some(model) = &self.model { - object.insert("model".to_string(), serde_json::Value::String(model.clone())); + object.insert( + "model".to_string(), + serde_json::Value::String(model.clone()), + ); } if let Some(model) = &self.detection_model { object.insert( @@ -623,14 +689,24 @@ mod tests { "step missing build_definition_id macro:\n{}", step.script ); - assert!(!step.script.contains("detection_model")); - assert!(!step.script.contains("threat_detection_enabled")); + assert!(!step.script.contains("\"threat_detection_enabled\"")); + assert!( + step.env + .contains_key(crate::engine::ADO_AW_MODEL_AGENT_COPILOT) + ); + assert!( + step.env + .contains_key(crate::engine::ADO_AW_MODEL_DETECTION_COPILOT) + ); + assert!( + step.env + .contains_key(crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT) + ); } #[test] fn explicit_model_emits_aw_info_model_metadata() { - let fm = - parse_fm("name: t\ndescription: x\nengine:\n id: copilot\n model: some-model\n"); + let fm = parse_fm("name: t\ndescription: x\nengine:\n id: copilot\n model: some-model\n"); let input_path = Path::new("agents/foo.md"); let ctx = CompileContext { agent_name: &fm.name, @@ -670,8 +746,7 @@ mod tests { let steps = agent_prepare_steps(&ctx); let step = bash_step(&steps[1]); assert!( - step.script - .contains("\"threat_detection_enabled\":false"), + step.script.contains("\"threat_detection_enabled\":false"), "{}", step.script ); @@ -690,9 +765,7 @@ mod tests { #[test] fn explicit_default_threat_detection_emits_enabled_state_only() { - let fm = parse_fm( - "name: t\ndescription: x\nsafe-outputs:\n threat-detection: true\n", - ); + let fm = parse_fm("name: t\ndescription: x\nsafe-outputs:\n threat-detection: true\n"); let input_path = Path::new("agents/foo.md"); let ctx = CompileContext { agent_name: &fm.name, @@ -711,7 +784,7 @@ mod tests { step.script ); assert!(!step.script.contains("\"detection_engine\"")); - assert!(!step.script.contains("\"detection_model\"")); + assert!(step.script.contains("ADO_AW_MODEL_DETECTION_COPILOT")); } #[test] diff --git a/src/engine.rs b/src/engine.rs index b01cbfe43..9ccf23c97 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -31,6 +31,9 @@ pub const BLOCKED_ENV_KEYS: &[&str] = &[ "COPILOT_OTEL_ENABLED", "COPILOT_OTEL_EXPORTER_TYPE", "COPILOT_OTEL_FILE_EXPORTER_PATH", + "ADO_AW_MODEL_AGENT_COPILOT", + "ADO_AW_MODEL_DETECTION_COPILOT", + "ADO_AW_DEFAULT_MODEL_COPILOT", // Shell/system vars that could affect AWF or pipeline behavior "PATH", "HOME", @@ -74,6 +77,28 @@ pub const COPILOT_PROVIDER_EXPR_ENV_KEYS: &[&str] = &[ /// select the provider subset for the Detection step. const COPILOT_PROVIDER_PREFIX: &str = "COPILOT_PROVIDER_"; +/// Runtime pipeline-variable override for the main Copilot agent model. +pub const ADO_AW_MODEL_AGENT_COPILOT: &str = "ADO_AW_MODEL_AGENT_COPILOT"; +/// Runtime pipeline-variable override for the Detection Copilot model. +pub const ADO_AW_MODEL_DETECTION_COPILOT: &str = "ADO_AW_MODEL_DETECTION_COPILOT"; +/// Shared runtime pipeline-variable fallback for Copilot models. +pub const ADO_AW_DEFAULT_MODEL_COPILOT: &str = "ADO_AW_DEFAULT_MODEL_COPILOT"; + +#[derive(Debug, Clone, Copy)] +enum RuntimeModelRole { + Agent, + Detection, +} + +impl RuntimeModelRole { + fn specific_var(self) -> &'static str { + match self { + RuntimeModelRole::Agent => ADO_AW_MODEL_AGENT_COPILOT, + RuntimeModelRole::Detection => ADO_AW_MODEL_DETECTION_COPILOT, + } + } +} + /// Returns true when `key` is an allowlisted BYOM/BYOK provider env-var key that /// may carry ADO macro/runtime expressions in `engine.env`. Case-sensitive: see /// [`COPILOT_PROVIDER_EXPR_ENV_KEYS`] for why exact case is required. @@ -431,11 +456,12 @@ impl Engine { prompt_path, mcp_config_path, &args, + Some(RuntimeModelRole::Agent), ) } - /// Generate an invocation using an explicit engine configuration. - pub fn invocation_with_config( + /// Generate a Detection-job invocation using an explicit engine configuration. + pub fn detection_invocation_with_config( &self, engine_config: &EngineConfig, front_matter: &FrontMatter, @@ -444,7 +470,13 @@ impl Engine { mcp_config_path: Option<&str>, ) -> Result { let args = self.args_with_config(engine_config, front_matter, extension_declarations)?; - self.invocation_with_args(engine_config, prompt_path, mcp_config_path, &args) + self.invocation_with_args( + engine_config, + prompt_path, + mcp_config_path, + &args, + Some(RuntimeModelRole::Detection), + ) } fn invocation_with_args( @@ -453,6 +485,7 @@ impl Engine { prompt_path: &str, mcp_config_path: Option<&str>, args: &str, + runtime_model_role: Option, ) -> Result { match self { Engine::Copilot => { @@ -474,6 +507,7 @@ impl Engine { prompt_path, mcp_config_path, args, + runtime_model_role.filter(|_| engine_config.model().is_none()), )) } } @@ -657,20 +691,8 @@ fn copilot_args( let mut params = Vec::new(); - // Validate model name to prevent shell injection — copilot_params are embedded - // inside a single-quoted bash string in the AWF command. if let Some(model) = engine_config.model() { - if model.is_empty() - || !model - .chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | ':' | '-')) - { - anyhow::bail!( - "Model name '{}' contains invalid characters. \ - Only ASCII alphanumerics, '.', '_', ':', and '-' are allowed.", - model - ); - } + validate_model_name(model)?; params.push(format!("--model {}", model)); } if let Some(0) = engine_config.timeout_minutes() { @@ -740,6 +762,24 @@ fn copilot_args( Ok(params.join(" ")) } +fn validate_model_name(model: &str) -> Result<()> { + // Validate model name to prevent shell injection — copilot_params are embedded + // inside a single-quoted bash string in the AWF command, and runtime-selected + // models are later passed as quoted arguments. + if model.is_empty() + || !model + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | ':' | '-')) + { + anyhow::bail!( + "Model name '{}' contains invalid characters. \ + Only ASCII alphanumerics, '.', '_', ':', and '-' are allowed.", + model + ); + } + Ok(()) +} + /// The masked, same-job pipeline variable the `github-app-token` ado-script /// bundle sets. When `engine.github-app-token` is configured, the Copilot /// engine's `GITHUB_TOKEN` is sourced from this variable (set by the mint step @@ -835,6 +875,9 @@ fn copilot_env(engine_config: &EngineConfig) -> Result { "COPILOT_OTEL_EXPORTER_TYPE: \"file\"".to_string(), "COPILOT_OTEL_FILE_EXPORTER_PATH: \"/tmp/awf-tools/staging/otel.jsonl\"".to_string(), ]; + if engine_config.model().is_none() { + add_runtime_model_env_lines(&mut lines, RuntimeModelRole::Agent); + } // Wire engine.env — merge user-provided environment variables plus any // `COPILOT_PROVIDER_*` vars derived from an `engine.provider` block. @@ -851,6 +894,23 @@ fn copilot_env(engine_config: &EngineConfig) -> Result { Ok(lines.join("\n")) } +fn add_runtime_model_env_lines(lines: &mut Vec, role: RuntimeModelRole) { + let specific = role.specific_var(); + lines.push(format!("{specific}: $({specific})")); + lines.push(format!( + "{ADO_AW_DEFAULT_MODEL_COPILOT}: $({ADO_AW_DEFAULT_MODEL_COPILOT})" + )); +} + +fn add_runtime_model_env_pairs(pairs: &mut Vec<(String, String)>, role: RuntimeModelRole) { + let specific = role.specific_var(); + pairs.push((specific.to_string(), format!("$({specific})"))); + pairs.push(( + ADO_AW_DEFAULT_MODEL_COPILOT.to_string(), + format!("$({ADO_AW_DEFAULT_MODEL_COPILOT})"), + )); +} + /// Return the `COPILOT_PROVIDER_*` entries of `engine.env` as validated, /// **raw** `(key, value)` pairs (value un-rendered; empty vec when none present), /// sorted by key. @@ -928,6 +988,9 @@ pub fn copilot_detection_env(engine_config: &EngineConfig) -> Result, args: &str, + runtime_model_role: Option, ) -> String { - let mut parts = vec![ + let mut base_parts = vec![ command_path.to_string(), format!("--prompt=\"$(cat {prompt_path})\""), ]; if let Some(mcp_path) = mcp_config_path { - parts.push(format!("--additional-mcp-config @{mcp_path}")); + base_parts.push(format!("--additional-mcp-config @{mcp_path}")); } if !args.is_empty() { - parts.push(args.to_string()); + base_parts.push(args.to_string()); + } + + let Some(role) = runtime_model_role else { + return base_parts.join(" "); + }; + + let mut model_parts = vec![ + command_path.to_string(), + format!("--prompt=\"$(cat {prompt_path})\""), + ]; + if let Some(mcp_path) = mcp_config_path { + model_parts.push(format!("--additional-mcp-config @{mcp_path}")); + } + model_parts.push("--model \"$ADO_AW_EFFECTIVE_MODEL\"".to_string()); + if !args.is_empty() { + model_parts.push(args.to_string()); } - parts.join(" ") + format!( + "{}\nif [ -n \"$ADO_AW_EFFECTIVE_MODEL\" ]; then\n {}\nelse\n {}\nfi", + runtime_model_preamble(role), + model_parts.join(" "), + base_parts.join(" ") + ) +} + +fn runtime_model_preamble(role: RuntimeModelRole) -> String { + let specific = role.specific_var(); + format!( + "ADO_AW_EFFECTIVE_MODEL=\"\"\n\ + for ADO_AW_CANDIDATE_MODEL in \"${{{specific}:-}}\" \"${{{ADO_AW_DEFAULT_MODEL_COPILOT}:-}}\"; do\n\ + \x20\x20if [ -z \"$ADO_AW_CANDIDATE_MODEL\" ] || [ \"$ADO_AW_CANDIDATE_MODEL\" = \"\\$({specific})\" ] || [ \"$ADO_AW_CANDIDATE_MODEL\" = \"\\$({ADO_AW_DEFAULT_MODEL_COPILOT})\" ]; then\n\ + \x20\x20\x20\x20continue\n\ + \x20\x20fi\n\ + \x20\x20case \"$ADO_AW_CANDIDATE_MODEL\" in\n\ + \x20\x20\x20\x20*[!A-Za-z0-9._:-]*)\n\ + \x20\x20\x20\x20\x20\x20echo \"ERROR: runtime Copilot model from {specific}/{ADO_AW_DEFAULT_MODEL_COPILOT} contains invalid characters. Only ASCII alphanumerics, ., _, :, and - are allowed.\" >&2\n\ + \x20\x20\x20\x20\x20\x20exit 1\n\ + \x20\x20\x20\x20\x20\x20;;\n\ + \x20\x20esac\n\ + \x20\x20ADO_AW_EFFECTIVE_MODEL=\"$ADO_AW_CANDIDATE_MODEL\"\n\ + \x20\x20break\n\ + done" + ) } #[cfg(test)] mod tests { use super::{ + ADO_AW_DEFAULT_MODEL_COPILOT, ADO_AW_MODEL_AGENT_COPILOT, ADO_AW_MODEL_DETECTION_COPILOT, Engine, GITHUB_APP_TOKEN_VAR, copilot_byom_active, copilot_byom_credential_keys, - copilot_provider_env, get_engine, github_app_token_secrecy_advisory, - github_token_source_var, normalize_version_tag, validate_engine_feature_support, + copilot_detection_env, copilot_provider_env, get_engine, + github_app_token_secrecy_advisory, github_token_source_var, normalize_version_tag, + validate_engine_feature_support, }; use crate::compile::{ extensions::{CompileContext, CompilerExtension, Declarations, collect_extensions}, @@ -1419,6 +1526,65 @@ mod tests { assert!(params.contains("--model gpt-5")); } + #[test] + fn copilot_invocation_uses_runtime_agent_model_precedence_when_no_explicit_model() { + let (front_matter, _) = + parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); + let invocation = Engine::Copilot + .invocation( + &front_matter, + &declarations_for(&front_matter), + "/tmp/prompt.md", + Some("/tmp/mcp.json"), + ) + .unwrap(); + + assert!(invocation.contains("ADO_AW_MODEL_AGENT_COPILOT")); + assert!(invocation.contains("ADO_AW_DEFAULT_MODEL_COPILOT")); + assert!(invocation.contains("--model \"$ADO_AW_EFFECTIVE_MODEL\"")); + assert!(invocation.contains("*[!A-Za-z0-9._:-]*)")); + } + + #[test] + fn copilot_invocation_keeps_explicit_model_static() { + let (front_matter, _) = parse_markdown( + "---\nname: test\ndescription: test\nengine:\n id: copilot\n model: gpt-5\n---\n", + ) + .unwrap(); + let invocation = Engine::Copilot + .invocation( + &front_matter, + &declarations_for(&front_matter), + "/tmp/prompt.md", + None, + ) + .unwrap(); + + assert!(invocation.contains("--model gpt-5")); + assert!(!invocation.contains("ADO_AW_MODEL_AGENT_COPILOT")); + assert!(!invocation.contains("ADO_AW_EFFECTIVE_MODEL")); + } + + #[test] + fn copilot_detection_invocation_uses_independent_runtime_model() { + let (front_matter, _) = + parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); + let invocation = Engine::Copilot + .detection_invocation_with_config( + &front_matter.engine, + &front_matter, + &declarations_for(&front_matter), + "/tmp/threat.md", + None, + ) + .unwrap(); + + assert!(invocation.contains("ADO_AW_MODEL_DETECTION_COPILOT")); + assert!(invocation.contains("ADO_AW_DEFAULT_MODEL_COPILOT")); + assert!(!invocation.contains("ADO_AW_MODEL_AGENT_COPILOT")); + assert!(invocation.contains("--model \"$ADO_AW_EFFECTIVE_MODEL\"")); + } + #[test] fn copilot_engine_rejects_invalid_explicit_model() { let (front_matter, _) = parse_markdown( @@ -1445,6 +1611,64 @@ mod tests { assert!(!env.contains("AZURE_DEVOPS_EXT_PAT")); } + #[test] + fn copilot_engine_env_maps_runtime_agent_model_vars_when_no_explicit_model() { + let (front_matter, _) = + parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); + let env = Engine::Copilot.env(&front_matter.engine).unwrap(); + + assert!(env.contains(&format!( + "{ADO_AW_MODEL_AGENT_COPILOT}: $({ADO_AW_MODEL_AGENT_COPILOT})" + ))); + assert!(env.contains(&format!( + "{ADO_AW_DEFAULT_MODEL_COPILOT}: $({ADO_AW_DEFAULT_MODEL_COPILOT})" + ))); + } + + #[test] + fn copilot_engine_env_omits_runtime_agent_model_vars_for_explicit_model() { + let (front_matter, _) = parse_markdown( + "---\nname: test\ndescription: test\nengine:\n id: copilot\n model: gpt-5\n---\n", + ) + .unwrap(); + let env = Engine::Copilot.env(&front_matter.engine).unwrap(); + + assert!(!env.contains(ADO_AW_MODEL_AGENT_COPILOT)); + assert!(!env.contains(ADO_AW_DEFAULT_MODEL_COPILOT)); + } + + #[test] + fn copilot_detection_env_maps_independent_runtime_model_vars() { + let (front_matter, _) = + parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); + let env = copilot_detection_env(&front_matter.engine).unwrap(); + + assert!(env.contains(&( + ADO_AW_MODEL_DETECTION_COPILOT.to_string(), + format!("$({ADO_AW_MODEL_DETECTION_COPILOT})") + ))); + assert!(env.contains(&( + ADO_AW_DEFAULT_MODEL_COPILOT.to_string(), + format!("$({ADO_AW_DEFAULT_MODEL_COPILOT})") + ))); + assert!(!env.iter().any(|(key, _)| key == ADO_AW_MODEL_AGENT_COPILOT)); + } + + #[test] + fn copilot_engine_env_rejects_user_runtime_model_var_override() { + let (front_matter, _) = parse_markdown( + "---\nname: test\ndescription: test\nengine:\n id: copilot\n env:\n ADO_AW_MODEL_AGENT_COPILOT: gpt-5\n---\n", + ) + .unwrap(); + let err = Engine::Copilot + .env(&front_matter.engine) + .unwrap_err() + .to_string(); + + assert!(err.contains("compiler-controlled environment variable")); + assert!(err.contains(ADO_AW_MODEL_AGENT_COPILOT)); + } + #[test] fn copilot_engine_env_sources_github_token_from_app_token_var_when_configured() { let src = "---\nname: test\ndescription: test\nengine:\n id: copilot\n \ From 5cff95bffa98fd27e71bc21bc866cdc215d8010c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:53:04 +0000 Subject: [PATCH 03/24] docs(engine): document runtime model controls Co-authored-by: jamesadevine <4742697+jamesadevine@users.noreply.github.com> --- docs/engine.md | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/engine.md b/docs/engine.md index ce7724e07..cc7434473 100644 --- a/docs/engine.md +++ b/docs/engine.md @@ -22,7 +22,7 @@ engine: | Field | Type | Default | Description | |-------|------|---------|-------------| | `id` | string | `copilot` | Engine identifier. Currently only `copilot` (GitHub Copilot CLI) is supported. | -| `model` | string | *(none)* | AI model to use (e.g., `gpt-5-mini`). When omitted, the compiler does not emit `--model` and the Copilot CLI chooses its own default. When set, the compiler passes the value directly to the Copilot CLI `--model` flag — any model identifier the Copilot CLI accepts is valid. | +| `model` | string | *(none)* | AI model to use (e.g., `gpt-5-mini`). When set, the compiler passes the value directly to the Copilot CLI `--model` flag. When omitted, runtime model controls can select a model; if no runtime control is set, the compiler omits `--model` and the Copilot CLI chooses its own default. | | `timeout-minutes` | integer | *(none)* | Maximum time in minutes the agent job is allowed to run. Sets `timeoutInMinutes` on the `Agent` job in the generated pipeline. | | `version` | string | *(none)* | Engine CLI version to install (e.g., `"1.0.70"`, `"latest"`). Overrides the pinned `COPILOT_CLI_VERSION`. Set to `"latest"` to use the newest available version. | | `agent` | string | *(none)* | Custom agent file identifier (Copilot only). Adds `--agent ` to the CLI invocation, selecting a custom agent from `.github/agents/`. | @@ -34,6 +34,34 @@ engine: | `github-app-token` | map | *(none)* | GitHub App-backed Copilot engine authentication. When set, the compiler mints (and, by default, revokes) a GitHub App installation token in the Agent and Detection jobs and sources `GITHUB_TOKEN` from it (for Copilot only). See [GitHub App-backed Copilot engine auth](#github-app-backed-copilot-engine-auth). | +### Runtime model controls + +For the Copilot engine, operators can switch models at Azure DevOps pipeline +runtime without editing workflow markdown or recompiling lock files. Configure +these as pipeline variables or variable-group entries: + +| Variable | Applies to | +|----------|------------| +| `ADO_AW_MODEL_AGENT_COPILOT` | Agent job only | +| `ADO_AW_MODEL_DETECTION_COPILOT` | Detection job only | +| `ADO_AW_DEFAULT_MODEL_COPILOT` | Fallback for both jobs | + +Precedence is: + +1. Explicit `engine.model` for the effective engine config. +2. Role-specific runtime variable (`ADO_AW_MODEL_AGENT_COPILOT` or + `ADO_AW_MODEL_DETECTION_COPILOT`). +3. Shared runtime variable (`ADO_AW_DEFAULT_MODEL_COPILOT`). +4. Existing default behavior (no `--model`; the Copilot CLI chooses). + +Detection uses its effective engine config after applying +`safe-outputs.threat-detection.engine`, so an inherited or nested explicit model +still wins over runtime variables. Runtime values are passed through typed step +environment mappings, validated for model-identifier characters at runtime, and +then supplied to Copilot as a quoted `--model` argument. The emitted +`aw_info.json` run metadata records the effective runtime-selected model when a +runtime variable is used. + ### `timeout-minutes` The `timeout-minutes` field sets a wall-clock limit (in minutes) for the entire agent job. It maps to the Azure DevOps `timeoutInMinutes` job property on `Agent`. This is useful for: @@ -246,7 +274,8 @@ runtime (raw `engine.env` cross-job macros like `$(Setup.FOUNDRY_TOKEN)` do | `token` | optional | `COPILOT_PROVIDER_API_KEY` | Compiler-minted credential via Azure CLI (see below). Mutually exclusive with `api-key`. | | `api-key` | optional | `COPILOT_PROVIDER_API_KEY` | Static API key, typically a `$(VAR)` secret pipeline variable. Mutually exclusive with `token`. | -The model itself is set via `engine.model` (or a `COPILOT_MODEL` env var). +The model itself is set via `engine.model` or the +[`ADO_AW_MODEL_*`](#runtime-model-controls) runtime controls. #### Compiler-owned token acquisition (`provider.token`) From 3fd45455ac8ada6ea5fe97030a633a40187aee26 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:54:28 +0000 Subject: [PATCH 04/24] test(shell): validate runtime model metadata script Co-authored-by: jamesadevine <4742697+jamesadevine@users.noreply.github.com> --- src/compile/extensions/ado_aw_marker.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/compile/extensions/ado_aw_marker.rs b/src/compile/extensions/ado_aw_marker.rs index 74c7cbab7..0f1ed1ee0 100644 --- a/src/compile/extensions/ado_aw_marker.rs +++ b/src/compile/extensions/ado_aw_marker.rs @@ -65,7 +65,7 @@ set -eo pipefail ado_aw_runtime_model() { local specific_var="$1" - local specific_value="${!specific_var-}" + local specific_value="$2" local candidate for candidate in "$specific_value" "${ADO_AW_DEFAULT_MODEL_COPILOT:-}"; do if [ -z "$candidate" ] \ @@ -104,14 +104,18 @@ AW_INFO_EOF ADO_AW_INFO_JSON="$AGENT_TEMP/staging/aw_info.json" if ! grep -q '"model"' "$ADO_AW_INFO_JSON"; then - ADO_AW_AGENT_RUNTIME_MODEL="$(ado_aw_runtime_model ADO_AW_MODEL_AGENT_COPILOT)" + ADO_AW_AGENT_RUNTIME_MODEL="$(ado_aw_runtime_model \ + ADO_AW_MODEL_AGENT_COPILOT \ + "$ADO_AW_MODEL_AGENT_COPILOT")" ado_aw_append_model_field \ "model" \ "$ADO_AW_AGENT_RUNTIME_MODEL" \ "$ADO_AW_INFO_JSON" fi if ! grep -q '"detection_model"' "$ADO_AW_INFO_JSON"; then - ADO_AW_DETECTION_RUNTIME_MODEL="$(ado_aw_runtime_model ADO_AW_MODEL_DETECTION_COPILOT)" + ADO_AW_DETECTION_RUNTIME_MODEL="$(ado_aw_runtime_model \ + ADO_AW_MODEL_DETECTION_COPILOT \ + "$ADO_AW_MODEL_DETECTION_COPILOT")" ado_aw_append_model_field \ "detection_model" \ "$ADO_AW_DETECTION_RUNTIME_MODEL" \ @@ -657,7 +661,7 @@ mod tests { step.script ); assert!( - !step.script.contains("\"model\""), + !step.script.contains("\"model\":\""), "step should omit model when no model is configured:\n{}", step.script ); From c2d4cf2c7a292276112ea1b0c9b991cc4341fe22 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:01:15 +0000 Subject: [PATCH 05/24] fix(engine): harden runtime model selection Co-authored-by: jamesadevine <4742697+jamesadevine@users.noreply.github.com> --- src/compile/extensions/ado_aw_marker.rs | 18 +++++++++++++- src/engine.rs | 31 ++++++++++++++++++------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/compile/extensions/ado_aw_marker.rs b/src/compile/extensions/ado_aw_marker.rs index 0f1ed1ee0..9b99918bb 100644 --- a/src/compile/extensions/ado_aw_marker.rs +++ b/src/compile/extensions/ado_aw_marker.rs @@ -68,6 +68,7 @@ ado_aw_runtime_model() { local specific_value="$2" local candidate for candidate in "$specific_value" "${ADO_AW_DEFAULT_MODEL_COPILOT:-}"; do + # Azure DevOps leaves an undefined macro as the literal $(VAR); treat that as unset. if [ -z "$candidate" ] \ || [ "$candidate" = "\$($specific_var)" ] \ || [ "$candidate" = "\$(ADO_AW_DEFAULT_MODEL_COPILOT)" ]; then @@ -82,6 +83,7 @@ ado_aw_runtime_model() { printf '%s' "$candidate" return 0 done + return 0 } ado_aw_append_model_field() { @@ -91,9 +93,23 @@ ado_aw_append_model_field() { if [ -z "$value" ] || grep -q "\"$field\"" "$file"; then return 0 fi + local json local tmp + local separator="," + json="$(cat "$file")" + if [ "$json" = "{}" ]; then + separator="" + else + case "$json" in + \{*\}) ;; + *) + echo "ERROR: aw_info.json is not a single-line JSON object" >&2 + exit 1 + ;; + esac + fi tmp="$(mktemp)" - sed "$ s/}$/,\"$field\":\"$value\"/" "$file" > "$tmp" + printf '%s%s"%s":"%s"}' "${json%?}" "$separator" "$field" "$value" > "$tmp" mv "$tmp" "$file" } diff --git a/src/engine.rs b/src/engine.rs index 9ccf23c97..ea84eeb32 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -460,6 +460,23 @@ impl Engine { ) } + /// Generate an invocation using an explicit engine configuration. + /// + /// Retained for callers that need a plain invocation with no role-specific + /// runtime model controls. + #[allow(dead_code)] + pub fn invocation_with_config( + &self, + engine_config: &EngineConfig, + front_matter: &FrontMatter, + extension_declarations: &[Declarations], + prompt_path: &str, + mcp_config_path: Option<&str>, + ) -> Result { + let args = self.args_with_config(engine_config, front_matter, extension_declarations)?; + self.invocation_with_args(engine_config, prompt_path, mcp_config_path, &args, None) + } + /// Generate a Detection-job invocation using an explicit engine configuration. pub fn detection_invocation_with_config( &self, @@ -1417,15 +1434,16 @@ fn copilot_invocation( args: &str, runtime_model_role: Option, ) -> String { - let mut base_parts = vec![ + let mut common_parts = vec![ command_path.to_string(), format!("--prompt=\"$(cat {prompt_path})\""), ]; if let Some(mcp_path) = mcp_config_path { - base_parts.push(format!("--additional-mcp-config @{mcp_path}")); + common_parts.push(format!("--additional-mcp-config @{mcp_path}")); } + let mut base_parts = common_parts.clone(); if !args.is_empty() { base_parts.push(args.to_string()); } @@ -1434,13 +1452,7 @@ fn copilot_invocation( return base_parts.join(" "); }; - let mut model_parts = vec![ - command_path.to_string(), - format!("--prompt=\"$(cat {prompt_path})\""), - ]; - if let Some(mcp_path) = mcp_config_path { - model_parts.push(format!("--additional-mcp-config @{mcp_path}")); - } + let mut model_parts = common_parts; model_parts.push("--model \"$ADO_AW_EFFECTIVE_MODEL\"".to_string()); if !args.is_empty() { model_parts.push(args.to_string()); @@ -1459,6 +1471,7 @@ fn runtime_model_preamble(role: RuntimeModelRole) -> String { format!( "ADO_AW_EFFECTIVE_MODEL=\"\"\n\ for ADO_AW_CANDIDATE_MODEL in \"${{{specific}:-}}\" \"${{{ADO_AW_DEFAULT_MODEL_COPILOT}:-}}\"; do\n\ + \x20\x20# Azure DevOps leaves an undefined macro as the literal $(VAR); treat that as unset.\n\ \x20\x20if [ -z \"$ADO_AW_CANDIDATE_MODEL\" ] || [ \"$ADO_AW_CANDIDATE_MODEL\" = \"\\$({specific})\" ] || [ \"$ADO_AW_CANDIDATE_MODEL\" = \"\\$({ADO_AW_DEFAULT_MODEL_COPILOT})\" ]; then\n\ \x20\x20\x20\x20continue\n\ \x20\x20fi\n\ From d34ee67b36f777744e0257d8ceab7464dd64040d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:29:11 +0000 Subject: [PATCH 06/24] style(engine): use raw string for runtime model preamble Co-authored-by: jamesadevine <4742697+jamesadevine@users.noreply.github.com> --- src/engine.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/engine.rs b/src/engine.rs index ea84eeb32..1444e6487 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -1469,21 +1469,21 @@ fn copilot_invocation( fn runtime_model_preamble(role: RuntimeModelRole) -> String { let specific = role.specific_var(); format!( - "ADO_AW_EFFECTIVE_MODEL=\"\"\n\ - for ADO_AW_CANDIDATE_MODEL in \"${{{specific}:-}}\" \"${{{ADO_AW_DEFAULT_MODEL_COPILOT}:-}}\"; do\n\ - \x20\x20# Azure DevOps leaves an undefined macro as the literal $(VAR); treat that as unset.\n\ - \x20\x20if [ -z \"$ADO_AW_CANDIDATE_MODEL\" ] || [ \"$ADO_AW_CANDIDATE_MODEL\" = \"\\$({specific})\" ] || [ \"$ADO_AW_CANDIDATE_MODEL\" = \"\\$({ADO_AW_DEFAULT_MODEL_COPILOT})\" ]; then\n\ - \x20\x20\x20\x20continue\n\ - \x20\x20fi\n\ - \x20\x20case \"$ADO_AW_CANDIDATE_MODEL\" in\n\ - \x20\x20\x20\x20*[!A-Za-z0-9._:-]*)\n\ - \x20\x20\x20\x20\x20\x20echo \"ERROR: runtime Copilot model from {specific}/{ADO_AW_DEFAULT_MODEL_COPILOT} contains invalid characters. Only ASCII alphanumerics, ., _, :, and - are allowed.\" >&2\n\ - \x20\x20\x20\x20\x20\x20exit 1\n\ - \x20\x20\x20\x20\x20\x20;;\n\ - \x20\x20esac\n\ - \x20\x20ADO_AW_EFFECTIVE_MODEL=\"$ADO_AW_CANDIDATE_MODEL\"\n\ - \x20\x20break\n\ - done" + r#"ADO_AW_EFFECTIVE_MODEL="" +for ADO_AW_CANDIDATE_MODEL in "${{{specific}:-}}" "${{{ADO_AW_DEFAULT_MODEL_COPILOT}:-}}"; do + # Azure DevOps leaves an undefined macro as the literal $(VAR); treat that as unset. + if [ -z "$ADO_AW_CANDIDATE_MODEL" ] || [ "$ADO_AW_CANDIDATE_MODEL" = "\$({specific})" ] || [ "$ADO_AW_CANDIDATE_MODEL" = "\$({ADO_AW_DEFAULT_MODEL_COPILOT})" ]; then + continue + fi + case "$ADO_AW_CANDIDATE_MODEL" in + *[!A-Za-z0-9._:-]*) + echo "ERROR: runtime Copilot model from {specific}/{ADO_AW_DEFAULT_MODEL_COPILOT} contains invalid characters. Only ASCII alphanumerics, ., _, :, and - are allowed." >&2 + exit 1 + ;; + esac + ADO_AW_EFFECTIVE_MODEL="$ADO_AW_CANDIDATE_MODEL" + break +done"# ) } From cd04b82bccbe47e45eaa048c678ea82a953cd4ca Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 26 Sep 2026 06:36:30 +0000 Subject: [PATCH 07/24] test(engine): cover runtime model feedback Co-authored-by: jamesadevine <4742697+jamesadevine@users.noreply.github.com> --- src/compile/extensions/ado_aw_marker.rs | 77 +++++++++++++++++ src/engine.rs | 109 +++++++++++++++++++----- 2 files changed, 165 insertions(+), 21 deletions(-) diff --git a/src/compile/extensions/ado_aw_marker.rs b/src/compile/extensions/ado_aw_marker.rs index 9b99918bb..b2b5e1c83 100644 --- a/src/compile/extensions/ado_aw_marker.rs +++ b/src/compile/extensions/ado_aw_marker.rs @@ -75,6 +75,7 @@ ado_aw_runtime_model() { continue fi case "$candidate" in + # Keep this character set in sync with engine::validate_model_name and runtime_model_preamble. *[!A-Za-z0-9._:-]*) echo "ERROR: runtime Copilot model from $specific_var/ADO_AW_DEFAULT_MODEL_COPILOT contains invalid characters. Only ASCII alphanumerics, ., _, :, and - are allowed." >&2 exit 1 @@ -543,8 +544,10 @@ fn bash_single_quote_escape(s: &str) -> String { mod tests { use super::*; use crate::compile::extensions::CompileContext; + use crate::compile::shell::ShellScript; use crate::compile::types::FrontMatter; use std::path::Path; + use std::process::{Command, Output}; fn parse_fm(yaml: &str) -> FrontMatter { serde_yaml::from_str(yaml).expect("front matter parses") @@ -564,6 +567,26 @@ mod tests { } } + fn run_aw_info_script(envs: &[(&str, &str)], aw_info_json: &str) -> (Output, tempfile::TempDir) { + let temp = tempfile::tempdir().expect("temp dir"); + let script = ShellScript::new(&EMIT_AW_INFO) + .bind_text("AGENT_TEMP", temp.path().display().to_string()) + .fragment("aw_info_json", aw_info_json.to_string()) + .render(); + let mut command = Command::new("bash"); + command.arg("-c").arg(script).env_clear(); + for (key, value) in envs { + command.env(key, value); + } + (command.output().expect("bash should run"), temp) + } + + fn read_aw_info_json(temp: &tempfile::TempDir) -> serde_json::Value { + let path = temp.path().join("staging/aw_info.json"); + let contents = std::fs::read_to_string(path).expect("aw_info.json should be written"); + serde_json::from_str(&contents).expect("aw_info.json should parse") + } + #[test] fn returns_no_step_when_input_path_absent() { let fm = parse_fm("name: t\ndescription: x\n"); @@ -724,6 +747,60 @@ mod tests { ); } + #[test] + fn aw_info_runtime_models_prefer_role_specific_over_default() { + let (output, temp) = run_aw_info_script( + &[ + (crate::engine::ADO_AW_MODEL_AGENT_COPILOT, "agent-model"), + ( + crate::engine::ADO_AW_MODEL_DETECTION_COPILOT, + "detector-model", + ), + (crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT, "default-model"), + ], + r#"{"schema":"ado-aw/aw_info/1"}"#, + ); + + assert!(output.status.success(), "{output:?}"); + let value = read_aw_info_json(&temp); + assert_eq!(value["model"], "agent-model"); + assert_eq!(value["detection_model"], "detector-model"); + } + + #[test] + fn aw_info_runtime_models_use_default_when_specific_missing_or_unexpanded() { + let (output, temp) = run_aw_info_script( + &[ + ( + crate::engine::ADO_AW_MODEL_AGENT_COPILOT, + "$(ADO_AW_MODEL_AGENT_COPILOT)", + ), + (crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT, "default-model"), + ], + r#"{"schema":"ado-aw/aw_info/1"}"#, + ); + + assert!(output.status.success(), "{output:?}"); + let value = read_aw_info_json(&temp); + assert_eq!(value["model"], "default-model"); + assert_eq!(value["detection_model"], "default-model"); + } + + #[test] + fn aw_info_runtime_model_rejects_invalid_value() { + let (output, _temp) = run_aw_info_script( + &[( + crate::engine::ADO_AW_MODEL_AGENT_COPILOT, + "gpt-5 && curl evil.example", + )], + r#"{"schema":"ado-aw/aw_info/1"}"#, + ); + + assert!(!output.status.success(), "{output:?}"); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("invalid characters"), "{stderr}"); + } + #[test] fn explicit_model_emits_aw_info_model_metadata() { let fm = parse_fm("name: t\ndescription: x\nengine:\n id: copilot\n model: some-model\n"); diff --git a/src/engine.rs b/src/engine.rs index 1444e6487..a0d571b05 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -460,23 +460,6 @@ impl Engine { ) } - /// Generate an invocation using an explicit engine configuration. - /// - /// Retained for callers that need a plain invocation with no role-specific - /// runtime model controls. - #[allow(dead_code)] - pub fn invocation_with_config( - &self, - engine_config: &EngineConfig, - front_matter: &FrontMatter, - extension_declarations: &[Declarations], - prompt_path: &str, - mcp_config_path: Option<&str>, - ) -> Result { - let args = self.args_with_config(engine_config, front_matter, extension_declarations)?; - self.invocation_with_args(engine_config, prompt_path, mcp_config_path, &args, None) - } - /// Generate a Detection-job invocation using an explicit engine configuration. pub fn detection_invocation_with_config( &self, @@ -782,7 +765,8 @@ fn copilot_args( fn validate_model_name(model: &str) -> Result<()> { // Validate model name to prevent shell injection — copilot_params are embedded // inside a single-quoted bash string in the AWF command, and runtime-selected - // models are later passed as quoted arguments. + // models are later passed as quoted arguments. Keep this character set in + // sync with runtime_model_preamble and ado_aw_marker::EMIT_AW_INFO. if model.is_empty() || !model .chars() @@ -1476,6 +1460,7 @@ for ADO_AW_CANDIDATE_MODEL in "${{{specific}:-}}" "${{{ADO_AW_DEFAULT_MODEL_COPI continue fi case "$ADO_AW_CANDIDATE_MODEL" in + # Keep this character set in sync with validate_model_name and EMIT_AW_INFO. *[!A-Za-z0-9._:-]*) echo "ERROR: runtime Copilot model from {specific}/{ADO_AW_DEFAULT_MODEL_COPILOT} contains invalid characters. Only ASCII alphanumerics, ., _, :, and - are allowed." >&2 exit 1 @@ -1491,15 +1476,16 @@ done"# mod tests { use super::{ ADO_AW_DEFAULT_MODEL_COPILOT, ADO_AW_MODEL_AGENT_COPILOT, ADO_AW_MODEL_DETECTION_COPILOT, - Engine, GITHUB_APP_TOKEN_VAR, copilot_byom_active, copilot_byom_credential_keys, - copilot_detection_env, copilot_provider_env, get_engine, + Engine, GITHUB_APP_TOKEN_VAR, RuntimeModelRole, copilot_byom_active, + copilot_byom_credential_keys, copilot_detection_env, copilot_provider_env, get_engine, github_app_token_secrecy_advisory, github_token_source_var, normalize_version_tag, - validate_engine_feature_support, + runtime_model_preamble, validate_engine_feature_support, }; use crate::compile::{ extensions::{CompileContext, CompilerExtension, Declarations, collect_extensions}, parse_markdown, }; + use std::process::Command; fn declarations_for(fm: &crate::compile::types::FrontMatter) -> Vec { let extensions = collect_extensions(fm); @@ -1510,6 +1496,22 @@ mod tests { .collect() } + fn run_runtime_model_preamble( + role: RuntimeModelRole, + envs: &[(&str, &str)], + ) -> std::process::Output { + let script = format!( + "{}\nprintf '%s' \"$ADO_AW_EFFECTIVE_MODEL\"", + runtime_model_preamble(role) + ); + let mut command = Command::new("bash"); + command.arg("-c").arg(script).env_clear(); + for (key, value) in envs { + command.env(key, value); + } + command.output().expect("bash should run") + } + #[test] fn copilot_engine_command() { assert_eq!(Engine::Copilot.command(), "copilot"); @@ -1598,6 +1600,57 @@ mod tests { assert!(invocation.contains("--model \"$ADO_AW_EFFECTIVE_MODEL\"")); } + #[test] + fn runtime_model_preamble_uses_role_specific_before_default() { + let output = run_runtime_model_preamble( + RuntimeModelRole::Agent, + &[ + (ADO_AW_MODEL_AGENT_COPILOT, "agent-model"), + (ADO_AW_DEFAULT_MODEL_COPILOT, "default-model"), + ], + ); + + assert!(output.status.success(), "{output:?}"); + assert_eq!(String::from_utf8(output.stdout).unwrap(), "agent-model"); + } + + #[test] + fn runtime_model_preamble_uses_default_when_role_specific_missing() { + let output = run_runtime_model_preamble( + RuntimeModelRole::Detection, + &[(ADO_AW_DEFAULT_MODEL_COPILOT, "default-model")], + ); + + assert!(output.status.success(), "{output:?}"); + assert_eq!(String::from_utf8(output.stdout).unwrap(), "default-model"); + } + + #[test] + fn runtime_model_preamble_treats_unexpanded_ado_macro_as_missing() { + let output = run_runtime_model_preamble( + RuntimeModelRole::Agent, + &[ + (ADO_AW_MODEL_AGENT_COPILOT, "$(ADO_AW_MODEL_AGENT_COPILOT)"), + (ADO_AW_DEFAULT_MODEL_COPILOT, "default-model"), + ], + ); + + assert!(output.status.success(), "{output:?}"); + assert_eq!(String::from_utf8(output.stdout).unwrap(), "default-model"); + } + + #[test] + fn runtime_model_preamble_rejects_invalid_runtime_model() { + let output = run_runtime_model_preamble( + RuntimeModelRole::Detection, + &[(ADO_AW_MODEL_DETECTION_COPILOT, "gpt-5 && curl evil.example")], + ); + + assert!(!output.status.success(), "{output:?}"); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("invalid characters"), "{stderr}"); + } + #[test] fn copilot_engine_rejects_invalid_explicit_model() { let (front_matter, _) = parse_markdown( @@ -1667,6 +1720,20 @@ mod tests { assert!(!env.iter().any(|(key, _)| key == ADO_AW_MODEL_AGENT_COPILOT)); } + #[test] + fn copilot_detection_env_omits_runtime_model_vars_for_explicit_model() { + let (front_matter, _) = parse_markdown( + "---\nname: test\ndescription: test\nsafe-outputs:\n threat-detection:\n engine:\n model: detector-model\n---\n", + ) + .unwrap(); + let threat_detection = front_matter.threat_detection_config().unwrap(); + let detection_engine = front_matter.effective_detection_engine(&threat_detection); + let env = copilot_detection_env(&detection_engine).unwrap(); + + assert!(!env.iter().any(|(key, _)| key == ADO_AW_MODEL_DETECTION_COPILOT)); + assert!(!env.iter().any(|(key, _)| key == ADO_AW_DEFAULT_MODEL_COPILOT)); + } + #[test] fn copilot_engine_env_rejects_user_runtime_model_var_override() { let (front_matter, _) = parse_markdown( From 2de9d8b2f9266921c2d0acc0915c1df68d36aafb Mon Sep 17 00:00:00 2001 From: James Devine Date: Wed, 30 Sep 2026 22:56:58 +0100 Subject: [PATCH 08/24] fix(engine): resolve detection model in detection scope Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- docs/audit.md | 6 +- docs/engine.md | 4 +- src/audit/analyzers/detection.rs | 84 +++++++- src/audit/cli.rs | 84 +++++++- src/audit/model.rs | 8 +- src/compile/agentic_pipeline.rs | 251 ++++++++++++++++++++++- src/compile/extensions/ado_aw_marker.rs | 260 ++++++++++++++++-------- src/compile/extensions/mod.rs | 1 + src/engine.rs | 30 ++- 9 files changed, 618 insertions(+), 110 deletions(-) diff --git a/docs/audit.md b/docs/audit.md index e2fdf6980..e6388c560 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -61,13 +61,17 @@ URL-encoded project segments are decoded before the ADO context is resolved. `t= │ ├── mcpg/ # MCP Gateway logs (includes the SafeOutputs stdio child's stdout/stderr) │ └── agent-output.txt # Filtered agent stdout ├── analyzed_outputs[_]/ # Downloaded artifact (Detection stage) +│ ├── aw_info.json # Agent metadata + Detection runtime model │ ├── threat-analysis.json # Aggregate verdict + reasons │ └── threat-analysis-output.txt └── safe_outputs[_]/ # Downloaded artifact (SafeOutputs stage) └── safe-outputs-executed.ndjson # Per-item execution log ``` -`aw_info.json`, `otel.jsonl`, and `safe_outputs.ndjson` are searched in `staging/` first and then at the artifact top level so older layouts still audit cleanly. +Agent `aw_info.json`, `otel.jsonl`, and `safe_outputs.ndjson` are searched in +`staging/` first and then at the artifact top level so older layouts still +audit cleanly. When present, the Detection-enriched `aw_info.json` from +`analyzed_outputs` overlays only Detection-owned runtime fields in the report. ## Report shape (`AuditData`) diff --git a/docs/engine.md b/docs/engine.md index cc7434473..b94274f1b 100644 --- a/docs/engine.md +++ b/docs/engine.md @@ -60,7 +60,9 @@ still wins over runtime variables. Runtime values are passed through typed step environment mappings, validated for model-identifier characters at runtime, and then supplied to Copilot as a quoted `--model` argument. The emitted `aw_info.json` run metadata records the effective runtime-selected model when a -runtime variable is used. +runtime variable is used. Agent resolves its field in the Agent job; Detection +enriches the copied metadata in `analyzed_outputs_` from the Detection +job's own variable scope. `ado-aw audit` merges those job-owned fields. ### `timeout-minutes` diff --git a/src/audit/analyzers/detection.rs b/src/audit/analyzers/detection.rs index 49432ee02..23592cb1e 100644 --- a/src/audit/analyzers/detection.rs +++ b/src/audit/analyzers/detection.rs @@ -1,10 +1,10 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use log::{debug, warn}; use serde_json::Value; use std::io::ErrorKind; use std::path::{Path, PathBuf}; -use crate::audit::model::{DetectionAnalysis, DetectionThreats}; +use crate::audit::model::{AwInfo, DetectionAnalysis, DetectionThreats}; /// Read the detection verdict from `analyzed_outputs_/threat-analysis.json`. /// @@ -65,7 +65,36 @@ pub async fn analyze_detection(download_root: &Path) -> Result` artifact. +pub async fn load_aw_info(download_root: &Path) -> Result> { + let Some(directory) = find_analyzed_outputs_dir(download_root).await else { + return Ok(None); + }; + let path = [ + directory.join("aw_info.json"), + directory.join("staging").join("aw_info.json"), + ] + .into_iter() + .find(|path| path.is_file()); + let Some(path) = path else { + return Ok(None); + }; + let contents = tokio::fs::read_to_string(&path) + .await + .with_context(|| format!("Failed to read Detection aw_info file {}", path.display()))?; + serde_json::from_str(&contents) + .with_context(|| format!("Failed to parse Detection aw_info file {}", path.display())) + .map(Some) +} + async fn find_verdict_path(download_root: &Path) -> Option { + find_analyzed_outputs_dir(download_root) + .await + .map(|directory| directory.join("threat-analysis.json")) +} + +async fn find_analyzed_outputs_dir(download_root: &Path) -> Option { let mut entries = match tokio::fs::read_dir(download_root).await { Ok(entries) => entries, Err(err) if err.kind() == ErrorKind::NotFound => return None, @@ -122,7 +151,7 @@ async fn find_verdict_path(download_root: &Path) -> Option { } } - latest_dir.map(|(_, dir)| dir.join("threat-analysis.json")) + latest_dir.map(|(_, dir)| dir) } fn extract_bool(v: &Value, key: &str) -> bool { @@ -163,7 +192,7 @@ fn extract_reasons(v: &Value, verdict_path: &Path) -> Vec { #[cfg(test)] mod tests { - use super::analyze_detection; + use super::{analyze_detection, load_aw_info}; use crate::audit::model::DetectionThreats; use tempfile::TempDir; @@ -180,6 +209,14 @@ mod tests { .unwrap(); } + async fn write_aw_info(temp_dir: &TempDir, dir_name: &str, contents: &str) { + let dir = temp_dir.path().join(dir_name); + tokio::fs::create_dir_all(&dir).await.unwrap(); + tokio::fs::write(dir.join("aw_info.json"), contents) + .await + .unwrap(); + } + async fn write_verdict(temp_dir: &TempDir, dir_name: &str, contents: &str) { let dir = temp_dir.path().join(dir_name); tokio::fs::create_dir_all(&dir).await.unwrap(); @@ -346,4 +383,43 @@ mod tests { Some(expected_verdict_path("analyzed_outputs_10")) ); } + + #[tokio::test] + async fn loads_detection_enriched_aw_info_from_latest_artifact() { + let temp_dir = TempDir::new().unwrap(); + write_aw_info( + &temp_dir, + "analyzed_outputs_9", + r#"{"detection_model":"older"}"#, + ) + .await; + write_aw_info( + &temp_dir, + "analyzed_outputs_10", + r#"{"detection_model":"detector-model"}"#, + ) + .await; + + let aw_info = load_aw_info(temp_dir.path()).await.unwrap().unwrap(); + + assert_eq!(aw_info.detection_model.as_deref(), Some("detector-model")); + } + + #[tokio::test] + async fn detection_aw_info_is_optional_for_older_artifacts() { + let temp_dir = TempDir::new().unwrap(); + create_analyzed_outputs_dir(&temp_dir, "analyzed_outputs_42").await; + + assert!(load_aw_info(temp_dir.path()).await.unwrap().is_none()); + } + + #[tokio::test] + async fn malformed_detection_aw_info_is_an_error() { + let temp_dir = TempDir::new().unwrap(); + write_aw_info(&temp_dir, "analyzed_outputs_42", "{not valid json").await; + + let error = load_aw_info(temp_dir.path()).await.unwrap_err().to_string(); + + assert!(error.contains("Failed to parse Detection aw_info"), "{error}"); + } } diff --git a/src/audit/cli.rs b/src/audit/cli.rs index 978273654..293a80141 100644 --- a/src/audit/cli.rs +++ b/src/audit/cli.rs @@ -15,7 +15,7 @@ use crate::audit::analyzers::{ use crate::audit::cache::{RunSummary, load_run_summary, save_run_summary}; use crate::audit::find_artifact_dir; use crate::audit::findings; -use crate::audit::model::{AuditData, ErrorInfo, FileInfo, OverviewData}; +use crate::audit::model::{AuditData, AwInfo, ErrorInfo, FileInfo, OverviewData}; use crate::audit::pipeline_graph; use crate::audit::render; use crate::audit::url::{ParsedBuildRef, parse_build_ref}; @@ -458,6 +458,16 @@ async fn run_analyzers( detection::analyze_detection(run_dir).await, |a, result| a.detection_analysis = result, ); + match detection::load_aw_info(run_dir).await { + Ok(Some(detection_aw_info)) => { + merge_detection_aw_info(&mut audit.overview.aw_info, detection_aw_info) + } + Ok(None) => {} + Err(error) => { + log::warn!("{error:#}"); + crate::audit::push_warning_once(audit, crate::audit::malformed_aw_info_warning()); + } + } } run_analyzer( audit, @@ -468,6 +478,16 @@ async fn run_analyzers( ); } +fn merge_detection_aw_info(base: &mut Option, detection: AwInfo) { + let Some(base) = base.as_mut() else { + *base = Some(detection); + return; + }; + if detection.detection_model.is_some() { + base.detection_model = detection.detection_model; + } +} + fn artifact_family_selected(filters: Option<&[String]>, family: &str) -> bool { filters.is_none_or(|filters| filters.iter().any(|filter| filter == family)) } @@ -1039,7 +1059,67 @@ fn render_audit(audit: &AuditData, json: bool) -> Result<()> { #[cfg(test)] mod tests { use super::*; - use crate::audit::model::{CustomSafeOutputJobAudit, Finding, JobData, Recommendation}; + use crate::audit::model::{ + AwInfo, CustomSafeOutputJobAudit, Finding, JobData, Recommendation, + }; + + #[test] + fn detection_aw_info_overlays_only_detection_owned_fields() { + let mut base = Some(AwInfo { + engine: Some(String::from("copilot")), + model: Some(String::from("agent-model")), + detection_model: Some(String::from("old-detector")), + source: Some(String::from("agents/test.md")), + ..Default::default() + }); + let detection = AwInfo { + engine: Some(String::from("must-not-replace")), + model: Some(String::from("must-not-replace")), + detection_model: Some(String::from("detector-model")), + source: Some(String::from("must-not-replace")), + ..Default::default() + }; + + merge_detection_aw_info(&mut base, detection); + + let merged = base.unwrap(); + assert_eq!(merged.engine.as_deref(), Some("copilot")); + assert_eq!(merged.model.as_deref(), Some("agent-model")); + assert_eq!(merged.source.as_deref(), Some("agents/test.md")); + assert_eq!( + merged.detection_model.as_deref(), + Some("detector-model") + ); + } + + #[test] + fn detection_only_aw_info_populates_overview_metadata() { + let mut base = None; + let detection = AwInfo { + engine: Some(String::from("copilot")), + detection_model: Some(String::from("detector-model")), + ..Default::default() + }; + + merge_detection_aw_info(&mut base, detection.clone()); + + assert_eq!(base, Some(detection)); + } + + #[test] + fn older_detection_aw_info_does_not_erase_static_model() { + let mut base = Some(AwInfo { + detection_model: Some(String::from("static-detector")), + ..Default::default() + }); + + merge_detection_aw_info(&mut base, AwInfo::default()); + + assert_eq!( + base.unwrap().detection_model.as_deref(), + Some("static-detector") + ); + } #[tokio::test] async fn agent_output_analyzers_load_proxy_logs_from_canonical_path() { diff --git a/src/audit/model.rs b/src/audit/model.rs index 2b9e6365a..2ee1ab1e2 100644 --- a/src/audit/model.rs +++ b/src/audit/model.rs @@ -153,14 +153,16 @@ pub struct OverviewData { /// Local path where build logs or downloaded artifacts were stored. #[serde(skip_serializing_if = "Option::is_none")] pub logs_path: Option, - /// Runtime-emitted AW metadata from `staging/aw_info.json`. + /// Runtime-emitted AW metadata merged from Agent and Detection artifacts. #[serde(skip_serializing_if = "Option::is_none")] pub aw_info: Option, } /// Runtime-emitted agentic workflow metadata. /// -/// This is read from `staging/aw_info.json`, which mirrors the compiled marker metadata plus runtime context. +/// Agent metadata is read from `staging/aw_info.json`; Detection may enrich the +/// copied `aw_info.json` in its analyzed-output artifact with Detection-owned +/// runtime fields. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct AwInfo { @@ -176,7 +178,7 @@ pub struct AwInfo { /// Engine identifier used by the Detection job when explicitly configured. #[serde(skip_serializing_if = "Option::is_none")] pub detection_engine: Option, - /// Model identifier used by the Detection job when explicitly configured. + /// Effective model selected by the Detection job, when available. #[serde(skip_serializing_if = "Option::is_none")] pub detection_model: Option, /// Agent name emitted by the compiled workflow metadata. diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index 0bf517e20..a38d5d0d6 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -1657,6 +1657,9 @@ fn build_detection_job( steps.push(Step::RawYaml(step_to_raw_yaml_string(user_step)?)); } steps.push(Step::Bash(prepare_analyzed_outputs_step())); + if cfg.detection_engine_config.model().is_none() { + steps.push(Step::Bash(record_detection_runtime_model_step())); + } steps.push(Step::Bash(evaluate_threat_analysis_step())); } else { steps.push(Step::Bash(prepare_analyzed_outputs_passthrough_step())); @@ -6206,8 +6209,20 @@ shell_script! { RUN_THREAT_ANALYSIS { interpreter: Bash, bindings: [AGENT_TEMP, PIPELINE_WORKSPACE, ALLOWED_DOMAINS], - externals: [WORKING_DIRECTORY], - fragments: [image_flags, exclude_env, engine_run_detection], + externals: [ + WORKING_DIRECTORY, + ADO_AW_MODEL_DETECTION_COPILOT, + ADO_AW_DEFAULT_MODEL_COPILOT, + ADO_AW_EFFECTIVE_MODEL + ], + fragments: [capture_runtime_model, image_flags, exclude_env, engine_run_detection], + fragment_uses: [ + capture_runtime_model => [ + ADO_AW_MODEL_DETECTION_COPILOT, + ADO_AW_DEFAULT_MODEL_COPILOT, + ADO_AW_EFFECTIVE_MODEL, + ], + ], body: r###" set -o pipefail @@ -6215,6 +6230,10 @@ set -o pipefail THREAT_OUTPUT_FILE="$AGENT_TEMP/threat-analysis-output.txt" AGENT_EXIT_CODE=0 +# Capture the model selected from this task's Detection-scoped environment +# before trusted post-steps can change pipeline variables. +# ado-aw:fragment capture_runtime_model + # The argument list is assembled into an array so runtime-supplied # fragments splice in as ordinary shell statements (`AWF_ARGS+=(...)`) # — no `\`-continuation chain to break with fragment marker comments. @@ -6284,6 +6303,17 @@ fn run_threat_analysis_step( } }; let engine_run_detection_line = format!("AWF_ARGS+=(-- '{engine_run_detection}')"); + let runtime_model_enabled = detection_engine_env + .iter() + .any(|(key, _)| key == crate::engine::ADO_AW_MODEL_DETECTION_COPILOT); + let capture_runtime_model = if runtime_model_enabled { + format!( + "{}\nprintf '%s' \"$ADO_AW_EFFECTIVE_MODEL\" > \"$AGENT_TEMP/detection-runtime-model\"", + crate::engine::runtime_model_preamble(crate::engine::RuntimeModelRole::Detection) + ) + } else { + ":".to_string() + }; let mut step = ShellScript::new(&RUN_THREAT_ANALYSIS) .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) @@ -6292,6 +6322,7 @@ fn run_threat_analysis_step( Binding::ado_macro("Pipeline.Workspace"), ) .bind_text("ALLOWED_DOMAINS", allowed_domains) + .fragment("capture_runtime_model", capture_runtime_model) .fragment("image_flags", image_flags_line) .fragment("exclude_env", exclude_env_line) .fragment("engine_run_detection", engine_run_detection_line) @@ -6374,6 +6405,48 @@ fn prepare_analyzed_outputs_step() -> BashStep { .with_condition(Condition::Always) } +shell_script! { + /// Detection job: resolve the effective runtime model in Detection's own + /// variable scope and enrich the copied Agent metadata. + RECORD_DETECTION_RUNTIME_MODEL { + interpreter: Bash, + bindings: [AGENT_TEMP], + externals: [], + fragments: [append_aw_info_field], + phases: [append_aw_info_field = super::extensions::APPEND_AW_INFO_FIELD], + body: r###" +set -eo pipefail + +ADO_AW_INFO_JSON="$AGENT_TEMP/analyzed_outputs/aw_info.json" +ADO_AW_MODEL_FILE="$AGENT_TEMP/detection-runtime-model" +if [ ! -f "$ADO_AW_INFO_JSON" ]; then + echo "ERROR: Detection could not find copied aw_info.json at $ADO_AW_INFO_JSON" >&2 + exit 1 +fi +if [ ! -f "$ADO_AW_MODEL_FILE" ]; then + exit 0 +fi + +# ado-aw:fragment append_aw_info_field +ado_aw_append_info_field \ + "detection_model" \ + "$(cat "$ADO_AW_MODEL_FILE")" \ + "$ADO_AW_INFO_JSON" +"###, + } +} + +fn record_detection_runtime_model_step() -> BashStep { + ShellScript::new(&RECORD_DETECTION_RUNTIME_MODEL) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .fragment( + "append_aw_info_field", + phase_body(&super::extensions::APPEND_AW_INFO_FIELD), + ) + .into_step("Record Detection runtime model") + .with_condition(Condition::Always) +} + shell_script! { /// Detection job (AI threat detection disabled): copy Agent proposals to /// `analyzed_outputs/` unchanged. The Detection stage still runs as a @@ -6964,6 +7037,8 @@ const _SUBMODULES_OPT_BIND: Option = None; mod tests { use super::*; use crate::compile::mcpg::McpgLaunchEnvironment; + #[cfg(unix)] + use std::process::{Command, Output}; fn test_front_matter(yaml: &str) -> FrontMatter { serde_yaml::from_str(yaml).expect("front matter should parse") @@ -8217,6 +8292,8 @@ safe-outputs: let fm = parse_and_resolve(source); let threat_detection = fm.threat_detection_config().unwrap(); let detection_engine_config = fm.effective_detection_engine(&threat_detection); + let detection_engine_env = + crate::engine::copilot_detection_env(&detection_engine_config).unwrap(); let ctx = super::super::extensions::CompileContext::for_test(&fm); let extensions = super::super::extensions::collect_extensions(&fm); let decls: Vec<_> = extensions @@ -8284,7 +8361,7 @@ safe-outputs: debug_pipeline: false, byom_exclude_keys: vec![], detection_byom_exclude_keys: vec![], - detection_engine_env: vec![], + detection_engine_env, }; build_canonical_jobs( &fm, @@ -8302,6 +8379,68 @@ safe-outputs: jobs.iter().find(|j| j.id.as_ref() == id).map(|j| &j.pool) } + fn detection_runtime_model_step(jobs: &[Job]) -> Option<&BashStep> { + jobs.iter() + .find(|job| job.id.as_ref() == "Detection") + .and_then(|job| { + job.steps.iter().find_map(|step| match step { + Step::Bash(step) if step.display_name == "Record Detection runtime model" => { + Some(step) + } + _ => None, + }) + }) + } + + fn detection_run_step(jobs: &[Job]) -> Option<&BashStep> { + jobs.iter() + .find(|job| job.id.as_ref() == "Detection") + .and_then(|job| { + job.steps.iter().find_map(|step| match step { + Step::Bash(step) + if step.display_name == "Run threat analysis (AWF network isolated)" => + { + Some(step) + } + _ => None, + }) + }) + } + + #[cfg(unix)] + fn run_detection_runtime_model_script(model: Option<&str>) -> (Output, tempfile::TempDir) { + let temp = tempfile::tempdir().expect("temp dir"); + let analyzed_outputs = temp.path().join("analyzed_outputs"); + std::fs::create_dir_all(&analyzed_outputs).expect("create analyzed outputs"); + std::fs::write( + analyzed_outputs.join("aw_info.json"), + r#"{"schema":"ado-aw/aw_info/1"}"#, + ) + .expect("write aw_info.json"); + if let Some(model) = model { + std::fs::write(temp.path().join("detection-runtime-model"), model) + .expect("write runtime model"); + } + let script = ShellScript::new(&RECORD_DETECTION_RUNTIME_MODEL) + .bind_text("AGENT_TEMP", temp.path().display().to_string()) + .fragment( + "append_aw_info_field", + phase_body(&super::super::extensions::APPEND_AW_INFO_FIELD), + ) + .render(); + let mut command = Command::new("bash"); + command.arg("-c").arg(script).env_clear(); + (command.output().expect("bash should run"), temp) + } + + #[cfg(unix)] + fn read_detection_aw_info(temp: &tempfile::TempDir) -> serde_json::Value { + let contents = + std::fs::read_to_string(temp.path().join("analyzed_outputs/aw_info.json")) + .expect("read aw_info.json"); + serde_json::from_str(&contents).expect("parse aw_info.json") + } + #[test] fn threat_detection_enabled_and_disabled_match_expected_ir_graph() { use std::collections::{BTreeMap, BTreeSet}; @@ -8381,6 +8520,7 @@ safe-outputs: assert_eq!(location.job, job("Detection")); assert_eq!(&location.outputs, outputs); } + } let disabled_detection = disabled_jobs @@ -8418,6 +8558,111 @@ safe-outputs: assert!(reviewed_index < copy_logs_index); } + #[test] + fn detection_runtime_model_metadata_uses_detection_job_scope_only_when_enabled() { + let runtime = build_jobs( + "---\nname: test\ndescription: test\nsafe-outputs:\n threat-detection: true\n---\nbody\n", + ); + let disabled = build_jobs( + "---\nname: test\ndescription: test\nsafe-outputs:\n threat-detection: false\n---\nbody\n", + ); + let static_model = build_jobs( + "---\nname: test\ndescription: test\nengine:\n model: static-model\nsafe-outputs:\n threat-detection: true\n---\nbody\n", + ); + + let step = + detection_runtime_model_step(&runtime).expect("runtime Detection emits metadata step"); + assert!(matches!(step.condition, Some(Condition::Always))); + assert!(step.env.is_empty()); + assert!(!step.script.contains("ADO_AW_MODEL_DETECTION_COPILOT")); + assert!(!step.script.contains("ADO_AW_DEFAULT_MODEL_COPILOT")); + + let run_step = detection_run_step(&runtime).expect("enabled Detection runs analysis"); + assert!( + run_step + .env + .contains_key(crate::engine::ADO_AW_MODEL_DETECTION_COPILOT) + ); + assert!( + run_step + .env + .contains_key(crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT) + ); + assert!(matches!( + run_step + .env + .get(crate::engine::ADO_AW_MODEL_DETECTION_COPILOT), + Some(EnvValue::PipelineVar(name)) + if name == crate::engine::ADO_AW_MODEL_DETECTION_COPILOT + )); + assert!(matches!( + run_step + .env + .get(crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT), + Some(EnvValue::PipelineVar(name)) + if name == crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT + )); + assert!( + run_step + .script + .contains("$AGENT_TEMP/detection-runtime-model") + ); + assert!( + run_step + .script + .find("$AGENT_TEMP/detection-runtime-model") + .unwrap() + < run_step + .script + .find("\"$PIPELINE_WORKSPACE/awf/awf\"") + .unwrap(), + "runtime model must be captured before the Detection engine runs" + ); + assert!( + !run_step + .script + .contains("$(ADO_AW_MODEL_DETECTION_COPILOT)") + ); + assert!( + !run_step + .script + .contains("$(ADO_AW_DEFAULT_MODEL_COPILOT)") + ); + + assert!( + detection_runtime_model_step(&disabled).is_none(), + "disabled Detection must not resolve or validate model variables" + ); + assert!( + detection_runtime_model_step(&static_model).is_none(), + "static Detection models are already present in compile-time metadata" + ); + } + + #[test] + #[cfg(unix)] + fn detection_runtime_metadata_records_captured_model() { + let (output, temp) = run_detection_runtime_model_script(Some("detector-model")); + + assert!(output.status.success(), "{output:?}"); + assert_eq!( + read_detection_aw_info(&temp)["detection_model"], + "detector-model" + ); + } + + #[test] + #[cfg(unix)] + fn detection_runtime_metadata_omits_missing_model() { + let (output, temp) = run_detection_runtime_model_script(None); + assert!(output.status.success(), "{output:?}"); + assert!( + read_detection_aw_info(&temp) + .get("detection_model") + .is_none() + ); + } + #[test] fn pool_overrides_detection_only_flows_to_compiled_job() { let source = concat!( diff --git a/src/compile/extensions/ado_aw_marker.rs b/src/compile/extensions/ado_aw_marker.rs index b2b5e1c83..8f6cb05f5 100644 --- a/src/compile/extensions/ado_aw_marker.rs +++ b/src/compile/extensions/ado_aw_marker.rs @@ -49,45 +49,17 @@ shell_script! { } shell_script! { - /// Write `aw_info.json` to Agent.TempDirectory/staging. + /// Append one validated string field to a single-line JSON object. /// - /// The JSON is spliced as a fragment because it may (harmlessly) contain - /// substrings that `Binding::document`'s SECRET_NAMES check would reject - /// as false-positives, and the quoted heredoc delimiter here means the - /// splice is *shell data*, not shell to execute. - EMIT_AW_INFO { + /// This phase is shared by the Agent metadata writer and the Detection + /// metadata enrichment step so their file-update behavior cannot drift. + APPEND_AW_INFO_FIELD { interpreter: Bash, - bindings: [AGENT_TEMP], - externals: [ADO_AW_MODEL_AGENT_COPILOT, ADO_AW_MODEL_DETECTION_COPILOT, ADO_AW_DEFAULT_MODEL_COPILOT], - fragments: [aw_info_json], + bindings: [], + externals: [], + fragments: [], body: r#" -set -eo pipefail - -ado_aw_runtime_model() { - local specific_var="$1" - local specific_value="$2" - local candidate - for candidate in "$specific_value" "${ADO_AW_DEFAULT_MODEL_COPILOT:-}"; do - # Azure DevOps leaves an undefined macro as the literal $(VAR); treat that as unset. - if [ -z "$candidate" ] \ - || [ "$candidate" = "\$($specific_var)" ] \ - || [ "$candidate" = "\$(ADO_AW_DEFAULT_MODEL_COPILOT)" ]; then - continue - fi - case "$candidate" in - # Keep this character set in sync with engine::validate_model_name and runtime_model_preamble. - *[!A-Za-z0-9._:-]*) - echo "ERROR: runtime Copilot model from $specific_var/ADO_AW_DEFAULT_MODEL_COPILOT contains invalid characters. Only ASCII alphanumerics, ., _, :, and - are allowed." >&2 - exit 1 - ;; - esac - printf '%s' "$candidate" - return 0 - done - return 0 -} - -ado_aw_append_model_field() { +ado_aw_append_info_field() { local field="$1" local value="$2" local file="$3" @@ -113,6 +85,34 @@ ado_aw_append_model_field() { printf '%s%s"%s":"%s"}' "${json%?}" "$separator" "$field" "$value" > "$tmp" mv "$tmp" "$file" } +"#, + } +} + +shell_script! { + /// Write `aw_info.json` to Agent.TempDirectory/staging. + /// + /// The JSON is spliced as a fragment because it may (harmlessly) contain + /// substrings that `Binding::document`'s SECRET_NAMES check would reject + /// as false-positives, and the quoted heredoc delimiter here means the + /// splice is *shell data*, not shell to execute. + EMIT_AW_INFO { + interpreter: Bash, + bindings: [AGENT_TEMP], + externals: [ADO_AW_MODEL_AGENT_COPILOT, ADO_AW_DEFAULT_MODEL_COPILOT, ADO_AW_EFFECTIVE_MODEL], + fragments: [append_aw_info_field, aw_info_json, runtime_model_resolution], + phases: [append_aw_info_field = APPEND_AW_INFO_FIELD], + fragment_uses: [ + runtime_model_resolution => [ + ADO_AW_MODEL_AGENT_COPILOT, + ADO_AW_DEFAULT_MODEL_COPILOT, + ADO_AW_EFFECTIVE_MODEL, + ], + ], + body: r#" +set -eo pipefail + +# ado-aw:fragment append_aw_info_field mkdir -p "$AGENT_TEMP/staging" cat >"$AGENT_TEMP/staging/aw_info.json" <<'AW_INFO_EOF' @@ -121,21 +121,10 @@ AW_INFO_EOF ADO_AW_INFO_JSON="$AGENT_TEMP/staging/aw_info.json" if ! grep -q '"model"' "$ADO_AW_INFO_JSON"; then - ADO_AW_AGENT_RUNTIME_MODEL="$(ado_aw_runtime_model \ - ADO_AW_MODEL_AGENT_COPILOT \ - "$ADO_AW_MODEL_AGENT_COPILOT")" - ado_aw_append_model_field \ + # ado-aw:fragment runtime_model_resolution + ado_aw_append_info_field \ "model" \ - "$ADO_AW_AGENT_RUNTIME_MODEL" \ - "$ADO_AW_INFO_JSON" -fi -if ! grep -q '"detection_model"' "$ADO_AW_INFO_JSON"; then - ADO_AW_DETECTION_RUNTIME_MODEL="$(ado_aw_runtime_model \ - ADO_AW_MODEL_DETECTION_COPILOT \ - "$ADO_AW_MODEL_DETECTION_COPILOT")" - ado_aw_append_model_field \ - "detection_model" \ - "$ADO_AW_DETECTION_RUNTIME_MODEL" \ + "$ADO_AW_EFFECTIVE_MODEL" \ "$ADO_AW_INFO_JSON" fi "#, @@ -281,16 +270,20 @@ fn marker_bash_step(metadata: &CompileMetadata) -> BashStep { fn aw_info_bash_step(metadata: &CompileMetadata) -> BashStep { ShellScript::new(&EMIT_AW_INFO) .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .fragment( + "append_aw_info_field", + APPEND_AW_INFO_FIELD.body.trim().to_string(), + ) .fragment("aw_info_json", metadata.aw_info_json()) + .fragment( + "runtime_model_resolution", + crate::engine::runtime_model_preamble(crate::engine::RuntimeModelRole::Agent), + ) .into_step("Emit aw_info.json") .with_env( crate::engine::ADO_AW_MODEL_AGENT_COPILOT, EnvValue::pipeline_var(crate::engine::ADO_AW_MODEL_AGENT_COPILOT), ) - .with_env( - crate::engine::ADO_AW_MODEL_DETECTION_COPILOT, - EnvValue::pipeline_var(crate::engine::ADO_AW_MODEL_DETECTION_COPILOT), - ) .with_env( crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT, EnvValue::pipeline_var(crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT), @@ -326,23 +319,20 @@ impl CompileMetadata { .front_matter .safe_outputs .contains_key(crate::compile::types::THREAT_DETECTION_KEY); - let (threat_detection_enabled, detection_engine, detection_model) = - if explicit_threat_detection { - let config = ctx.front_matter.threat_detection_config()?; - let (engine, model) = if config.engine.is_some() { - let effective = ctx.front_matter.effective_detection_engine(&config); - let engine = crate::engine::get_engine(effective.engine_id())?; - let model = match engine { - crate::engine::Engine::Copilot => effective.model().map(str::to_string), - }; - (Some(effective.engine_id().to_string()), model) - } else { - (None, None) - }; - (Some(config.is_enabled()), engine, model) - } else { - (None, None, None) - }; + let config = ctx.front_matter.threat_detection_config()?; + let effective = ctx.front_matter.effective_detection_engine(&config); + let engine = crate::engine::get_engine(effective.engine_id())?; + let detection_model = if config.is_enabled() { + match engine { + crate::engine::Engine::Copilot => effective.model().map(str::to_string), + } + } else { + None + }; + let threat_detection_enabled = + explicit_threat_detection.then_some(config.is_enabled()); + let detection_engine = (explicit_threat_detection && config.engine.is_some()) + .then_some(effective.engine_id().to_string()); Ok(Some(Self { source: super::super::common::normalize_source_path(input_path), org: ctx @@ -544,9 +534,11 @@ fn bash_single_quote_escape(s: &str) -> String { mod tests { use super::*; use crate::compile::extensions::CompileContext; + #[cfg(unix)] use crate::compile::shell::ShellScript; use crate::compile::types::FrontMatter; use std::path::Path; + #[cfg(unix)] use std::process::{Command, Output}; fn parse_fm(yaml: &str) -> FrontMatter { @@ -567,11 +559,20 @@ mod tests { } } + #[cfg(unix)] fn run_aw_info_script(envs: &[(&str, &str)], aw_info_json: &str) -> (Output, tempfile::TempDir) { let temp = tempfile::tempdir().expect("temp dir"); let script = ShellScript::new(&EMIT_AW_INFO) .bind_text("AGENT_TEMP", temp.path().display().to_string()) + .fragment( + "append_aw_info_field", + APPEND_AW_INFO_FIELD.body.trim().to_string(), + ) .fragment("aw_info_json", aw_info_json.to_string()) + .fragment( + "runtime_model_resolution", + crate::engine::runtime_model_preamble(crate::engine::RuntimeModelRole::Agent), + ) .render(); let mut command = Command::new("bash"); command.arg("-c").arg(script).env_clear(); @@ -581,6 +582,7 @@ mod tests { (command.output().expect("bash should run"), temp) } + #[cfg(unix)] fn read_aw_info_json(temp: &tempfile::TempDir) -> serde_json::Value { let path = temp.path().join("staging/aw_info.json"); let contents = std::fs::read_to_string(path).expect("aw_info.json should be written"); @@ -738,24 +740,37 @@ mod tests { .contains_key(crate::engine::ADO_AW_MODEL_AGENT_COPILOT) ); assert!( - step.env + !step + .env .contains_key(crate::engine::ADO_AW_MODEL_DETECTION_COPILOT) ); assert!( step.env .contains_key(crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT) ); + assert!( + !step.script.contains("$(ADO_AW_MODEL_AGENT_COPILOT)"), + "runtime model macros must only appear in env mappings:\n{}", + step.script + ); + assert!( + !step.script.contains("$(ADO_AW_MODEL_DETECTION_COPILOT)"), + "runtime model macros must only appear in env mappings:\n{}", + step.script + ); + assert!( + !step.script.contains("$(ADO_AW_DEFAULT_MODEL_COPILOT)"), + "runtime model macros must only appear in env mappings:\n{}", + step.script + ); } #[test] - fn aw_info_runtime_models_prefer_role_specific_over_default() { + #[cfg(unix)] + fn aw_info_runtime_model_prefers_role_specific_over_default() { let (output, temp) = run_aw_info_script( &[ (crate::engine::ADO_AW_MODEL_AGENT_COPILOT, "agent-model"), - ( - crate::engine::ADO_AW_MODEL_DETECTION_COPILOT, - "detector-model", - ), (crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT, "default-model"), ], r#"{"schema":"ado-aw/aw_info/1"}"#, @@ -764,11 +779,12 @@ mod tests { assert!(output.status.success(), "{output:?}"); let value = read_aw_info_json(&temp); assert_eq!(value["model"], "agent-model"); - assert_eq!(value["detection_model"], "detector-model"); + assert!(value.get("detection_model").is_none()); } #[test] - fn aw_info_runtime_models_use_default_when_specific_missing_or_unexpanded() { + #[cfg(unix)] + fn aw_info_runtime_model_uses_default_when_specific_missing_or_unexpanded() { let (output, temp) = run_aw_info_script( &[ ( @@ -783,10 +799,11 @@ mod tests { assert!(output.status.success(), "{output:?}"); let value = read_aw_info_json(&temp); assert_eq!(value["model"], "default-model"); - assert_eq!(value["detection_model"], "default-model"); + assert!(value.get("detection_model").is_none()); } #[test] + #[cfg(unix)] fn aw_info_runtime_model_rejects_invalid_value() { let (output, _temp) = run_aw_info_script( &[( @@ -824,7 +841,7 @@ mod tests { } #[test] - fn explicit_threat_detection_emits_detector_metadata() { + fn disabled_threat_detection_omits_detector_model() { let fm = parse_fm( "name: t\ndescription: x\nengine:\n id: copilot\n model: agent-model\n\ safe-outputs:\n threat-detection:\n enabled: false\n engine:\n \ @@ -852,17 +869,87 @@ mod tests { "{}", step.script ); + assert!(!step.script.contains("\"detection_model\""), "{}", step.script); + } + + #[test] + fn explicit_default_threat_detection_emits_enabled_state_only() { + let fm = parse_fm("name: t\ndescription: x\nsafe-outputs:\n threat-detection: true\n"); + let input_path = Path::new("agents/foo.md"); + let ctx = CompileContext { + agent_name: &fm.name, + front_matter: &fm, + ado_context: None, + engine: crate::engine::Engine::Copilot, + compile_dir: None, + input_path: Some(input_path), + imported_prompt_body: String::new(), + }; + let steps = agent_prepare_steps(&ctx); + let step = bash_step(&steps[1]); + assert!( + step.script.contains("\"threat_detection_enabled\":true"), + "{}", + step.script + ); + assert!(!step.script.contains("\"detection_engine\"")); + assert!(!step.script.contains("ADO_AW_MODEL_DETECTION_COPILOT")); + } + + #[test] + fn inherited_detection_model_is_emitted_as_static_metadata() { + let fm = parse_fm( + "name: t\ndescription: x\nengine:\n id: copilot\n model: agent-model\n\ + safe-outputs:\n threat-detection: true\n", + ); + let input_path = Path::new("agents/foo.md"); + let ctx = CompileContext { + agent_name: &fm.name, + front_matter: &fm, + ado_context: None, + engine: crate::engine::Engine::Copilot, + compile_dir: None, + input_path: Some(input_path), + imported_prompt_body: String::new(), + }; + let steps = agent_prepare_steps(&ctx); + let step = bash_step(&steps[1]); assert!( + step.script.contains("\"detection_model\":\"agent-model\""), + "{}", step.script - .contains("\"detection_model\":\"detector-model\""), + ); + } + + #[test] + fn implicit_detection_inherits_static_agent_model_metadata() { + let fm = + parse_fm("name: t\ndescription: x\nengine:\n id: copilot\n model: agent-model\n"); + let input_path = Path::new("agents/foo.md"); + let ctx = CompileContext { + agent_name: &fm.name, + front_matter: &fm, + ado_context: None, + engine: crate::engine::Engine::Copilot, + compile_dir: None, + input_path: Some(input_path), + imported_prompt_body: String::new(), + }; + let steps = agent_prepare_steps(&ctx); + let step = bash_step(&steps[1]); + assert!( + step.script + .contains("\"detection_model\":\"agent-model\""), "{}", step.script ); } #[test] - fn explicit_default_threat_detection_emits_enabled_state_only() { - let fm = parse_fm("name: t\ndescription: x\nsafe-outputs:\n threat-detection: true\n"); + fn enabled_detection_specific_static_model_is_emitted() { + let fm = parse_fm( + "name: t\ndescription: x\nsafe-outputs:\n threat-detection:\n enabled: true\n engine:\n model: detector-model\n", + ); let input_path = Path::new("agents/foo.md"); let ctx = CompileContext { agent_name: &fm.name, @@ -876,12 +963,11 @@ mod tests { let steps = agent_prepare_steps(&ctx); let step = bash_step(&steps[1]); assert!( - step.script.contains("\"threat_detection_enabled\":true"), + step.script + .contains("\"detection_model\":\"detector-model\""), "{}", step.script ); - assert!(!step.script.contains("\"detection_engine\"")); - assert!(step.script.contains("ADO_AW_MODEL_DETECTION_COPILOT")); } #[test] diff --git a/src/compile/extensions/mod.rs b/src/compile/extensions/mod.rs index 26a7f6736..a40aca34a 100644 --- a/src/compile/extensions/mod.rs +++ b/src/compile/extensions/mod.rs @@ -686,6 +686,7 @@ pub use crate::runtimes::python::PythonExtension; pub use crate::tools::azure_devops::AzureDevOpsExtension; pub use crate::tools::cache_memory::CacheMemoryExtension; pub use ado_aw_marker::AdoAwMarkerExtension; +pub(crate) use ado_aw_marker::APPEND_AW_INFO_FIELD; pub use ado_script::AdoScriptExtension; pub use azure_cli::AzureCliExtension; pub use exec_context::{ diff --git a/src/engine.rs b/src/engine.rs index a0d571b05..54cf11aad 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -85,7 +85,7 @@ pub const ADO_AW_MODEL_DETECTION_COPILOT: &str = "ADO_AW_MODEL_DETECTION_COPILOT pub const ADO_AW_DEFAULT_MODEL_COPILOT: &str = "ADO_AW_DEFAULT_MODEL_COPILOT"; #[derive(Debug, Clone, Copy)] -enum RuntimeModelRole { +pub(crate) enum RuntimeModelRole { Agent, Detection, } @@ -1450,13 +1450,16 @@ fn copilot_invocation( ) } -fn runtime_model_preamble(role: RuntimeModelRole) -> String { +pub(crate) fn runtime_model_preamble(role: RuntimeModelRole) -> String { let specific = role.specific_var(); format!( r#"ADO_AW_EFFECTIVE_MODEL="" +ADO_AW_MACRO_PREFIX='$' +ADO_AW_UNRESOLVED_SPECIFIC_MODEL="${{ADO_AW_MACRO_PREFIX}}({specific})" +ADO_AW_UNRESOLVED_DEFAULT_MODEL="${{ADO_AW_MACRO_PREFIX}}({ADO_AW_DEFAULT_MODEL_COPILOT})" for ADO_AW_CANDIDATE_MODEL in "${{{specific}:-}}" "${{{ADO_AW_DEFAULT_MODEL_COPILOT}:-}}"; do # Azure DevOps leaves an undefined macro as the literal $(VAR); treat that as unset. - if [ -z "$ADO_AW_CANDIDATE_MODEL" ] || [ "$ADO_AW_CANDIDATE_MODEL" = "\$({specific})" ] || [ "$ADO_AW_CANDIDATE_MODEL" = "\$({ADO_AW_DEFAULT_MODEL_COPILOT})" ]; then + if [ -z "$ADO_AW_CANDIDATE_MODEL" ] || [ "$ADO_AW_CANDIDATE_MODEL" = "$ADO_AW_UNRESOLVED_SPECIFIC_MODEL" ] || [ "$ADO_AW_CANDIDATE_MODEL" = "$ADO_AW_UNRESOLVED_DEFAULT_MODEL" ]; then continue fi case "$ADO_AW_CANDIDATE_MODEL" in @@ -1476,15 +1479,17 @@ done"# mod tests { use super::{ ADO_AW_DEFAULT_MODEL_COPILOT, ADO_AW_MODEL_AGENT_COPILOT, ADO_AW_MODEL_DETECTION_COPILOT, - Engine, GITHUB_APP_TOKEN_VAR, RuntimeModelRole, copilot_byom_active, - copilot_byom_credential_keys, copilot_detection_env, copilot_provider_env, get_engine, - github_app_token_secrecy_advisory, github_token_source_var, normalize_version_tag, - runtime_model_preamble, validate_engine_feature_support, + Engine, GITHUB_APP_TOKEN_VAR, copilot_byom_active, copilot_byom_credential_keys, + copilot_detection_env, copilot_provider_env, get_engine, github_app_token_secrecy_advisory, + github_token_source_var, normalize_version_tag, validate_engine_feature_support, }; + #[cfg(unix)] + use super::{RuntimeModelRole, runtime_model_preamble}; use crate::compile::{ extensions::{CompileContext, CompilerExtension, Declarations, collect_extensions}, parse_markdown, }; + #[cfg(unix)] use std::process::Command; fn declarations_for(fm: &crate::compile::types::FrontMatter) -> Vec { @@ -1496,6 +1501,7 @@ mod tests { .collect() } + #[cfg(unix)] fn run_runtime_model_preamble( role: RuntimeModelRole, envs: &[(&str, &str)], @@ -1558,6 +1564,8 @@ mod tests { assert!(invocation.contains("ADO_AW_DEFAULT_MODEL_COPILOT")); assert!(invocation.contains("--model \"$ADO_AW_EFFECTIVE_MODEL\"")); assert!(invocation.contains("*[!A-Za-z0-9._:-]*)")); + assert!(!invocation.contains("$(ADO_AW_MODEL_AGENT_COPILOT)")); + assert!(!invocation.contains("$(ADO_AW_DEFAULT_MODEL_COPILOT)")); } #[test] @@ -1601,6 +1609,7 @@ mod tests { } #[test] + #[cfg(unix)] fn runtime_model_preamble_uses_role_specific_before_default() { let output = run_runtime_model_preamble( RuntimeModelRole::Agent, @@ -1615,6 +1624,7 @@ mod tests { } #[test] + #[cfg(unix)] fn runtime_model_preamble_uses_default_when_role_specific_missing() { let output = run_runtime_model_preamble( RuntimeModelRole::Detection, @@ -1626,6 +1636,7 @@ mod tests { } #[test] + #[cfg(unix)] fn runtime_model_preamble_treats_unexpanded_ado_macro_as_missing() { let output = run_runtime_model_preamble( RuntimeModelRole::Agent, @@ -1640,6 +1651,7 @@ mod tests { } #[test] + #[cfg(unix)] fn runtime_model_preamble_rejects_invalid_runtime_model() { let output = run_runtime_model_preamble( RuntimeModelRole::Detection, @@ -1853,7 +1865,7 @@ mod tests { Some("/tmp/mcp.json"), ) .unwrap(); - assert!(result.starts_with("/usr/local/bin/my-copilot ")); + assert_eq!(result.matches("/usr/local/bin/my-copilot ").count(), 2); assert!(!result.contains("/tmp/awf-tools/copilot")); } @@ -1868,7 +1880,7 @@ mod tests { Some("/tmp/mcp.json"), ) .unwrap(); - assert!(result.starts_with("/tmp/awf-tools/copilot ")); + assert_eq!(result.matches("/tmp/awf-tools/copilot ").count(), 2); } #[test] From 407d67cdbf5dec09a9dae47d6071e87f425718c2 Mon Sep 17 00:00:00 2001 From: James Devine Date: Thu, 1 Oct 2026 09:30:57 +0100 Subject: [PATCH 09/24] fix(engine): align runtime model selection Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- docs/audit.md | 12 +- docs/engine.md | 39 ++++-- src/audit/analyzers/otel.rs | 44 ++++++- src/audit/cli.rs | 5 +- src/audit/mod.rs | 26 ++++ src/audit/model.rs | 7 +- src/audit/render/console.rs | 46 ++++++- src/compile/agentic_pipeline.rs | 143 ++++++++++++++++++++- src/compile/extensions/ado_aw_marker.rs | 149 ++++++++-------------- src/engine.rs | 163 ++++++++++++++---------- tests/compiler_tests.rs | 49 ++++++- 11 files changed, 488 insertions(+), 195 deletions(-) diff --git a/docs/audit.md b/docs/audit.md index e6388c560..b2bd46de1 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -73,13 +73,21 @@ Agent `aw_info.json`, `otel.jsonl`, and `safe_outputs.ndjson` are searched in audit cleanly. When present, the Detection-enriched `aw_info.json` from `analyzed_outputs` overlays only Detection-owned runtime fields in the report. +`overview.aw_info.model` is the model requested for the Agent's Copilot +session. Copilot custom-agent definitions may pin a different model. When OTel +contains `gen_ai.request.model`, `engine_config.model` is the model observed +during execution; otherwise it falls back to the requested model. Console +output shows `requested_model` and `observed_model` separately when they differ. +`overview.aw_info.detection_model` is the requested Detection session model; +Detection OTel is not currently included in the analyzed artifact. + ## Report shape (`AuditData`) Current top-level keys include the following. Optional sections are omitted from `--json` when empty. | Key | Source | | --- | --- | -| `overview` | ADO build metadata + `aw_info.json` (engine, model, optional threat-detection enabled/engine/model, agent name, source, target). | +| `overview` | ADO build metadata + `aw_info.json` (engine, requested session model, optional threat-detection enabled/engine/requested model, agent name, source, target). | | `task_domain` | Audit heuristics over the run's prompts and outputs. | | `behavior_fingerprint` | Higher-level audit heuristics over the run's behavior. | | `agentic_assessments` | Higher-level audit assessments emitted by the analyzers. | @@ -87,7 +95,7 @@ Current top-level keys include the following. Optional sections are omitted from | `key_findings` | Heuristic rules + analyzer-emitted findings (for example aggregate-gate rejection). | | `recommendations` | Follow-up actions derived from findings. | | `performance_metrics` | Derived from `metrics`, runtime duration, tool usage, and firewall counts. | -| `engine_config` | Runtime engine configuration derived from `aw_info.json`. | +| `engine_config` | Runtime engine configuration; the Agent model prefers the OTel-observed model and falls back to the requested `aw_info.json` model. | | `safe_output_summary` | Counts of proposed / executed / rejected / not processed items. | | `safe_output_execution` | Per-item trace joining proposal + detection + execution. | | `rejected_safe_outputs` | Rollup of rejections by reason / threat flag. | diff --git a/docs/engine.md b/docs/engine.md index b94274f1b..c056d7b5b 100644 --- a/docs/engine.md +++ b/docs/engine.md @@ -27,8 +27,8 @@ engine: | `version` | string | *(none)* | Engine CLI version to install (e.g., `"1.0.70"`, `"latest"`). Overrides the pinned `COPILOT_CLI_VERSION`. Set to `"latest"` to use the newest available version. | | `agent` | string | *(none)* | Custom agent file identifier (Copilot only). Adds `--agent ` to the CLI invocation, selecting a custom agent from `.github/agents/`. | | `api-target` | string | *(none)* | Custom API endpoint hostname for GHES/GHEC (e.g., `"api.acme.ghe.com"`). Adds `--api-target ` to the CLI invocation and adds the hostname to the AWF network allowlist. | -| `args` | list | `[]` | Custom CLI arguments appended after compiler-generated args. Subject to shell-safety validation and blocked from overriding compiler-controlled flags (`--prompt`, `--additional-mcp-config`, `--allow-tool`, `--allow-all-tools`, `--allow-all-paths`, `--disable-builtin-mcps`, `--no-ask-user`, `--ask-user`). | -| `env` | map | *(none)* | Engine-specific environment variables merged into the sandbox step's `env:` block. Keys must be valid env var names. Values are literal-only and must not contain ADO expressions (`$(`, `${{`, `$[`) or pipeline command injection (`##vso[`), **except** the Copilot provider keys (`COPILOT_PROVIDER_BASE_URL`, `COPILOT_PROVIDER_API_KEY`, `COPILOT_PROVIDER_BEARER_TOKEN`, `COPILOT_PROVIDER_WIRE_API`), which may carry an ADO macro (`$(...)`) expression. Prefer the typed [`provider`](#copilot-model-provider-byok-configuration) block over raw provider env keys. Compiler-controlled keys (`GITHUB_TOKEN`, `PATH`, `BASH_ENV`, etc.) are blocked. | +| `args` | list | `[]` | Custom CLI arguments appended after compiler-generated args. Subject to shell-safety validation and blocked from overriding compiler-controlled flags (`--prompt`, `--model`, `--additional-mcp-config`, `--allow-tool`, `--allow-all-tools`, `--allow-all-paths`, `--disable-builtin-mcps`, `--no-ask-user`, `--ask-user`). Use `engine.model` or the runtime variables below instead of a raw `--model` argument. | +| `env` | map | *(none)* | Engine-specific environment variables merged into the sandbox step's `env:` block. Keys must be valid env var names. Values are literal-only and must not contain ADO expressions (`$(`, `${{`, `$[`) or pipeline command injection (`##vso[`), **except** the Copilot provider keys (`COPILOT_PROVIDER_BASE_URL`, `COPILOT_PROVIDER_API_KEY`, `COPILOT_PROVIDER_BEARER_TOKEN`, `COPILOT_PROVIDER_WIRE_API`), which may carry an ADO macro (`$(...)`) expression. Prefer the typed [`provider`](#copilot-model-provider-byok-configuration) block over raw provider env keys. Compiler-controlled keys (`GITHUB_TOKEN`, `COPILOT_MODEL`, `PATH`, `BASH_ENV`, etc.) are blocked. | | `provider` | map | *(none)* | Copilot external model-provider (BYOK) configuration: `base-url`, `type`, `wire-api`, `token` (compiler-minted bearer via a service connection), `api-key`. Maps to the `COPILOT_PROVIDER_*` env vars. See [Copilot model provider (BYOK) configuration](#copilot-model-provider-byok-configuration). | | `command` | string | *(none)* | Custom engine executable path (skips the default engine binary installation — NuGet for `target: 1es`, GitHub Releases for all other targets). The path must be accessible inside the AWF container (e.g., `/tmp/...` or workspace-mounted paths). | | `github-app-token` | map | *(none)* | GitHub App-backed Copilot engine authentication. When set, the compiler mints (and, by default, revokes) a GitHub App installation token in the Agent and Detection jobs and sources `GITHUB_TOKEN` from it (for Copilot only). See [GitHub App-backed Copilot engine auth](#github-app-backed-copilot-engine-auth). | @@ -52,17 +52,36 @@ Precedence is: 2. Role-specific runtime variable (`ADO_AW_MODEL_AGENT_COPILOT` or `ADO_AW_MODEL_DETECTION_COPILOT`). 3. Shared runtime variable (`ADO_AW_DEFAULT_MODEL_COPILOT`). -4. Existing default behavior (no `--model`; the Copilot CLI chooses). +4. Existing default behavior (`COPILOT_MODEL` is unset; the Copilot CLI + chooses). Detection uses its effective engine config after applying `safe-outputs.threat-detection.engine`, so an inherited or nested explicit model -still wins over runtime variables. Runtime values are passed through typed step -environment mappings, validated for model-identifier characters at runtime, and -then supplied to Copilot as a quoted `--model` argument. The emitted -`aw_info.json` run metadata records the effective runtime-selected model when a -runtime variable is used. Agent resolves its field in the Agent job; Detection -enriches the copied metadata in `analyzed_outputs_` from the Detection -job's own variable scope. `ado-aw audit` merges those job-owned fields. +still wins over runtime variables. This mirrors gh-aw and means the runtime +variables are an operational escape hatch only for workflows that leave +`engine.model` unset; changing a pinned frontmatter model still requires +recompilation. + +Runtime values are passed through typed step environment mappings and validated +for model-identifier characters at runtime. The task that starts Copilot exports +the selected value through Copilot CLI's native `COPILOT_MODEL` environment +variable. When no value resolves, the task leaves `COPILOT_MODEL` unset rather +than supplying a compiler default. Raw `engine.args --model` and +`engine.env.COPILOT_MODEL` are rejected so they cannot bypass this precedence. + +Agent resolves and records its requested session model in the actual Agent run +task, after user-authored steps. A prior trusted step can therefore use +`##vso[task.setvariable]` and both execution and metadata see the task-start +value. Detection captures its requested model in the Detection run task and +later enriches the copied metadata in `analyzed_outputs_` from its own +job scope. `ado-aw audit` merges those job-owned fields. + +When `engine.agent` selects a custom agent whose definition declares `model` or +`models`, Copilot CLI may use that agent-pinned model instead of the requested +session model. `aw_info.json` records the requested session model; when Copilot +OTel is present, `ado-aw audit` reports the observed Agent model separately. +Detection metadata is requested-model-only because the analyzed artifact does +not currently include Detection OTel. ### `timeout-minutes` diff --git a/src/audit/analyzers/otel.rs b/src/audit/analyzers/otel.rs index ae24740bc..a06de22f6 100644 --- a/src/audit/analyzers/otel.rs +++ b/src/audit/analyzers/otel.rs @@ -12,6 +12,7 @@ pub struct OtelAnalysis { pub engine_config: Option, pub performance: Option, pub aw_info: Option, + pub observed_model: Option, pub warnings: Vec, } @@ -36,6 +37,7 @@ pub async fn analyze_otel(agent_outputs_dir: &std::path::Path) -> anyhow::Result let stats = AgentStats::from_otel_file(&otel_path, "audit") .await .with_context(|| format!("Failed to analyze OTel file: {}", otel_path.display()))?; + analysis.observed_model = stats.model.clone(); let total_tokens = stats.input_tokens + stats.output_tokens; analysis.metrics = MetricsData { @@ -81,7 +83,10 @@ pub async fn analyze_otel(agent_outputs_dir: &std::path::Path) -> anyhow::Result Ok(aw_info) => { analysis.engine_config = Some(AuditEngineConfig { engine: aw_info.engine.clone().unwrap_or_default(), - model: aw_info.model.clone(), + model: analysis + .observed_model + .clone() + .or_else(|| aw_info.model.clone()), version: aw_info.compiler_version.clone(), timeout_minutes: None, }); @@ -176,6 +181,10 @@ mod tests { assert_eq!(analysis.metrics.token_usage, 33185); assert_eq!(analysis.metrics.turns, 2); + assert_eq!( + analysis.observed_model.as_deref(), + Some("claude-sonnet-4.5") + ); assert!(analysis.engine_config.is_none()); assert!(analysis.aw_info.is_none()); } @@ -198,6 +207,10 @@ mod tests { .and_then(|config| config.model.as_deref()), Some("claude-sonnet-4.5") ); + assert_eq!( + analysis.observed_model.as_deref(), + Some("claude-sonnet-4.5") + ); assert_eq!( analysis .aw_info @@ -214,6 +227,35 @@ mod tests { ); } + #[tokio::test] + async fn observed_otel_model_overrides_requested_model_in_engine_config() { + let temp_dir = TempDir::new().unwrap(); + let staging_dir = temp_dir.path().join("staging"); + write_file(&staging_dir.join("otel.jsonl"), COPILOT_OTEL_FIXTURE).await; + write_file( + &staging_dir.join("aw_info.json"), + &AW_INFO_JSON.replace("claude-sonnet-4.5", "requested-session-model"), + ) + .await; + + let analysis = analyze_otel(temp_dir.path()).await.unwrap(); + + assert_eq!( + analysis + .aw_info + .as_ref() + .and_then(|info| info.model.as_deref()), + Some("requested-session-model") + ); + assert_eq!( + analysis + .engine_config + .as_ref() + .and_then(|config| config.model.as_deref()), + Some("claude-sonnet-4.5") + ); + } + #[tokio::test] async fn malformed_aw_info_preserves_otel_metrics_and_emits_shared_warning() { let temp_dir = TempDir::new().unwrap(); diff --git a/src/audit/cli.rs b/src/audit/cli.rs index 293a80141..99eb73b60 100644 --- a/src/audit/cli.rs +++ b/src/audit/cli.rs @@ -465,7 +465,10 @@ async fn run_analyzers( Ok(None) => {} Err(error) => { log::warn!("{error:#}"); - crate::audit::push_warning_once(audit, crate::audit::malformed_aw_info_warning()); + crate::audit::push_warning_once( + audit, + crate::audit::malformed_detection_aw_info_warning(), + ); } } } diff --git a/src/audit/mod.rs b/src/audit/mod.rs index bae5429cf..646036747 100644 --- a/src/audit/mod.rs +++ b/src/audit/mod.rs @@ -27,12 +27,38 @@ pub(crate) fn malformed_aw_info_warning() -> model::ErrorInfo { } } +pub(crate) fn malformed_detection_aw_info_warning() -> model::ErrorInfo { + model::ErrorInfo { + source: String::from("audit::detection_aw_info"), + message: String::from( + "Detection's analyzed aw_info.json could not be read or parsed; Detection runtime model enrichment is unavailable", + ), + timestamp: None, + } +} + pub(crate) fn push_warning_once(audit: &mut model::AuditData, warning: model::ErrorInfo) { if !audit.warnings.contains(&warning) { audit.warnings.push(warning); } } +#[cfg(test)] +mod tests { + #[test] + fn malformed_detection_metadata_warning_is_scoped_to_detection_enrichment() { + let warning = super::malformed_detection_aw_info_warning(); + + assert_eq!(warning.source, "audit::detection_aw_info"); + assert!( + warning + .message + .contains("Detection runtime model enrichment") + ); + assert!(!warning.message.contains("pipeline graph")); + } +} + /// Compare two `_` directory names by their trailing /// integer suffix, falling back to a full lexicographic comparison /// when the suffix isn't a u64. diff --git a/src/audit/model.rs b/src/audit/model.rs index 2ee1ab1e2..0ab1466f3 100644 --- a/src/audit/model.rs +++ b/src/audit/model.rs @@ -169,7 +169,10 @@ pub struct AwInfo { /// Configured engine name for the run. #[serde(skip_serializing_if = "Option::is_none")] pub engine: Option, - /// Model identifier used by the agent runtime. + /// Model identifier requested for the Agent's Copilot session. + /// + /// A selected custom agent may pin a different model. When Copilot OTel is + /// available, `AuditData.engine_config.model` reports the observed model. #[serde(skip_serializing_if = "Option::is_none")] pub model: Option, /// Whether AI threat detection was enabled for this workflow. @@ -178,7 +181,7 @@ pub struct AwInfo { /// Engine identifier used by the Detection job when explicitly configured. #[serde(skip_serializing_if = "Option::is_none")] pub detection_engine: Option, - /// Effective model selected by the Detection job, when available. + /// Model requested for the Detection Copilot session, when available. #[serde(skip_serializing_if = "Option::is_none")] pub detection_model: Option, /// Agent name emitted by the compiled workflow metadata. diff --git a/src/audit/render/console.rs b/src/audit/render/console.rs index ecdb8661d..210d03cfe 100644 --- a/src/audit/render/console.rs +++ b/src/audit/render/console.rs @@ -87,14 +87,23 @@ fn render_overview_section( .filter(|config| !config.engine.is_empty()) .map(|config| config.engine.clone()) }); - let model = aw_info + let requested_model = aw_info .and_then(|info| info.model.as_deref()) .filter(|value| !value.is_empty()) - .map(str::to_string) - .or_else(|| engine_config.and_then(|config| config.model.clone())); + .map(str::to_string); + let observed_model = engine_config + .and_then(|config| config.model.as_deref()) + .filter(|value| !value.is_empty()) + .map(str::to_string); push_opt_owned_row(&mut rows, "engine", engine); - push_opt_owned_row(&mut rows, "model", model); + match (&requested_model, &observed_model) { + (Some(requested), Some(observed)) if requested != observed => { + push_opt_owned_row(&mut rows, "requested_model", requested_model); + push_opt_owned_row(&mut rows, "observed_model", observed_model); + } + _ => push_opt_owned_row(&mut rows, "model", observed_model.or(requested_model)), + } if let Some(enabled) = aw_info.and_then(|info| info.threat_detection_enabled) { rows.push(("threat_detection_enabled".to_string(), enabled.to_string())); } @@ -1235,6 +1244,35 @@ mod tests { assert_eq!(headings, vec!["## Overview", "## Metrics"]); } + #[test] + fn overview_distinguishes_requested_and_observed_models() { + let audit = AuditData { + overview: crate::audit::model::OverviewData { + aw_info: Some(AwInfo { + engine: Some("copilot".to_string()), + model: Some("requested-model".to_string()), + ..Default::default() + }), + ..Default::default() + }, + engine_config: Some(AuditEngineConfig { + engine: "copilot".to_string(), + model: Some("observed-model".to_string()), + ..Default::default() + }), + ..Default::default() + }; + + let out = render_console(&audit); + + assert!(out.contains("- requested_model: requested-model"), "{out}"); + assert!(out.contains("- observed_model: observed-model"), "{out}"); + assert!( + !out.contains("- model: requested-model"), + "{out}" + ); + } + #[test] fn ado_proxy_analysis_renders_lifecycle_rollups_and_recent_events() { let audit = AuditData { diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index a38d5d0d6..f13d38488 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -1391,6 +1391,7 @@ fn build_agent_job( &cfg.working_directory, &cfg.engine_run, &cfg.engine_env, + front_matter.engine.model().is_none(), &cfg.byom_exclude_keys, front_matter.supply_chain(), ado_proxy_enabled, @@ -3626,7 +3627,7 @@ shell_script! { externals: [], fragments: [tail], body: r###" -set -eo pipefail +set -o pipefail mkdir -p "$DEST" locate_one() { @@ -4464,10 +4465,33 @@ shell_script! { RUN_AGENT { interpreter: Bash, bindings: [AGENT_TEMP, PIPELINE_WORKSPACE, ALLOWED_DOMAINS], - externals: [WORKING_DIRECTORY], - fragments: [topology_attach, image_flags, exclude_env, awf_mounts, routed_engine_run], + externals: [ + WORKING_DIRECTORY, + ADO_AW_MODEL_AGENT_COPILOT, + ADO_AW_DEFAULT_MODEL_COPILOT, + ADO_AW_EFFECTIVE_MODEL, + COPILOT_MODEL + ], + fragments: [ + resolve_runtime_model, + append_aw_info_field, + topology_attach, + image_flags, + exclude_env, + awf_mounts, + routed_engine_run + ], + phases: [append_aw_info_field = super::extensions::APPEND_AW_INFO_FIELD], + fragment_uses: [ + resolve_runtime_model => [ + ADO_AW_MODEL_AGENT_COPILOT, + ADO_AW_DEFAULT_MODEL_COPILOT, + ADO_AW_EFFECTIVE_MODEL, + COPILOT_MODEL, + ], + ], body: r###" -set -o pipefail +set -eo pipefail AGENT_OUTPUT_FILE="$AGENT_TEMP/staging/logs/agent-output.txt" mkdir -p "$AGENT_TEMP/staging/logs" @@ -4476,6 +4500,23 @@ AGENT_EXIT_CODE=0 echo "=== Running AI agent with AWF network isolation ===" echo "Allowed domains: $ALLOWED_DOMAINS" +# Resolve the task-scoped runtime model after all user-authored Agent steps. +set -e +# ado-aw:fragment resolve_runtime_model + +# The marker extension always creates this file for production compile contexts. +ADO_AW_INFO_JSON="$AGENT_TEMP/staging/aw_info.json" +if [ ! -f "$ADO_AW_INFO_JSON" ]; then + echo "ERROR: Agent could not find aw_info.json at $ADO_AW_INFO_JSON" >&2 + exit 1 +fi +# ado-aw:fragment append_aw_info_field +ado_aw_append_info_field \ + "model" \ + "${COPILOT_MODEL:-}" \ + "$ADO_AW_INFO_JSON" +set +e + # AWF provides L7 domain whitelisting via a rootless Docker topology. # The named MCPG container is attached to AWF's internal network as a # trusted endpoint; the agent has no route to the host. @@ -4528,6 +4569,7 @@ fn run_agent_step( working_directory: &str, engine_run: &str, engine_env: &str, + runtime_model_enabled: bool, byom_exclude_keys: &[String], supply_chain: Option<&SupplyChainConfig>, ado_proxy_enabled: bool, @@ -4568,6 +4610,11 @@ fn run_agent_step( }; let image_flags_block = awf_image_flags(supply_chain); let exclude_env_block = awf_exclude_env_flags(byom_exclude_keys); + let resolve_runtime_model = if runtime_model_enabled { + crate::engine::runtime_model_preamble(crate::engine::RuntimeModelRole::Agent) + } else { + ":".to_string() + }; // AWF attaches externally-launched trusted containers to its internal // network by name. The flag is repeatable, which is what lets the policy @@ -4635,6 +4682,11 @@ fn run_agent_step( Binding::ado_macro("Pipeline.Workspace"), ) .bind_text("ALLOWED_DOMAINS", allowed_domains) + .fragment("resolve_runtime_model", resolve_runtime_model) + .fragment( + "append_aw_info_field", + phase_body(&super::extensions::APPEND_AW_INFO_FIELD), + ) .fragment("topology_attach", topology_attach_block) .fragment("image_flags", image_flags_line) .fragment("exclude_env", exclude_env_line) @@ -6213,7 +6265,8 @@ shell_script! { WORKING_DIRECTORY, ADO_AW_MODEL_DETECTION_COPILOT, ADO_AW_DEFAULT_MODEL_COPILOT, - ADO_AW_EFFECTIVE_MODEL + ADO_AW_EFFECTIVE_MODEL, + COPILOT_MODEL ], fragments: [capture_runtime_model, image_flags, exclude_env, engine_run_detection], fragment_uses: [ @@ -6221,6 +6274,7 @@ shell_script! { ADO_AW_MODEL_DETECTION_COPILOT, ADO_AW_DEFAULT_MODEL_COPILOT, ADO_AW_EFFECTIVE_MODEL, + COPILOT_MODEL, ], ], body: r###" @@ -6308,7 +6362,7 @@ fn run_threat_analysis_step( .any(|(key, _)| key == crate::engine::ADO_AW_MODEL_DETECTION_COPILOT); let capture_runtime_model = if runtime_model_enabled { format!( - "{}\nprintf '%s' \"$ADO_AW_EFFECTIVE_MODEL\" > \"$AGENT_TEMP/detection-runtime-model\"", + "{}\nprintf '%s' \"${{COPILOT_MODEL:-}}\" > \"$AGENT_TEMP/detection-runtime-model\"", crate::engine::runtime_model_preamble(crate::engine::RuntimeModelRole::Detection) ) } else { @@ -7738,6 +7792,7 @@ safe-outputs: "/work", "copilot -p prompt", "FOO: bar", + false, &[], None, ado_proxy_enabled, @@ -7746,6 +7801,21 @@ safe-outputs: .script } + fn runtime_agent_step_for_test() -> BashStep { + run_agent_step( + "example.com", + "\\", + "/work", + "copilot -p prompt", + "ADO_AW_MODEL_AGENT_COPILOT: $(ADO_AW_MODEL_AGENT_COPILOT)\nADO_AW_DEFAULT_MODEL_COPILOT: $(ADO_AW_DEFAULT_MODEL_COPILOT)", + true, + &[], + None, + false, + ) + .expect("run_agent_step should build") + } + #[test] fn agent_attaches_only_mcpg_when_the_policy_engine_is_disabled() { let script = agent_step_for_test(false); @@ -7781,6 +7851,61 @@ safe-outputs: ))); } + #[test] + fn agent_runtime_model_is_resolved_recorded_and_exported_in_run_task() { + let step = runtime_agent_step_for_test(); + assert!(matches!( + step.env + .get(crate::engine::ADO_AW_MODEL_AGENT_COPILOT), + Some(EnvValue::PipelineVar(name)) + if name == crate::engine::ADO_AW_MODEL_AGENT_COPILOT + )); + assert!(matches!( + step.env + .get(crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT), + Some(EnvValue::PipelineVar(name)) + if name == crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT + )); + assert!(step.script.contains("export COPILOT_MODEL=\"$ADO_AW_EFFECTIVE_MODEL\"")); + assert!(step.script.contains("\"model\"")); + assert!(step.script.contains("\"${COPILOT_MODEL:-}\"")); + let metadata_index = step + .script + .find("ado_aw_append_info_field") + .expect("metadata append"); + let awf_index = step + .script + .find("\"$PIPELINE_WORKSPACE/awf/awf\"") + .expect("AWF invocation"); + assert!(metadata_index < awf_index); + assert!(!step.script.contains("$(ADO_AW_MODEL_AGENT_COPILOT)")); + assert!(!step.script.contains("$(ADO_AW_DEFAULT_MODEL_COPILOT)")); + assert!(!step.script.contains("--model")); + } + + #[test] + fn prefixed_user_env_key_does_not_unset_static_model() { + let step = run_agent_step( + "example.com", + "\\", + "/work", + "copilot -p prompt", + "COPILOT_MODEL: static-model\nADO_AW_MODEL_AGENT_COPILOT_X: harmless", + false, + &[], + None, + false, + ) + .expect("run_agent_step should build"); + + assert!(matches!( + step.env.get(crate::engine::COPILOT_MODEL), + Some(EnvValue::Literal(value)) if value == "static-model" + )); + assert!(!step.script.contains("ADO_AW_EFFECTIVE_MODEL")); + assert!(!step.script.contains("unset COPILOT_MODEL")); + } + #[test] fn enabling_the_policy_engine_changes_only_attachment_and_no_proxy() { // Guards against the continuation-indent damage that a hand-built @@ -8607,6 +8732,12 @@ safe-outputs: .script .contains("$AGENT_TEMP/detection-runtime-model") ); + assert!( + run_step + .script + .contains("export COPILOT_MODEL=\"$ADO_AW_EFFECTIVE_MODEL\"") + ); + assert!(!run_step.script.contains("--model")); assert!( run_step .script diff --git a/src/compile/extensions/ado_aw_marker.rs b/src/compile/extensions/ado_aw_marker.rs index 8f6cb05f5..af5da1e0f 100644 --- a/src/compile/extensions/ado_aw_marker.rs +++ b/src/compile/extensions/ado_aw_marker.rs @@ -24,7 +24,6 @@ use super::{CompileContext, CompilerExtension, Declarations, ExtensionPhase}; use crate::compile::ir::condition::Condition; -use crate::compile::ir::env::EnvValue; use crate::compile::ir::step::{BashStep, Step}; use crate::compile::shell::{Binding, ShellScript}; use crate::shell_script; @@ -63,7 +62,7 @@ ado_aw_append_info_field() { local field="$1" local value="$2" local file="$3" - if [ -z "$value" ] || grep -q "\"$field\"" "$file"; then + if [ -z "$value" ] || grep -Eq "\"${field}\"[[:space:]]*:" "$file"; then return 0 fi local json @@ -99,34 +98,15 @@ shell_script! { EMIT_AW_INFO { interpreter: Bash, bindings: [AGENT_TEMP], - externals: [ADO_AW_MODEL_AGENT_COPILOT, ADO_AW_DEFAULT_MODEL_COPILOT, ADO_AW_EFFECTIVE_MODEL], - fragments: [append_aw_info_field, aw_info_json, runtime_model_resolution], - phases: [append_aw_info_field = APPEND_AW_INFO_FIELD], - fragment_uses: [ - runtime_model_resolution => [ - ADO_AW_MODEL_AGENT_COPILOT, - ADO_AW_DEFAULT_MODEL_COPILOT, - ADO_AW_EFFECTIVE_MODEL, - ], - ], + externals: [], + fragments: [aw_info_json], body: r#" set -eo pipefail -# ado-aw:fragment append_aw_info_field - mkdir -p "$AGENT_TEMP/staging" cat >"$AGENT_TEMP/staging/aw_info.json" <<'AW_INFO_EOF' # ado-aw:fragment aw_info_json AW_INFO_EOF - -ADO_AW_INFO_JSON="$AGENT_TEMP/staging/aw_info.json" -if ! grep -q '"model"' "$ADO_AW_INFO_JSON"; then - # ado-aw:fragment runtime_model_resolution - ado_aw_append_info_field \ - "model" \ - "$ADO_AW_EFFECTIVE_MODEL" \ - "$ADO_AW_INFO_JSON" -fi "#, } } @@ -270,24 +250,8 @@ fn marker_bash_step(metadata: &CompileMetadata) -> BashStep { fn aw_info_bash_step(metadata: &CompileMetadata) -> BashStep { ShellScript::new(&EMIT_AW_INFO) .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) - .fragment( - "append_aw_info_field", - APPEND_AW_INFO_FIELD.body.trim().to_string(), - ) .fragment("aw_info_json", metadata.aw_info_json()) - .fragment( - "runtime_model_resolution", - crate::engine::runtime_model_preamble(crate::engine::RuntimeModelRole::Agent), - ) .into_step("Emit aw_info.json") - .with_env( - crate::engine::ADO_AW_MODEL_AGENT_COPILOT, - EnvValue::pipeline_var(crate::engine::ADO_AW_MODEL_AGENT_COPILOT), - ) - .with_env( - crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT, - EnvValue::pipeline_var(crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT), - ) .with_condition(Condition::Always) } @@ -534,8 +498,6 @@ fn bash_single_quote_escape(s: &str) -> String { mod tests { use super::*; use crate::compile::extensions::CompileContext; - #[cfg(unix)] - use crate::compile::shell::ShellScript; use crate::compile::types::FrontMatter; use std::path::Path; #[cfg(unix)] @@ -560,31 +522,32 @@ mod tests { } #[cfg(unix)] - fn run_aw_info_script(envs: &[(&str, &str)], aw_info_json: &str) -> (Output, tempfile::TempDir) { + fn run_append_aw_info_field( + field: &str, + value: &str, + aw_info_json: &str, + ) -> (Output, tempfile::TempDir) { let temp = tempfile::tempdir().expect("temp dir"); - let script = ShellScript::new(&EMIT_AW_INFO) - .bind_text("AGENT_TEMP", temp.path().display().to_string()) - .fragment( - "append_aw_info_field", - APPEND_AW_INFO_FIELD.body.trim().to_string(), - ) - .fragment("aw_info_json", aw_info_json.to_string()) - .fragment( - "runtime_model_resolution", - crate::engine::runtime_model_preamble(crate::engine::RuntimeModelRole::Agent), - ) - .render(); + let path = temp.path().join("aw_info.json"); + std::fs::write(&path, aw_info_json).expect("write aw_info.json"); + let script = format!( + "{}\nado_aw_append_info_field \"$FIELD\" \"$VALUE\" \"$FILE\"", + APPEND_AW_INFO_FIELD.body.trim() + ); let mut command = Command::new("bash"); - command.arg("-c").arg(script).env_clear(); - for (key, value) in envs { - command.env(key, value); - } + command + .arg("-c") + .arg(script) + .env_clear() + .env("FIELD", field) + .env("VALUE", value) + .env("FILE", &path); (command.output().expect("bash should run"), temp) } #[cfg(unix)] fn read_aw_info_json(temp: &tempfile::TempDir) -> serde_json::Value { - let path = temp.path().join("staging/aw_info.json"); + let path = temp.path().join("aw_info.json"); let contents = std::fs::read_to_string(path).expect("aw_info.json should be written"); serde_json::from_str(&contents).expect("aw_info.json should parse") } @@ -735,23 +698,20 @@ mod tests { step.script ); assert!(!step.script.contains("\"threat_detection_enabled\"")); - assert!( - step.env - .contains_key(crate::engine::ADO_AW_MODEL_AGENT_COPILOT) - ); assert!( !step .env .contains_key(crate::engine::ADO_AW_MODEL_DETECTION_COPILOT) ); assert!( - step.env - .contains_key(crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT) + !step + .env + .contains_key(crate::engine::ADO_AW_MODEL_AGENT_COPILOT) ); assert!( - !step.script.contains("$(ADO_AW_MODEL_AGENT_COPILOT)"), - "runtime model macros must only appear in env mappings:\n{}", - step.script + !step + .env + .contains_key(crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT) ); assert!( !step.script.contains("$(ADO_AW_MODEL_DETECTION_COPILOT)"), @@ -767,55 +727,46 @@ mod tests { #[test] #[cfg(unix)] - fn aw_info_runtime_model_prefers_role_specific_over_default() { - let (output, temp) = run_aw_info_script( - &[ - (crate::engine::ADO_AW_MODEL_AGENT_COPILOT, "agent-model"), - (crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT, "default-model"), - ], - r#"{"schema":"ado-aw/aw_info/1"}"#, + fn append_aw_info_field_does_not_confuse_value_with_model_key() { + let (output, temp) = run_append_aw_info_field( + "model", + "gpt-5", + r#"{"agent_name":"model","schema":"ado-aw/aw_info/1"}"#, ); assert!(output.status.success(), "{output:?}"); let value = read_aw_info_json(&temp); - assert_eq!(value["model"], "agent-model"); - assert!(value.get("detection_model").is_none()); + assert_eq!(value["agent_name"], "model"); + assert_eq!(value["model"], "gpt-5"); } #[test] #[cfg(unix)] - fn aw_info_runtime_model_uses_default_when_specific_missing_or_unexpanded() { - let (output, temp) = run_aw_info_script( - &[ - ( - crate::engine::ADO_AW_MODEL_AGENT_COPILOT, - "$(ADO_AW_MODEL_AGENT_COPILOT)", - ), - (crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT, "default-model"), - ], - r#"{"schema":"ado-aw/aw_info/1"}"#, + fn append_aw_info_field_does_not_confuse_value_with_detection_model_key() { + let (output, temp) = run_append_aw_info_field( + "detection_model", + "gpt-5-mini", + r#"{"agent_name":"detection_model","schema":"ado-aw/aw_info/1"}"#, ); assert!(output.status.success(), "{output:?}"); let value = read_aw_info_json(&temp); - assert_eq!(value["model"], "default-model"); - assert!(value.get("detection_model").is_none()); + assert_eq!(value["agent_name"], "detection_model"); + assert_eq!(value["detection_model"], "gpt-5-mini"); } #[test] #[cfg(unix)] - fn aw_info_runtime_model_rejects_invalid_value() { - let (output, _temp) = run_aw_info_script( - &[( - crate::engine::ADO_AW_MODEL_AGENT_COPILOT, - "gpt-5 && curl evil.example", - )], - r#"{"schema":"ado-aw/aw_info/1"}"#, + fn append_aw_info_field_preserves_existing_key_with_whitespace() { + let (output, temp) = run_append_aw_info_field( + "model", + "replacement", + r#"{"model" : "original","schema":"ado-aw/aw_info/1"}"#, ); - assert!(!output.status.success(), "{output:?}"); - let stderr = String::from_utf8(output.stderr).unwrap(); - assert!(stderr.contains("invalid characters"), "{stderr}"); + assert!(output.status.success(), "{output:?}"); + let value = read_aw_info_json(&temp); + assert_eq!(value["model"], "original"); } #[test] diff --git a/src/engine.rs b/src/engine.rs index 54cf11aad..b101608b7 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -24,6 +24,9 @@ const BLOCKED_ARG_PREFIXES: &[&str] = &[ "--ask-user", ]; +/// Native Copilot CLI environment variable used for model selection. +pub const COPILOT_MODEL: &str = "COPILOT_MODEL"; + /// Environment variable keys that the compiler controls — users must not override these. pub const BLOCKED_ENV_KEYS: &[&str] = &[ "GITHUB_TOKEN", @@ -31,6 +34,7 @@ pub const BLOCKED_ENV_KEYS: &[&str] = &[ "COPILOT_OTEL_ENABLED", "COPILOT_OTEL_EXPORTER_TYPE", "COPILOT_OTEL_FILE_EXPORTER_PATH", + COPILOT_MODEL, "ADO_AW_MODEL_AGENT_COPILOT", "ADO_AW_MODEL_DETECTION_COPILOT", "ADO_AW_DEFAULT_MODEL_COPILOT", @@ -451,13 +455,7 @@ impl Engine { mcp_config_path: Option<&str>, ) -> Result { let args = self.args(front_matter, extension_declarations)?; - self.invocation_with_args( - &front_matter.engine, - prompt_path, - mcp_config_path, - &args, - Some(RuntimeModelRole::Agent), - ) + self.invocation_with_args(&front_matter.engine, prompt_path, mcp_config_path, &args) } /// Generate a Detection-job invocation using an explicit engine configuration. @@ -470,13 +468,7 @@ impl Engine { mcp_config_path: Option<&str>, ) -> Result { let args = self.args_with_config(engine_config, front_matter, extension_declarations)?; - self.invocation_with_args( - engine_config, - prompt_path, - mcp_config_path, - &args, - Some(RuntimeModelRole::Detection), - ) + self.invocation_with_args(engine_config, prompt_path, mcp_config_path, &args) } fn invocation_with_args( @@ -485,7 +477,6 @@ impl Engine { prompt_path: &str, mcp_config_path: Option<&str>, args: &str, - runtime_model_role: Option, ) -> Result { match self { Engine::Copilot => { @@ -507,7 +498,6 @@ impl Engine { prompt_path, mcp_config_path, args, - runtime_model_role.filter(|_| engine_config.model().is_none()), )) } } @@ -638,6 +628,13 @@ fn validate_user_arg(arg: &str) -> Result<()> { arg ); } + if arg == "--model" || arg.starts_with("--model=") { + anyhow::bail!( + "engine.args entry '{}' conflicts with compiler-controlled model selection. \ + Use engine.model or the ADO_AW_MODEL_*_COPILOT pipeline variables instead.", + arg + ); + } // Reject args that attempt to override compiler-controlled flags for blocked in BLOCKED_ARG_PREFIXES { if arg.starts_with(blocked) { @@ -693,7 +690,6 @@ fn copilot_args( if let Some(model) = engine_config.model() { validate_model_name(model)?; - params.push(format!("--model {}", model)); } if let Some(0) = engine_config.timeout_minutes() { eprintln!( @@ -752,8 +748,7 @@ fn copilot_args( } // Wire engine.args — append user-provided CLI arguments after compiler-generated args. - // User args are additive; they cannot remove compiler security flags but may override - // non-security defaults via last-wins semantics (e.g., --model). + // User args are additive and cannot override compiler-controlled flags or model selection. for arg in engine_config.args() { validate_user_arg(arg)?; params.push(arg.to_string()); @@ -763,10 +758,9 @@ fn copilot_args( } fn validate_model_name(model: &str) -> Result<()> { - // Validate model name to prevent shell injection — copilot_params are embedded - // inside a single-quoted bash string in the AWF command, and runtime-selected - // models are later passed as quoted arguments. Keep this character set in - // sync with runtime_model_preamble and ado_aw_marker::EMIT_AW_INFO. + // Validate model names before they become task environment values or + // runtime-selected COPILOT_MODEL values. Keep this character set in sync + // with runtime_model_preamble. if model.is_empty() || !model .chars() @@ -876,7 +870,10 @@ fn copilot_env(engine_config: &EngineConfig) -> Result { "COPILOT_OTEL_EXPORTER_TYPE: \"file\"".to_string(), "COPILOT_OTEL_FILE_EXPORTER_PATH: \"/tmp/awf-tools/staging/otel.jsonl\"".to_string(), ]; - if engine_config.model().is_none() { + if let Some(model) = engine_config.model() { + validate_model_name(model)?; + lines.push(format!("{COPILOT_MODEL}: \"{model}\"")); + } else { add_runtime_model_env_lines(&mut lines, RuntimeModelRole::Agent); } @@ -919,9 +916,9 @@ fn add_runtime_model_env_pairs(pairs: &mut Vec<(String, String)>, role: RuntimeM /// Used by the Detection (threat-analysis) step so the detection Copilot run /// inherits the same BYOM/BYOK provider routing (and credential isolation) as /// the main agent. Mirrors gh-aw, whose detection engine config inherits the -/// main engine's `Env` (`threat_detection_inline_engine.go`). The main model is -/// already threaded via the `--model` flag on the detection invocation, so only -/// the provider routing/credential keys are needed here. +/// main engine's `Env` (`threat_detection_inline_engine.go`). Model delivery is +/// handled separately through compiler-owned `COPILOT_MODEL`, so only provider +/// routing/credential keys are selected here. /// /// Returning raw pairs (rather than a rendered YAML string) lets the call site /// build typed `EnvValue`s directly — no render-to-YAML-then-reparse round-trip, @@ -989,7 +986,10 @@ pub fn copilot_detection_env(engine_config: &EngineConfig) -> Result, args: &str, - runtime_model_role: Option, ) -> String { - let mut common_parts = vec![ + let mut parts = vec![ command_path.to_string(), format!("--prompt=\"$(cat {prompt_path})\""), ]; if let Some(mcp_path) = mcp_config_path { - common_parts.push(format!("--additional-mcp-config @{mcp_path}")); - } - - let mut base_parts = common_parts.clone(); - if !args.is_empty() { - base_parts.push(args.to_string()); + parts.push(format!("--additional-mcp-config @{mcp_path}")); } - - let Some(role) = runtime_model_role else { - return base_parts.join(" "); - }; - - let mut model_parts = common_parts; - model_parts.push("--model \"$ADO_AW_EFFECTIVE_MODEL\"".to_string()); if !args.is_empty() { - model_parts.push(args.to_string()); + parts.push(args.to_string()); } - - format!( - "{}\nif [ -n \"$ADO_AW_EFFECTIVE_MODEL\" ]; then\n {}\nelse\n {}\nfi", - runtime_model_preamble(role), - model_parts.join(" "), - base_parts.join(" ") - ) + parts.join(" ") } pub(crate) fn runtime_model_preamble(role: RuntimeModelRole) -> String { @@ -1471,7 +1452,12 @@ for ADO_AW_CANDIDATE_MODEL in "${{{specific}:-}}" "${{{ADO_AW_DEFAULT_MODEL_COPI esac ADO_AW_EFFECTIVE_MODEL="$ADO_AW_CANDIDATE_MODEL" break -done"# +done +if [ -n "$ADO_AW_EFFECTIVE_MODEL" ]; then + export COPILOT_MODEL="$ADO_AW_EFFECTIVE_MODEL" +else + unset COPILOT_MODEL +fi"# ) } @@ -1479,9 +1465,10 @@ done"# mod tests { use super::{ ADO_AW_DEFAULT_MODEL_COPILOT, ADO_AW_MODEL_AGENT_COPILOT, ADO_AW_MODEL_DETECTION_COPILOT, - Engine, GITHUB_APP_TOKEN_VAR, copilot_byom_active, copilot_byom_credential_keys, - copilot_detection_env, copilot_provider_env, get_engine, github_app_token_secrecy_advisory, - github_token_source_var, normalize_version_tag, validate_engine_feature_support, + COPILOT_MODEL, Engine, GITHUB_APP_TOKEN_VAR, copilot_byom_active, + copilot_byom_credential_keys, copilot_detection_env, copilot_provider_env, get_engine, + github_app_token_secrecy_advisory, github_token_source_var, normalize_version_tag, + validate_engine_feature_support, }; #[cfg(unix)] use super::{RuntimeModelRole, runtime_model_preamble}; @@ -1544,11 +1531,13 @@ mod tests { let params = Engine::Copilot .args(&front_matter, &declarations_for(&front_matter)) .unwrap(); - assert!(params.contains("--model gpt-5")); + assert!(!params.contains("--model")); + let env = Engine::Copilot.env(&front_matter.engine).unwrap(); + assert!(env.contains("COPILOT_MODEL: \"gpt-5\""), "{env}"); } #[test] - fn copilot_invocation_uses_runtime_agent_model_precedence_when_no_explicit_model() { + fn copilot_invocation_does_not_embed_runtime_model_resolution() { let (front_matter, _) = parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); let invocation = Engine::Copilot @@ -1560,10 +1549,9 @@ mod tests { ) .unwrap(); - assert!(invocation.contains("ADO_AW_MODEL_AGENT_COPILOT")); - assert!(invocation.contains("ADO_AW_DEFAULT_MODEL_COPILOT")); - assert!(invocation.contains("--model \"$ADO_AW_EFFECTIVE_MODEL\"")); - assert!(invocation.contains("*[!A-Za-z0-9._:-]*)")); + assert!(!invocation.contains("ADO_AW_MODEL_AGENT_COPILOT")); + assert!(!invocation.contains("ADO_AW_DEFAULT_MODEL_COPILOT")); + assert!(!invocation.contains("--model")); assert!(!invocation.contains("$(ADO_AW_MODEL_AGENT_COPILOT)")); assert!(!invocation.contains("$(ADO_AW_DEFAULT_MODEL_COPILOT)")); } @@ -1583,7 +1571,7 @@ mod tests { ) .unwrap(); - assert!(invocation.contains("--model gpt-5")); + assert!(!invocation.contains("--model")); assert!(!invocation.contains("ADO_AW_MODEL_AGENT_COPILOT")); assert!(!invocation.contains("ADO_AW_EFFECTIVE_MODEL")); } @@ -1602,10 +1590,19 @@ mod tests { ) .unwrap(); - assert!(invocation.contains("ADO_AW_MODEL_DETECTION_COPILOT")); - assert!(invocation.contains("ADO_AW_DEFAULT_MODEL_COPILOT")); + assert!(!invocation.contains("ADO_AW_MODEL_DETECTION_COPILOT")); + assert!(!invocation.contains("ADO_AW_DEFAULT_MODEL_COPILOT")); assert!(!invocation.contains("ADO_AW_MODEL_AGENT_COPILOT")); - assert!(invocation.contains("--model \"$ADO_AW_EFFECTIVE_MODEL\"")); + assert!(!invocation.contains("--model")); + let env = copilot_detection_env(&front_matter.engine).unwrap(); + assert!( + env.iter() + .any(|(key, _)| key == ADO_AW_MODEL_DETECTION_COPILOT) + ); + assert!( + env.iter() + .any(|(key, _)| key == ADO_AW_DEFAULT_MODEL_COPILOT) + ); } #[test] @@ -1663,6 +1660,38 @@ mod tests { assert!(stderr.contains("invalid characters"), "{stderr}"); } + #[test] + fn engine_args_reject_model_flag() { + for args in ["[--model, gpt-5]", "[--model=gpt-5]"] { + let source = format!( + "---\nname: test\ndescription: test\nengine:\n id: copilot\n args: {args}\n---\n" + ); + let (front_matter, _) = parse_markdown(&source).unwrap(); + let error = Engine::Copilot + .args(&front_matter, &declarations_for(&front_matter)) + .unwrap_err() + .to_string(); + assert!( + error.contains("compiler-controlled model selection"), + "{error}" + ); + } + } + + #[test] + fn engine_env_rejects_raw_copilot_model() { + let (front_matter, _) = parse_markdown( + "---\nname: test\ndescription: test\nengine:\n id: copilot\n env:\n COPILOT_MODEL: gpt-5\n---\n", + ) + .unwrap(); + let error = Engine::Copilot + .env(&front_matter.engine) + .unwrap_err() + .to_string(); + assert!(error.contains(COPILOT_MODEL), "{error}"); + assert!(error.contains("compiler-controlled"), "{error}"); + } + #[test] fn copilot_engine_rejects_invalid_explicit_model() { let (front_matter, _) = parse_markdown( @@ -1865,7 +1894,7 @@ mod tests { Some("/tmp/mcp.json"), ) .unwrap(); - assert_eq!(result.matches("/usr/local/bin/my-copilot ").count(), 2); + assert_eq!(result.matches("/usr/local/bin/my-copilot ").count(), 1); assert!(!result.contains("/tmp/awf-tools/copilot")); } @@ -1880,7 +1909,7 @@ mod tests { Some("/tmp/mcp.json"), ) .unwrap(); - assert_eq!(result.matches("/tmp/awf-tools/copilot ").count(), 2); + assert_eq!(result.matches("/tmp/awf-tools/copilot ").count(), 1); } #[test] diff --git a/tests/compiler_tests.rs b/tests/compiler_tests.rs index 179337dd7..2e2db3a15 100644 --- a/tests/compiler_tests.rs +++ b/tests/compiler_tests.rs @@ -8175,12 +8175,17 @@ safe-outputs: let agent = job_block(&compiled, "Agent"); let detection = job_block(&compiled, "Detection"); - assert!(agent.contains("--model agent-model"), "{agent}"); + assert!(agent.contains("COPILOT_MODEL: agent-model"), "{agent}"); assert!(agent.contains("--reasoning-effort=high"), "{agent}"); - assert!(!agent.contains("--model detection-model"), "{agent}"); + assert!(!agent.contains("--model"), "{agent}"); + assert!(!agent.contains("COPILOT_MODEL: detection-model"), "{agent}"); assert!(!agent.contains("DETECTION_ENV"), "{agent}"); - assert!(detection.contains("--model detection-model"), "{detection}"); + assert!( + detection.contains("COPILOT_MODEL: detection-model"), + "{detection}" + ); + assert!(!detection.contains("--model"), "{detection}"); assert!(detection.contains("--reasoning-effort=low"), "{detection}"); assert!( !detection.contains("--reasoning-effort=high"), @@ -8204,6 +8209,44 @@ safe-outputs: assert!(detection.contains("2.0.2"), "{detection}"); } +#[test] +fn runtime_model_controls_compile_across_all_targets() { + for target in ["standalone", "1es", "job", "stage"] { + let target_field = if target == "standalone" { + String::new() + } else { + format!("target: {target}\n") + }; + let source = format!( + "---\nname: Runtime Model {target}\ndescription: Runtime model target coverage\n\ + {target_field}safe-outputs:\n noop: {{}}\n threat-detection: true\n---\n\n## Agent\n" + ); + let (ok, compiled, stderr) = + compile_inline_source(&format!("runtime-model-{target}"), &source); + assert!(ok, "{target} should compile: {stderr}"); + assert!( + compiled.contains( + "ADO_AW_MODEL_AGENT_COPILOT: $(ADO_AW_MODEL_AGENT_COPILOT)" + ), + "{target}: missing Agent runtime model env mapping" + ); + assert!( + compiled.contains( + "ADO_AW_MODEL_DETECTION_COPILOT: $(ADO_AW_MODEL_DETECTION_COPILOT)" + ), + "{target}: missing Detection runtime model env mapping" + ); + assert!( + compiled.contains("export COPILOT_MODEL=\"$ADO_AW_EFFECTIVE_MODEL\""), + "{target}: runtime resolver must export COPILOT_MODEL" + ); + assert!( + !compiled.contains("--model"), + "{target}: compiler-generated model flags must be absent" + ); + } +} + #[test] fn threat_detection_disabled_preserves_outputs_artifacts_and_manual_review() { let source = r#"--- From 0d2007aba5e9ffd62658026bdce209737edf61fc Mon Sep 17 00:00:00 2001 From: James Devine Date: Thu, 1 Oct 2026 09:47:20 +0100 Subject: [PATCH 10/24] fix(compile): tolerate missing agent metadata Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- src/compile/agentic_pipeline.rs | 37 ++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index f13d38488..05fffae96 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -4506,15 +4506,15 @@ set -e # The marker extension always creates this file for production compile contexts. ADO_AW_INFO_JSON="$AGENT_TEMP/staging/aw_info.json" -if [ ! -f "$ADO_AW_INFO_JSON" ]; then - echo "ERROR: Agent could not find aw_info.json at $ADO_AW_INFO_JSON" >&2 - exit 1 +if [ -f "$ADO_AW_INFO_JSON" ]; then + # ado-aw:fragment append_aw_info_field + ado_aw_append_info_field \ + "model" \ + "${COPILOT_MODEL:-}" \ + "$ADO_AW_INFO_JSON" +else + echo "Warning: Agent metadata file not found at $ADO_AW_INFO_JSON; model metadata was not recorded" >&2 fi -# ado-aw:fragment append_aw_info_field -ado_aw_append_info_field \ - "model" \ - "${COPILOT_MODEL:-}" \ - "$ADO_AW_INFO_JSON" set +e # AWF provides L7 domain whitelisting via a rootless Docker topology. @@ -7883,6 +7883,27 @@ safe-outputs: assert!(!step.script.contains("--model")); } + #[test] + fn missing_agent_metadata_does_not_block_agent_execution() { + let step = runtime_agent_step_for_test(); + assert!(step + .script + .contains("if [ -f \"$ADO_AW_INFO_JSON\" ]; then")); + assert!(step.script.contains( + "Warning: Agent metadata file not found at $ADO_AW_INFO_JSON; model metadata was not recorded" + )); + assert!(!step + .script + .contains("ERROR: Agent could not find aw_info.json")); + + let warning_index = step.script.find("model metadata was not recorded").unwrap(); + let awf_index = step + .script + .find("\"$PIPELINE_WORKSPACE/awf/awf\"") + .expect("AWF invocation"); + assert!(warning_index < awf_index); + } + #[test] fn prefixed_user_env_key_does_not_unset_static_model() { let step = run_agent_step( From 4cc781dc8a0898ea61f137a6382ac9e4547665f2 Mon Sep 17 00:00:00 2001 From: James Devine Date: Thu, 1 Oct 2026 10:32:05 +0100 Subject: [PATCH 11/24] fix(workflows): inherit parent models for sub-agents Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- .github/workflows/pr-sous-chef.lock.yml | 2 +- .github/workflows/pr-sous-chef.md | 1 - .github/workflows/review-rust.lock.yml | 2 +- .github/workflows/review-rust.md | 1 - .github/workflows/review-typescript.lock.yml | 2 +- .github/workflows/review-typescript.md | 1 - 6 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-sous-chef.lock.yml b/.github/workflows/pr-sous-chef.lock.yml index f1ce12c41..6f325cf18 100644 --- a/.github/workflows/pr-sous-chef.lock.yml +++ b/.github/workflows/pr-sous-chef.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"d4e6e641206e227ad9c8c71f06a4f8d75b46814949bf0464d0bc3fb84cfc2706","body_hash":"0b760609f27236f11ded4123610e522e8a73558533e14609a69bf0d176abdd4f","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"d4e6e641206e227ad9c8c71f06a4f8d75b46814949bf0464d0bc3fb84cfc2706","body_hash":"c58344b90694d21c8e1521cfba92d39375d8b8877e0827d4c9dff5a150bc2613","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}]} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/pr-sous-chef.md b/.github/workflows/pr-sous-chef.md index 333c6595e..4a39ccd61 100644 --- a/.github/workflows/pr-sous-chef.md +++ b/.github/workflows/pr-sous-chef.md @@ -368,7 +368,6 @@ recommendations visible; wrap verbose detail in ## agent: `pr-processor` --- description: Decides skip/nudge actions for a single pull request using a minimal number of API calls -model: small --- You are given one PR number and its compact metadata. Decide what should happen to it, using as few tool calls as possible. diff --git a/.github/workflows/review-rust.lock.yml b/.github/workflows/review-rust.lock.yml index 1feaf1ebd..c507177fc 100644 --- a/.github/workflows/review-rust.lock.yml +++ b/.github/workflows/review-rust.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ea67889e811c8c0a7ac2e9a1b512ef32f072734d71026f82eae0a8489198f59f","body_hash":"c6aacd86f655324be0dfa10d467221cdb5bf4e14431f715b874744d2ee0304c5","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ea67889e811c8c0a7ac2e9a1b512ef32f072734d71026f82eae0a8489198f59f","body_hash":"5b9b49749ef63834afcced2cd715df56c5717bc69179daca0c867cc6941e0f84","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}],"has_pull_request":true} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/review-rust.md b/.github/workflows/review-rust.md index 136182b1b..2da67b265 100644 --- a/.github/workflows/review-rust.md +++ b/.github/workflows/review-rust.md @@ -169,7 +169,6 @@ and the themes in a `
` block. ## agent: `rust-critic` --- description: Hostile first-pass Rust reviewer that mines merge-blocking defects from changed lines -model: small --- You are a hostile senior Rust reviewer performing a first-pass audit. diff --git a/.github/workflows/review-typescript.lock.yml b/.github/workflows/review-typescript.lock.yml index 5af4a1983..24752ad10 100644 --- a/.github/workflows/review-typescript.lock.yml +++ b/.github/workflows/review-typescript.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"7259abb8515a4bb6dd1d124c1db92aa00b5e5357d4b61b78ae045ba045d425ac","body_hash":"684f375762c2d2dfb48458e8efe8760e29325adddb9edfdd9b22990ac7f3b8a7","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"7259abb8515a4bb6dd1d124c1db92aa00b5e5357d4b61b78ae045ba045d425ac","body_hash":"e370274e6f63c5c3accfa7545ad59e1499267f58360fe90346f2a5f5c58c12bd","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}],"has_pull_request":true} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/review-typescript.md b/.github/workflows/review-typescript.md index 4b99d2d16..a19db8f12 100644 --- a/.github/workflows/review-typescript.md +++ b/.github/workflows/review-typescript.md @@ -171,7 +171,6 @@ wrong output; otherwise `COMMENT`. ## agent: `ts-critic` --- description: Hostile first-pass TypeScript reviewer for bundled Azure DevOps runtime helpers -model: small --- You are a hostile senior TypeScript reviewer performing a first-pass audit of code that is bundled and executed on Azure DevOps build agents. From 31a3bdb45f9f2efe3f834114b6fe57698dc7ec76 Mon Sep 17 00:00:00 2001 From: James Devine Date: Thu, 1 Oct 2026 12:47:58 +0100 Subject: [PATCH 12/24] refactor(engine): invoke copilot through bundled harness Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- .github/workflows/copilot-cli-safeoutputs.yml | 8 + AGENTS.md | 3 +- docs/ado-script.md | 98 +++-- docs/engine.md | 25 +- scripts/ado-script/.gitignore | 1 + scripts/ado-script/package.json | 7 +- .../src/copilot-invoker/index.test.ts | 269 ++++++++++++ .../ado-script/src/copilot-invoker/index.ts | 370 ++++++++++++++++ .../test/azure-wif-isolation.test.ts | 3 + scripts/ado-script/test/smoke.test.ts | 57 +++ src/compile/ado_bundle.rs | 7 + src/compile/agentic_pipeline.rs | 338 +++++++------- src/compile/extensions/ado_script.rs | 193 +------- src/compile/extensions/exec_context/mod.rs | 112 ----- src/compile/extensions/exec_context/pr.rs | 7 - src/compile/extensions/mod.rs | 60 +-- src/compile/types.rs | 18 +- src/engine.rs | 414 ++++++++---------- src/validate.rs | 26 +- tests/awf-copilot-safeoutputs/run.sh | 42 +- tests/compiler_tests.rs | 183 ++++---- 21 files changed, 1344 insertions(+), 897 deletions(-) create mode 100644 scripts/ado-script/src/copilot-invoker/index.test.ts create mode 100644 scripts/ado-script/src/copilot-invoker/index.ts diff --git a/.github/workflows/copilot-cli-safeoutputs.yml b/.github/workflows/copilot-cli-safeoutputs.yml index eff862f20..71967b4b3 100644 --- a/.github/workflows/copilot-cli-safeoutputs.yml +++ b/.github/workflows/copilot-cli-safeoutputs.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - "src/**" + - "scripts/ado-script/**" - "tests/**" - "Cargo.toml" - "Cargo.lock" @@ -62,6 +63,12 @@ jobs: mcpg:MCPG_VERSION:src/compile/common.rs VERSIONS + - name: Build Copilot invoker bundle + run: | + set -euo pipefail + npm --prefix scripts/ado-script ci + npm --prefix scripts/ado-script run build:copilot-invoker + - name: Install compiler-pinned GitHub Copilot CLI run: | set -euo pipefail @@ -111,6 +118,7 @@ jobs: ADO_AW_BIN: ${{ github.workspace }}/target/debug/ado-aw AWF_BIN: ${{ runner.temp }}/bin/awf COPILOT_BIN: ${{ runner.temp }}/bin/copilot + COPILOT_INVOKER_BUNDLE: ${{ github.workspace }}/scripts/ado-script/copilot-invoker.js AWF_VERSION: ${{ steps.versions.outputs.awf }} MCPG_VERSION: ${{ steps.versions.outputs.mcpg }} run: bash tests/awf-copilot-safeoutputs/run.sh diff --git a/AGENTS.md b/AGENTS.md index 2cd957f00..ed63bfe48 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -319,6 +319,7 @@ fail-closed and only pauses when the agent actually proposed a reviewed output. │ ├── conclusion/ # Conclusion-job reporter source (bundled to conclusion.js) │ ├── approval-summary/ # Safe-outputs summary renderer (bundled to approval-summary.js; end-of-Agent-job summary tab) │ ├── github-app-token/ # GitHub App token minter (bundled to github-app-token.js; mints installation token in Agent + Detection when engine.github-app-token is set) +│ ├── copilot-invoker/ # Sandboxed Copilot process harness (bundled to copilot-invoker.js): strict versioned invocation/result documents, runtime model resolution, typed argv, signal forwarding, exact exit propagation │ ├── executor-e2e/ # Stage 3 safe-output E2E test harness (not a bundle; runs deterministic scenarios against a real ADO project and files a GitHub issue on failure) │ ├── compiler-smoke-e2e/ # Smoke E2E orchestrator (not a bundle): stages each case in `tests/smoke/cases.json` to the fixed `.smoke/pipeline.yml` path on its own per-case `ado-aw-mirror` ref, queues it against its credential *lane* definition, and asserts they go green. Two modes via `SMOKE_COMPILER_SOURCE`: `candidate` (compiler built from this commit, pinned pipeline-artifact) and `released` (latest release asset, release URLs required). Built to `test-bin/` by `build:compiler-smoke-e2e`, listed in `NON_BUNDLE_DIRS`. │ ├── prepare-pr-base/ # create-pull-request preparer (bundled to prepare-pr-base.js): Agent mode uses ADO diff metadata + bounded fallback; SafeOutputs fetches the target tip; cross-org targets use isolated credentials + exact remote matching @@ -463,7 +464,7 @@ index to jump to the right page. (`gate.js`, `import.js`, the execution-context `exec-context-*.js` bundles, `conclusion.js`, `approval-summary.js`, `github-app-token.js`, `prepare-pr-base.js`, and - `azure-wif-refresh.js`), schemars-driven + `azure-wif-refresh.js`, `copilot-invoker.js`), schemars-driven type codegen, the A2 design decision, the bundle env contract modelled in `src/compile/ado_bundle.rs`, and the `trigger-e2e/` gate-spec drift guard (kept in sync via `export-fact-catalog`). diff --git a/docs/ado-script.md b/docs/ado-script.md index fa1c726e5..9c932e97a 100644 --- a/docs/ado-script.md +++ b/docs/ado-script.md @@ -98,6 +98,13 @@ pipeline** as runtime helpers. Today it produces the following shipped bundles: never enter the agent, MCP environment, Docker arguments, logs, status documents, or artifacts. See [`mcp.md`](mcp.md#renewable-azure-workload-identity). +- `copilot-invoker.js` — the sandboxed Copilot process harness used by every + Agent job and every enabled Detection job. It validates a versioned, + compiler-emitted invocation document, resolves role-specific runtime model + variables, constructs Copilot argv without a shell, forwards termination + signals, publishes the requested-model result, and preserves Copilot's exit + status. It does not own AWF topology, credentials, retries, or OTel + interpretation. > **Internal-only.** `ado-script` is not a user-facing front-matter > feature. Authors never write an `ado-script:` block in their agent @@ -666,6 +673,9 @@ scripts/ado-script/ │ ├── azure-wif-refresh/ # azure-wif-refresh.js entry point + renewable WIF assertion sidecar │ │ ├── index.ts # main(): rotate a private token file for user-defined stdio MCP servers │ │ └── __tests__/ # unit tests for rotation and isolation behaviour +│ ├── copilot-invoker/ # copilot-invoker.js entry point + versioned Copilot process harness +│ │ ├── index.ts # document/result validation, model resolution, argv, signals, exact exit +│ │ └── index.test.ts # schema, precedence, argv, environment, result, and process tests │ ├── trigger-e2e/ # test-only: FACT_META gate-spec table + trigger-evaluation E2E scenarios (not a bundle) │ │ ├── gate-spec.ts # FACT_META mirror of Rust Fact::ALL; drift-guarded by export-fact-catalog + fact-catalog.gen.json │ │ ├── fact-catalog.gen.json # generated by `cargo run -- export-fact-catalog`; deep-compared by gate-spec.test.ts @@ -689,7 +699,8 @@ scripts/ado-script/ ├── github-app-token.js # ncc bundle output (gitignored) ├── prepare-pr-base.js # ncc bundle output (gitignored) ├── ado-proxy.js # ncc bundle output (gitignored) -└── azure-wif-refresh.js # ncc bundle output (gitignored) +├── azure-wif-refresh.js # ncc bundle output (gitignored) +└── copilot-invoker.js # ncc bundle output (gitignored) ``` The release workflow (`.github/workflows/release.yml`) runs @@ -700,8 +711,8 @@ captures every bundle, including `gate.js`, `import.js`, `exec-context-ci-push.js`, `exec-context-workitem.js`, `exec-context-schedule.js`, `exec-context-pr-checks.js`, `exec-context-repo.js`, `conclusion.js`, `approval-summary.js`, -`github-app-token.js`, `prepare-pr-base.js`, `ado-proxy.js`, and -`azure-wif-refresh.js` — into the +`github-app-token.js`, `prepare-pr-base.js`, `ado-proxy.js`, +`azure-wif-refresh.js`, and `copilot-invoker.js` — into the `ado-script.zip` release asset. Pipelines download that asset at runtime by URL pinned to the compiler's `CARGO_PKG_VERSION`, verify its SHA-256 against the `checksums.txt` asset, then extract. @@ -742,9 +753,8 @@ cargo run -- export-gate-schema --output schema/gate-spec.schema.json `AdoScriptExtension` (`src/compile/extensions/ado_script.rs`) is the always-on single -extension that owns all `ado-script` wiring. It has two independent -features, each emitted **into the job that actually consumes the -bundle**: +extension that owns shared `ado-script` delivery. Each download is emitted +**into the job that actually consumes the bundle**: ### Setup job (gate evaluator) @@ -762,12 +772,11 @@ returns three typed `Declarations::setup_steps` entries for the Setup job: runs the gate with `GATE_SPEC` and the env-var contract documented above. -### Agent job (runtime-import resolver + PR-context precompute) +### Agent job (Copilot invoker + optional helpers) -When `inlined-imports: false` (the default) OR the execution-context -PR contributor activates (`on.pr` configured and not disabled), -`AdoScriptExtension::declarations()` returns the install + download pair in -`Declarations::agent_prepare_steps` for the Agent job: +Every Agent job receives the install + download pair in +`Declarations::agent_prepare_steps`, because every Copilot run uses +`copilot-invoker.js`: 1. **`UseNode@1`** — same shape as above. 2. **`curl` download + verify + extract** — same artefact, same @@ -777,6 +786,11 @@ PR contributor activates (`on.pr` configured and not disabled), `/tmp/awf-tools/agent-prompt.md` in place. See [`runtime-imports.md`](runtime-imports.md) for marker syntax. **Only emitted when `inlined-imports: false`.** +4. The AWF run executes the fixed command + `node /tmp/ado-aw-scripts/ado-script/copilot-invoker.js run + /tmp/awf-tools/copilot-invocation.json`. Author values, prompt contents, and + Azure DevOps model macros are data in the invocation document or typed + environment mappings, never shell fragments. The PR-context precompute step (`node exec-context-pr.js`) is owned by `ExecContextExtension` (not `AdoScriptExtension`) and emitted through @@ -785,27 +799,53 @@ its own Tool-phase `Declarations::agent_prepare_steps`. Phase ordering guarantees the bundle is installed and on disk before the exec-context invocation runs. +### Detection job + +Every enabled Detection job installs Node and stages `ado-script.zip`, then +uses the same fixed invoker command with a Detection-role document. Detection +does not receive the Agent MCP configuration. Workflows that explicitly disable +threat detection emit no Detection download, document, model mappings, or +invoker command. + +### Copilot invocation and result protocol + +The compiler writes schema version 1 JSON under +`/tmp/awf-tools/copilot-invocation.json`. The document contains only +compiler-validated, non-secret data: role, command, prompt path, optional MCP +config path, argv entries, optional explicit model, and result path. The +invoker rejects unknown fields and unsupported versions, reads the prompt as +UTF-8, spawns Copilot without a shell, and sets or removes `COPILOT_MODEL` only +in the child environment. + +Before spawning Copilot, the invoker atomically writes +`/tmp/awf-tools/copilot-invocation-result.json` with the role and requested +model. Trusted host code validates that document with the invoker's +`read-result` mode and merges it into `aw_info.json`. The invoker never edits +trusted metadata and never receives Stage 3 credentials. + ### Per-job download (NOT a duplication bug) ADO jobs use **isolated VMs** — `/tmp` is not shared between jobs. The `ado-script.zip` bundle therefore has to be downloaded once per -job that consumes it. When both Setup and Agent need it, install + -download steps appear in **both**. That's correct architecture given -ADO's topology, not waste. +job that consumes it. Agent and enabled Detection therefore always stage the +bundle; Setup stages another copy when a gate or synthetic-PR resolver needs +one. That's correct architecture given ADO's topology, not waste. ### What gets emitted, by case The rows below assume the synthetic-PR resolver is **not** active (`pr_trigger_for_synth = None`): -| Setup consumer | Agent consumer | Setup-job steps | Agent-job extra steps | +| Setup consumer | Agent optional consumers | Setup-job steps | Agent-job steps | |---|---|---|---| -| no gate | none | (none) | (none) | -| no gate | `inlined-imports: false` only | (no Setup job) | install + download + resolver | -| no gate | execution-context contributor(s) only | (no Setup job) | install + download + exec-context bundle(s) | -| no gate | resolver + execution-context | (no Setup job) | install + download + resolver + exec-context bundle(s) | -| gate | none | install + download + gate | (none) | -| gate | any combination of resolver / exec-context | install + download + gate | install + download + (resolver?) + (exec-context bundle(s)?) | +| no gate | none | (none) | install + download + invoker | +| no gate | runtime imports | (none) | install + download + resolver + invoker | +| no gate | execution-context contributor(s) | (none) | install + download + exec-context bundle(s) + invoker | +| gate | none | install + download + gate | install + download + invoker | +| gate | resolver / exec-context | install + download + gate | install + download + optional helpers + invoker | + +Every row also has an enabled Detection job with its own install + download + +invoker unless threat detection is explicitly disabled. When the synthetic-PR resolver **is** active (`pr_trigger_for_synth = Some(_)`, i.e. `synthetic_pr_active()` is @@ -813,18 +853,16 @@ true) the Setup job gains the `synthPr` step (`node exec-context-pr-synth.js`) before any gate step — and the Setup job is emitted even with no gate: -| Setup consumer | Setup-job steps | Agent-job extra steps | +| Setup consumer | Setup-job steps | Agent-job steps | |---|---|---| -| synth-PR (no gate) | install + download + synth-PR | (per Agent consumer above) | -| gate (no synth-PR) | install + download + gate | (per Agent consumer above) | -| synth-PR + gate | install + download + synth-PR + gate | (per Agent consumer above) | +| synth-PR (no gate) | install + download + synth-PR | install + download + optional helpers + invoker | +| gate (no synth-PR) | install + download + gate | install + download + optional helpers + invoker | +| synth-PR + gate | install + download + synth-PR + gate | install + download + optional helpers + invoker | The "Setup consumer" column is gated on `filters:` lowering to non-empty -checks **or** `synthetic_pr_active()` being true. The "Agent consumer" -columns are gated on `inlined-imports: false` (resolver) and the PR -contributor's activation predicate (exec-context-pr; see -`pr_contributor_will_activate` in -`src/compile/extensions/exec_context/mod.rs`). +checks **or** `synthetic_pr_active()` being true. The Agent download is +unconditional; optional resolver and execution-context steps retain their own +feature predicates. The IR-to-bash codegen that produces the gate step is `compile_gate_step_external` in `src/compile/filter_ir.rs`. diff --git a/docs/engine.md b/docs/engine.md index c056d7b5b..0c39757bb 100644 --- a/docs/engine.md +++ b/docs/engine.md @@ -22,7 +22,7 @@ engine: | Field | Type | Default | Description | |-------|------|---------|-------------| | `id` | string | `copilot` | Engine identifier. Currently only `copilot` (GitHub Copilot CLI) is supported. | -| `model` | string | *(none)* | AI model to use (e.g., `gpt-5-mini`). When set, the compiler passes the value directly to the Copilot CLI `--model` flag. When omitted, runtime model controls can select a model; if no runtime control is set, the compiler omits `--model` and the Copilot CLI chooses its own default. | +| `model` | string | *(none)* | AI model to use (e.g., `gpt-5-mini`). When set, the compiler records it in the versioned Copilot invocation document and the invoker sets Copilot CLI's native `COPILOT_MODEL` environment variable. When omitted, runtime model controls can select a model; if no runtime control is set, `COPILOT_MODEL` remains unset and the Copilot CLI chooses its own default. | | `timeout-minutes` | integer | *(none)* | Maximum time in minutes the agent job is allowed to run. Sets `timeoutInMinutes` on the `Agent` job in the generated pipeline. | | `version` | string | *(none)* | Engine CLI version to install (e.g., `"1.0.70"`, `"latest"`). Overrides the pinned `COPILOT_CLI_VERSION`. Set to `"latest"` to use the newest available version. | | `agent` | string | *(none)* | Custom agent file identifier (Copilot only). Adds `--agent ` to the CLI invocation, selecting a custom agent from `.github/agents/`. | @@ -62,19 +62,22 @@ variables are an operational escape hatch only for workflows that leave `engine.model` unset; changing a pinned frontmatter model still requires recompilation. -Runtime values are passed through typed step environment mappings and validated -for model-identifier characters at runtime. The task that starts Copilot exports -the selected value through Copilot CLI's native `COPILOT_MODEL` environment -variable. When no value resolves, the task leaves `COPILOT_MODEL` unset rather -than supplying a compiler default. Raw `engine.args --model` and +Runtime values are passed through typed step environment mappings, so Azure +DevOps YAML variables, UI variables, variable groups, and variables set by an +earlier trusted `##vso[task.setvariable]` step all resolve at task start. Inside +AWF, the compiler-owned `copilot-invoker.js` validates the selected value and +sets Copilot CLI's native `COPILOT_MODEL` only in the child process environment. +When no value resolves, the invoker removes `COPILOT_MODEL` rather than +supplying a compiler default. Raw `engine.args --model` and `engine.env.COPILOT_MODEL` are rejected so they cannot bypass this precedence. -Agent resolves and records its requested session model in the actual Agent run -task, after user-authored steps. A prior trusted step can therefore use -`##vso[task.setvariable]` and both execution and metadata see the task-start -value. Detection captures its requested model in the Detection run task and +The invoker writes a strict result document before starting Copilot. The trusted +Agent host task reads that result after AWF returns and records the requested +session model in `aw_info.json`; Detection uses the same result contract and later enriches the copied metadata in `analyzed_outputs_` from its own -job scope. `ado-aw audit` merges those job-owned fields. +job scope. A prior trusted step can therefore set a variable and both execution +and metadata see the same task-start value. `ado-aw audit` merges those +job-owned fields. When `engine.agent` selects a custom agent whose definition declares `model` or `models`, Copilot CLI may use that agent-pinned model instead of the requested diff --git a/scripts/ado-script/.gitignore b/scripts/ado-script/.gitignore index 019aca5c2..196e2b199 100644 --- a/scripts/ado-script/.gitignore +++ b/scripts/ado-script/.gitignore @@ -17,6 +17,7 @@ github-app-token.js prepare-pr-base.js ado-proxy.js azure-wif-refresh.js +copilot-invoker.js schema *.tsbuildinfo test-bin diff --git a/scripts/ado-script/package.json b/scripts/ado-script/package.json index 4e6c5e1f7..049ae9d8a 100644 --- a/scripts/ado-script/package.json +++ b/scripts/ado-script/package.json @@ -7,8 +7,8 @@ "node": ">=20.0.0" }, "scripts": { - "build": "npm run codegen && npm run clean && npm run build:gate && npm run build:import && npm run build:exec-context-pr && npm run build:exec-context-pr-synth && npm run build:exec-context-manual && npm run build:exec-context-pipeline && npm run build:exec-context-ci-push && npm run build:exec-context-workitem && npm run build:exec-context-schedule && npm run build:exec-context-pr-checks && npm run build:exec-context-repo && npm run build:conclusion && npm run build:approval-summary && npm run build:github-app-token && npm run build:prepare-pr-base && npm run build:ado-proxy && npm run build:azure-wif-refresh", - "clean": "node -e \"const fs=require('node:fs'); fs.rmSync('.ado-build',{recursive:true,force:true}); for (const n of ['gate','import','exec-context-pr','exec-context-pr-synth','exec-context-manual','exec-context-pipeline','exec-context-ci-push','exec-context-workitem','exec-context-schedule','exec-context-pr-checks','exec-context-repo','conclusion','approval-summary','github-app-token','prepare-pr-base','ado-proxy','azure-wif-refresh']) fs.rmSync(n+'.js',{force:true});\"", + "build": "npm run codegen && npm run clean && npm run build:gate && npm run build:import && npm run build:exec-context-pr && npm run build:exec-context-pr-synth && npm run build:exec-context-manual && npm run build:exec-context-pipeline && npm run build:exec-context-ci-push && npm run build:exec-context-workitem && npm run build:exec-context-schedule && npm run build:exec-context-pr-checks && npm run build:exec-context-repo && npm run build:conclusion && npm run build:approval-summary && npm run build:github-app-token && npm run build:prepare-pr-base && npm run build:ado-proxy && npm run build:azure-wif-refresh && npm run build:copilot-invoker", + "clean": "node -e \"const fs=require('node:fs'); fs.rmSync('.ado-build',{recursive:true,force:true}); for (const n of ['gate','import','exec-context-pr','exec-context-pr-synth','exec-context-manual','exec-context-pipeline','exec-context-ci-push','exec-context-workitem','exec-context-schedule','exec-context-pr-checks','exec-context-repo','conclusion','approval-summary','github-app-token','prepare-pr-base','ado-proxy','azure-wif-refresh','copilot-invoker']) fs.rmSync(n+'.js',{force:true});\"", "build:gate": "ncc build src/gate/index.ts -o .ado-build/gate -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/gate/index.js','gate.js'); fs.rmSync('.ado-build/gate',{recursive:true,force:true});\"", "build:import": "ncc build src/import/index.ts -o .ado-build/import -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/import/index.js','import.js'); fs.rmSync('.ado-build/import',{recursive:true,force:true});\"", "build:exec-context-pr": "ncc build src/exec-context-pr/index.ts -o .ado-build/exec-context-pr -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/exec-context-pr/index.js','exec-context-pr.js'); fs.rmSync('.ado-build/exec-context-pr',{recursive:true,force:true});\"", @@ -26,13 +26,14 @@ "build:prepare-pr-base": "ncc build src/prepare-pr-base/index.ts -o .ado-build/prepare-pr-base -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/prepare-pr-base/index.js','prepare-pr-base.js'); fs.rmSync('.ado-build/prepare-pr-base',{recursive:true,force:true});\"", "build:ado-proxy": "ncc build src/ado-proxy/index.ts -o .ado-build/ado-proxy -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/ado-proxy/index.js','ado-proxy.js'); fs.rmSync('.ado-build/ado-proxy',{recursive:true,force:true});\"", "build:azure-wif-refresh": "ncc build src/azure-wif-refresh/index.ts -o .ado-build/azure-wif-refresh -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/azure-wif-refresh/index.js','azure-wif-refresh.js'); fs.rmSync('.ado-build/azure-wif-refresh',{recursive:true,force:true});\"", + "build:copilot-invoker": "ncc build src/copilot-invoker/index.ts -o .ado-build/copilot-invoker -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/copilot-invoker/index.js','copilot-invoker.js'); fs.rmSync('.ado-build/copilot-invoker',{recursive:true,force:true});\"", "build:executor-e2e": "ncc build src/executor-e2e/index.ts -o .ado-build/executor-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/executor-e2e/index.js','test-bin/executor-e2e.js'); fs.rmSync('.ado-build/executor-e2e',{recursive:true,force:true});\"", "build:trigger-e2e": "ncc build src/trigger-e2e/index.ts -o .ado-build/trigger-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/trigger-e2e/index.js','test-bin/trigger-e2e.js'); fs.rmSync('.ado-build/trigger-e2e',{recursive:true,force:true});\"", "build:compiler-smoke-e2e": "ncc build src/compiler-smoke-e2e/index.ts -o .ado-build/compiler-smoke-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/compiler-smoke-e2e/index.js','test-bin/compiler-smoke-e2e.js'); fs.rmSync('.ado-build/compiler-smoke-e2e',{recursive:true,force:true});\"", "build:check": "ls -lh gate.js && wc -c gate.js", "codegen": "node -e \"require('node:fs').mkdirSync('schema', { recursive: true })\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-gate-schema --output schema/gate-spec.schema.json && npx json2ts schema/gate-spec.schema.json -o src/shared/types.gen.ts --bannerComment \"// AUTO-GENERATED from Rust IR via cargo run -- export-gate-schema. Do not edit; run npm run codegen.\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-fact-catalog --output src/trigger-e2e/fact-catalog.gen.json && cargo run --quiet --manifest-path ../../Cargo.toml -- export-ado-proxy-catalog-schema --output schema/ado-proxy-catalog.schema.json && npx json2ts schema/ado-proxy-catalog.schema.json -o src/shared/ado-proxy-catalog.types.gen.ts --bannerComment \"// AUTO-GENERATED from Rust via cargo run -- export-ado-proxy-catalog-schema. Do not edit; run npm run codegen.\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-ado-proxy-catalog --output src/ado-proxy/catalog.gen.json", "test": "vitest run", - "test:smoke": "npm run build:gate && npm run build:import && npm run build:exec-context-pr && npm run build:exec-context-pr-synth && npm run build:exec-context-manual && npm run build:exec-context-pipeline && npm run build:exec-context-ci-push && npm run build:exec-context-workitem && npm run build:exec-context-schedule && npm run build:exec-context-pr-checks && npm run build:exec-context-repo && npm run build:conclusion && npm run build:approval-summary && npm run build:github-app-token && npm run build:prepare-pr-base && npm run build:ado-proxy && npm run build:azure-wif-refresh && vitest run -c vitest.config.smoke.ts", + "test:smoke": "npm run build:gate && npm run build:import && npm run build:exec-context-pr && npm run build:exec-context-pr-synth && npm run build:exec-context-manual && npm run build:exec-context-pipeline && npm run build:exec-context-ci-push && npm run build:exec-context-workitem && npm run build:exec-context-schedule && npm run build:exec-context-pr-checks && npm run build:exec-context-repo && npm run build:conclusion && npm run build:approval-summary && npm run build:github-app-token && npm run build:prepare-pr-base && npm run build:ado-proxy && npm run build:azure-wif-refresh && npm run build:copilot-invoker && vitest run -c vitest.config.smoke.ts", "lint": "echo TODO", "typecheck": "tsc --noEmit" }, diff --git a/scripts/ado-script/src/copilot-invoker/index.test.ts b/scripts/ado-script/src/copilot-invoker/index.test.ts new file mode 100644 index 000000000..5b7e77e76 --- /dev/null +++ b/scripts/ado-script/src/copilot-invoker/index.test.ts @@ -0,0 +1,269 @@ +import { EventEmitter } from "node:events"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; + +import { + buildChildEnvironment, + buildCopilotArgs, + parseInvocationDocument, + parseInvocationResult, + resolveRequestedModel, + runInvocation, + type InvocationDocument, + type InvocationResult, +} from "./index.js"; + +function document(overrides: Partial = {}): InvocationDocument { + return { + schema_version: 1, + role: "agent", + command: "/tmp/awf-tools/copilot", + prompt_path: "/tmp/awf-tools/agent-prompt.md", + mcp_config_path: "/tmp/awf-tools/mcp-config.json", + args: ["--disable-builtin-mcps", "--allow-tool", "shell(cat *)"], + explicit_model: null, + result_path: "/tmp/awf-tools/copilot-invocation-result.json", + ...overrides, + }; +} + +describe("copilot invoker document", () => { + it("parses a strict versioned document", () => { + expect(parseInvocationDocument(JSON.stringify(document()))).toEqual(document()); + }); + + describe("copilot invoker result", () => { + it("parses a strict result document", () => { + expect( + parseInvocationResult( + JSON.stringify({ + schema_version: 1, + role: "detection", + requested_model: "detector", + }), + ), + ).toEqual({ + schema_version: 1, + role: "detection", + requested_model: "detector", + }); + }); + + it.each([ + [{ schema_version: 2, role: "agent", requested_model: null }, "unsupported"], + [ + { schema_version: 1, role: "agent", requested_model: null, extra: true }, + "unknown field", + ], + [{ schema_version: 1, role: "other", requested_model: null }, "must be"], + [{ schema_version: 1, role: "agent", requested_model: "bad model" }, "invalid"], + ])("rejects malformed results %#", (value, message) => { + expect(() => parseInvocationResult(JSON.stringify(value))).toThrow(message); + }); + }); + + it.each([ + [{ ...document(), schema_version: 2 }, "unsupported invocation schema"], + [{ ...document(), unexpected: true }, "unknown field 'unexpected'"], + [{ ...document(), role: "other" }, "must be 'agent' or 'detection'"], + [{ ...document(), command: "copilot;sh" }, "field 'command' is invalid"], + [{ ...document(), command: ".." }, "field 'command' is invalid"], + [{ ...document(), command: "/tmp/../copilot" }, "field 'command' is invalid"], + [{ ...document(), prompt_path: "relative.md" }, "field 'prompt_path' is invalid"], + [{ ...document(), prompt_path: "/tmp/../prompt.md" }, "field 'prompt_path' is invalid"], + [{ ...document(), result_path: "/" }, "field 'result_path' is invalid"], + [{ ...document(), args: [1] }, "field 'args' must be an array of strings"], + [{ ...document(), explicit_model: "bad model" }, "field 'explicit_model' is invalid"], + ])("rejects malformed input %#", (value, message) => { + expect(() => parseInvocationDocument(JSON.stringify(value))).toThrow(message); + }); +}); + +describe("model resolution", () => { + it("keeps an explicit model authoritative", () => { + expect( + resolveRequestedModel(document({ explicit_model: "frontmatter-model" }), { + ADO_AW_MODEL_AGENT_COPILOT: "role-model", + ADO_AW_DEFAULT_MODEL_COPILOT: "default-model", + }), + ).toBe("frontmatter-model"); + }); + + it("uses role-specific then shared values", () => { + expect( + resolveRequestedModel(document(), { + ADO_AW_MODEL_AGENT_COPILOT: "role-model", + ADO_AW_DEFAULT_MODEL_COPILOT: "default-model", + }), + ).toBe("role-model"); + expect( + resolveRequestedModel(document({ role: "detection" }), { + ADO_AW_MODEL_DETECTION_COPILOT: "", + ADO_AW_DEFAULT_MODEL_COPILOT: "default-model", + }), + ).toBe("default-model"); + }); + + it("treats unresolved ADO macros as absent", () => { + expect( + resolveRequestedModel(document(), { + ADO_AW_MODEL_AGENT_COPILOT: "$(ADO_AW_MODEL_AGENT_COPILOT)", + ADO_AW_DEFAULT_MODEL_COPILOT: "$(ADO_AW_DEFAULT_MODEL_COPILOT)", + }), + ).toBeNull(); + }); + + it("rejects invalid runtime values without printing them", () => { + let message = ""; + try { + resolveRequestedModel(document(), { + ADO_AW_MODEL_AGENT_COPILOT: "secret value", + }); + } catch (error) { + message = error instanceof Error ? error.message : String(error); + } + expect(message).toContain("contains invalid characters"); + expect(message).not.toContain("secret value"); + }); +}); + +describe("argv and child environment", () => { + it("passes prompt, MCP config, and authored values as distinct argv elements", () => { + expect(buildCopilotArgs(document(), "line one\nline two")).toEqual([ + "--prompt=line one\nline two", + "--additional-mcp-config", + "@/tmp/awf-tools/mcp-config.json", + "--disable-builtin-mcps", + "--allow-tool", + "shell(cat *)", + ]); + }); + + it("sets or removes only the child COPILOT_MODEL", () => { + const original = { KEEP: "yes", COPILOT_MODEL: "old" }; + expect(buildChildEnvironment(original, "selected")).toEqual({ + KEEP: "yes", + COPILOT_MODEL: "selected", + }); + expect(buildChildEnvironment(original, null)).toEqual({ KEEP: "yes" }); + expect(original.COPILOT_MODEL).toBe("old"); + }); +}); + +describe("process lifecycle", () => { + it("publishes the result before spawning and preserves the child exit code", async () => { + const events: string[] = []; + const child = new EventEmitter() as EventEmitter & { + killed: boolean; + kill: ReturnType; + }; + child.killed = false; + child.kill = vi.fn(); + const results: InvocationResult[] = []; + let spawnedArgs: readonly string[] | undefined; + const spawn = vi.fn((_command: string, args: readonly string[]) => { + spawnedArgs = args; + events.push("spawn"); + queueMicrotask(() => child.emit("close", 23, null)); + return child; + }); + + const exit = await runInvocation( + document(), + { ADO_AW_MODEL_AGENT_COPILOT: "runtime-model" }, + { + readFile: () => "prompt", + writeResult: (_path, result) => { + events.push("result"); + results.push(result); + }, + spawn: spawn as never, + }, + ); + + expect(events).toEqual(["result", "spawn"]); + expect(results).toEqual([ + { schema_version: 1, role: "agent", requested_model: "runtime-model" }, + ]); + expect(exit).toBe(23); + expect(spawnedArgs).toEqual([ + "--prompt=prompt", + "--additional-mcp-config", + "@/tmp/awf-tools/mcp-config.json", + "--disable-builtin-mcps", + "--allow-tool", + "shell(cat *)", + ]); + }); + + it("writes results atomically with private permissions", async () => { + const directory = mkdtempSync(join(tmpdir(), "copilot-invoker-")); + const resultPath = join(directory, "result.json").replaceAll("\\", "/"); + const child = new EventEmitter() as EventEmitter & { + killed: boolean; + kill: ReturnType; + }; + child.killed = false; + child.kill = vi.fn(); + const invocation = document({ result_path: resultPath }); + const promise = runInvocation(invocation, {}, { + readFile: () => "prompt", + writeResult: (await import("./index.js")).writeResultAtomic, + spawn: (() => { + queueMicrotask(() => child.emit("close", 0, null)); + return child; + }) as never, + }); + await expect(promise).resolves.toBe(0); + expect(JSON.parse(readFileSync(resultPath, "utf8"))).toEqual({ + schema_version: 1, + role: "agent", + requested_model: null, + }); + }); + + it("returns a deterministic failure when Copilot cannot start", async () => { + const child = new EventEmitter() as EventEmitter & { + killed: boolean; + kill: ReturnType; + }; + child.killed = false; + child.kill = vi.fn(); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const promise = runInvocation(document(), {}, { + readFile: () => "prompt", + writeResult: () => undefined, + spawn: (() => { + queueMicrotask(() => child.emit("error", new Error("ENOENT"))); + return child; + }) as never, + }); + await expect(promise).resolves.toBe(1); + expect(error).toHaveBeenCalledWith( + "copilot-invoker: failed to start Copilot: ENOENT", + ); + error.mockRestore(); + }); + + it("forwards termination signals and maps a signaled exit", async () => { + const child = new EventEmitter() as EventEmitter & { + killed: boolean; + kill: ReturnType; + }; + child.killed = false; + child.kill = vi.fn((signal: NodeJS.Signals) => { + queueMicrotask(() => child.emit("close", null, signal)); + return true; + }); + const promise = runInvocation(document(), {}, { + readFile: () => "prompt", + writeResult: () => undefined, + spawn: (() => child) as never, + }); + process.emit("SIGTERM", "SIGTERM"); + await expect(promise).resolves.toBe(143); + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + }); +}); diff --git a/scripts/ado-script/src/copilot-invoker/index.ts b/scripts/ado-script/src/copilot-invoker/index.ts new file mode 100644 index 000000000..da5ec7358 --- /dev/null +++ b/scripts/ado-script/src/copilot-invoker/index.ts @@ -0,0 +1,370 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { readFileSync, renameSync, writeFileSync } from "node:fs"; +import { constants } from "node:os"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SCHEMA_VERSION = 1; +const MODEL_PATTERN = /^[A-Za-z0-9._:-]+$/; +const COMMAND_PATTERN = /^[A-Za-z0-9._/-]+$/; +const DOCUMENT_KEYS = new Set([ + "schema_version", + "role", + "command", + "prompt_path", + "mcp_config_path", + "args", + "explicit_model", + "result_path", +]); +const RESULT_KEYS = new Set(["schema_version", "role", "requested_model"]); + +export type InvocationRole = "agent" | "detection"; + +export interface InvocationDocument { + schema_version: 1; + role: InvocationRole; + command: string; + prompt_path: string; + mcp_config_path: string | null; + args: string[]; + explicit_model: string | null; + result_path: string; +} + +export interface InvocationResult { + schema_version: 1; + role: InvocationRole; + requested_model: string | null; +} + +interface SpawnLike { + ( + command: string, + args: readonly string[], + options: { + env: NodeJS.ProcessEnv; + stdio: "inherit"; + }, + ): ChildProcess; +} + +export interface RunDependencies { + spawn: SpawnLike; + readFile(path: string): string; + writeResult(path: string, result: InvocationResult): void; +} + +const DEFAULT_DEPENDENCIES: RunDependencies = { + spawn, + readFile: (path) => readFileSync(path, "utf8"), + writeResult: writeResultAtomic, +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requiredString( + value: Record, + key: string, + validator?: (input: string) => boolean, +): string { + const candidate = value[key]; + if (typeof candidate !== "string" || candidate.length === 0) { + throw new Error(`invocation document field '${key}' must be a non-empty string`); + } + if (candidate.includes("\0") || (validator && !validator(candidate))) { + throw new Error(`invocation document field '${key}' is invalid`); + } + return candidate; +} + +function nullableString( + value: Record, + key: string, + validator?: (input: string) => boolean, +): string | null { + const candidate = value[key]; + if (candidate === null) return null; + if (typeof candidate !== "string" || candidate.includes("\0")) { + throw new Error(`invocation document field '${key}' must be a string or null`); + } + if (validator && !validator(candidate)) { + throw new Error(`invocation document field '${key}' is invalid`); + } + return candidate; +} + +function hasSafePathSegments(value: string, allowBareCommand: boolean): boolean { + if ( + value.includes("\n") || + value.includes("\r") || + value.includes(":") || + value.endsWith("/") + ) { + return false; + } + if (allowBareCommand && !value.includes("/")) { + return value !== "." && value !== ".."; + } + if (!value.startsWith("/")) return false; + const segments = value.slice(1).split("/"); + return ( + segments.length > 0 && + segments.every((segment) => segment.length > 0 && segment !== "." && segment !== "..") + ); +} + +function isAbsoluteContainerPath(value: string): boolean { + return hasSafePathSegments(value, false); +} + +function isSafeCommand(value: string): boolean { + return COMMAND_PATTERN.test(value) && hasSafePathSegments(value, true); +} + +export function parseInvocationDocument(raw: string): InvocationDocument { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error("invocation document is not valid JSON"); + } + + if (!isRecord(parsed)) { + throw new Error("invocation document must be a JSON object"); + } + const unknown = Object.keys(parsed).filter((key) => !DOCUMENT_KEYS.has(key)); + if (unknown.length > 0) { + throw new Error(`invocation document contains unknown field '${unknown.sort()[0]}'`); + } + if (parsed.schema_version !== SCHEMA_VERSION) { + throw new Error( + `unsupported invocation schema version '${String(parsed.schema_version)}'`, + ); + } + if (parsed.role !== "agent" && parsed.role !== "detection") { + throw new Error("invocation document field 'role' must be 'agent' or 'detection'"); + } + if (!Array.isArray(parsed.args) || !parsed.args.every((arg) => typeof arg === "string")) { + throw new Error("invocation document field 'args' must be an array of strings"); + } + if (parsed.args.some((arg) => arg.includes("\0"))) { + throw new Error("invocation document field 'args' contains an invalid NUL byte"); + } + + return { + schema_version: SCHEMA_VERSION, + role: parsed.role, + command: requiredString(parsed, "command", isSafeCommand), + prompt_path: requiredString(parsed, "prompt_path", isAbsoluteContainerPath), + mcp_config_path: nullableString(parsed, "mcp_config_path", isAbsoluteContainerPath), + args: [...parsed.args], + explicit_model: nullableString(parsed, "explicit_model", (value) => + MODEL_PATTERN.test(value), + ), + result_path: requiredString(parsed, "result_path", isAbsoluteContainerPath), + }; +} + +export function parseInvocationResult(raw: string): InvocationResult { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error("invocation result is not valid JSON"); + } + if (!isRecord(parsed)) { + throw new Error("invocation result must be a JSON object"); + } + const unknown = Object.keys(parsed).filter((key) => !RESULT_KEYS.has(key)); + if (unknown.length > 0) { + throw new Error(`invocation result contains unknown field '${unknown.sort()[0]}'`); + } + if (parsed.schema_version !== SCHEMA_VERSION) { + throw new Error( + `unsupported invocation result schema version '${String(parsed.schema_version)}'`, + ); + } + if (parsed.role !== "agent" && parsed.role !== "detection") { + throw new Error("invocation result field 'role' must be 'agent' or 'detection'"); + } + const requestedModel = nullableString(parsed, "requested_model", (value) => + MODEL_PATTERN.test(value), + ); + return { + schema_version: SCHEMA_VERSION, + role: parsed.role, + requested_model: requestedModel, + }; +} + +function runtimeModelVariable(role: InvocationRole): string { + return role === "agent" + ? "ADO_AW_MODEL_AGENT_COPILOT" + : "ADO_AW_MODEL_DETECTION_COPILOT"; +} + +function isUnresolvedAdoMacro(value: string, variable: string): boolean { + return value === `$(${variable})`; +} + +export function resolveRequestedModel( + document: InvocationDocument, + env: NodeJS.ProcessEnv, +): string | null { + if (document.explicit_model !== null) { + return document.explicit_model; + } + const specific = runtimeModelVariable(document.role); + const candidates: Array<[string, string | undefined]> = [ + [specific, env[specific]], + ["ADO_AW_DEFAULT_MODEL_COPILOT", env.ADO_AW_DEFAULT_MODEL_COPILOT], + ]; + for (const [variable, candidate] of candidates) { + if ( + candidate === undefined || + candidate.length === 0 || + isUnresolvedAdoMacro(candidate, variable) + ) { + continue; + } + if (!MODEL_PATTERN.test(candidate)) { + throw new Error( + `runtime Copilot model from ${specific}/ADO_AW_DEFAULT_MODEL_COPILOT contains invalid characters`, + ); + } + return candidate; + } + return null; +} + +export function buildCopilotArgs( + document: InvocationDocument, + prompt: string, +): string[] { + const args = [`--prompt=${prompt}`]; + if (document.mcp_config_path !== null) { + args.push("--additional-mcp-config", `@${document.mcp_config_path}`); + } + args.push(...document.args); + return args; +} + +export function buildChildEnvironment( + env: NodeJS.ProcessEnv, + requestedModel: string | null, +): NodeJS.ProcessEnv { + const childEnv = { ...env }; + if (requestedModel === null) { + delete childEnv.COPILOT_MODEL; + } else { + childEnv.COPILOT_MODEL = requestedModel; + } + return childEnv; +} + +export function writeResultAtomic(path: string, result: InvocationResult): void { + const temporary = `${path}.tmp-${process.pid}`; + writeFileSync(temporary, `${JSON.stringify(result)}\n`, { + encoding: "utf8", + mode: 0o600, + }); + renameSync(temporary, path); +} + +function signalExitCode(signal: NodeJS.Signals): number { + return 128 + (constants.signals[signal] ?? 0); +} + +export async function runInvocation( + document: InvocationDocument, + env: NodeJS.ProcessEnv = process.env, + dependencies: RunDependencies = DEFAULT_DEPENDENCIES, +): Promise { + const requestedModel = resolveRequestedModel(document, env); + dependencies.writeResult(document.result_path, { + schema_version: SCHEMA_VERSION, + role: document.role, + requested_model: requestedModel, + }); + + const prompt = dependencies.readFile(document.prompt_path); + const args = buildCopilotArgs(document, prompt); + const child = dependencies.spawn(document.command, args, { + env: buildChildEnvironment(env, requestedModel), + stdio: "inherit", + }); + + return await new Promise((resolveExit) => { + let settled = false; + const settle = (code: number) => { + if (settled) return; + settled = true; + for (const signal of forwardedSignals) { + process.off(signal, handlers[signal]); + } + resolveExit(code); + }; + const forwardedSignals: NodeJS.Signals[] = ["SIGINT", "SIGTERM", "SIGHUP"]; + const handlers = Object.fromEntries( + forwardedSignals.map((signal) => [ + signal, + () => { + if (!child.killed) child.kill(signal); + }, + ]), + ) as Record void>; + for (const signal of forwardedSignals) { + process.on(signal, handlers[signal]); + } + child.once("error", (error) => { + console.error(`copilot-invoker: failed to start Copilot: ${error.message}`); + settle(1); + }); + child.once("close", (code, signal) => { + settle(code ?? (signal ? signalExitCode(signal) : 1)); + }); + }); +} + +export async function main(argv: string[]): Promise { + if (argv[0] === "read-result" && argv.length === 3) { + try { + const result = parseInvocationResult(readFileSync(argv[1]!, "utf8")); + if (result.role !== argv[2]) { + throw new Error( + `invocation result role '${result.role}' does not match expected role '${argv[2]}'`, + ); + } + process.stdout.write(result.requested_model ?? ""); + return 0; + } catch (error) { + const message = error instanceof Error ? error.message : "unknown error"; + console.error(`copilot-invoker: ${message}`); + return 1; + } + } + if (argv[0] !== "run" || argv.length !== 2) { + console.error( + "usage: copilot-invoker run | read-result ", + ); + return 2; + } + try { + const document = parseInvocationDocument(readFileSync(argv[1]!, "utf8")); + return await runInvocation(document); + } catch (error) { + const message = error instanceof Error ? error.message : "unknown error"; + console.error(`copilot-invoker: ${message}`); + return 1; + } +} + +const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; +if (invokedPath === fileURLToPath(import.meta.url)) { + void main(process.argv.slice(2)).then((code) => { + process.exitCode = code; + }); +} diff --git a/scripts/ado-script/test/azure-wif-isolation.test.ts b/scripts/ado-script/test/azure-wif-isolation.test.ts index 607a23443..0823661ea 100644 --- a/scripts/ado-script/test/azure-wif-isolation.test.ts +++ b/scripts/ado-script/test/azure-wif-isolation.test.ts @@ -260,6 +260,9 @@ printf '%s\\0' "$@" > '${capture}' const commandIndex = captured.indexOf("--"); expect(commandIndex).toBeGreaterThan(0); + expect(captured[commandIndex + 1]).toContain( + "copilot-invoker.js run /tmp/awf-tools/copilot-invocation.json", + ); const args: string[] = []; for (let i = 0; i < commandIndex; i++) { // This probe exercises filesystem/env isolation, not MCP networking: diff --git a/scripts/ado-script/test/smoke.test.ts b/scripts/ado-script/test/smoke.test.ts index 630ae1906..f95905657 100644 --- a/scripts/ado-script/test/smoke.test.ts +++ b/scripts/ado-script/test/smoke.test.ts @@ -10,6 +10,7 @@ import { spawnSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import { + chmodSync, copyFileSync, existsSync, mkdirSync, @@ -27,6 +28,7 @@ const gateBundlePath = resolve(__dirname, "../gate.js"); const importBundlePath = resolve(__dirname, "../import.js"); const execContextPrBundlePath = resolve(__dirname, "../exec-context-pr.js"); const preparePrBaseBundlePath = resolve(__dirname, "../prepare-pr-base.js"); +const copilotInvokerBundlePath = resolve(__dirname, "../copilot-invoker.js"); const gateFixturePath = resolve( __dirname, "fixtures/gate-spec-pr-title-match.json", @@ -131,6 +133,61 @@ describe("import.js smoke", () => { }, 20000); }); +describe.skipIf(process.platform === "win32")("copilot-invoker.js smoke", () => { + it("runs a fake Copilot child and publishes the requested model", () => { + withSmokeScratchDir("copilot-invoker", (dir) => { + const command = resolve(dir, "fake-copilot"); + const promptPath = resolve(dir, "prompt.md"); + const resultPath = resolve(dir, "result.json"); + const capturePath = resolve(dir, "capture.json"); + const documentPath = resolve(dir, "invocation.json"); + writeFileSync( + command, + `#!/bin/sh +node -e 'require("node:fs").writeFileSync(process.env.CAPTURE_PATH, JSON.stringify({ argv: process.argv.slice(1), model: process.env.COPILOT_MODEL }))' "$@" +exit 7 +`, + ); + chmodSync(command, 0o755); + writeFileSync(promptPath, "smoke prompt\n"); + writeFileSync( + documentPath, + JSON.stringify({ + schema_version: 1, + role: "agent", + command, + prompt_path: promptPath, + mcp_config_path: null, + args: ["--no-ask-user"], + explicit_model: "gpt-smoke", + result_path: resultPath, + }), + ); + + const run = spawnSync( + process.execPath, + [copilotInvokerBundlePath, "run", documentPath], + { + env: { ...process.env, CAPTURE_PATH: capturePath }, + encoding: "utf8", + }, + ); + expect(run.status).toBe(7); + expect(run.stdout).toBe(""); + expect(run.stderr).toBe(""); + expect(JSON.parse(readFileSync(capturePath, "utf8"))).toEqual({ + argv: ["--prompt=smoke prompt\n", "--no-ask-user"], + model: "gpt-smoke", + }); + expect(JSON.parse(readFileSync(resultPath, "utf8"))).toEqual({ + schema_version: 1, + role: "agent", + requested_model: "gpt-smoke", + }); + }); + }); +}); + function runGitInRepo(repoDir: string, args: string[]): void { const result = spawnSync("git", args, { cwd: repoDir, diff --git a/src/compile/ado_bundle.rs b/src/compile/ado_bundle.rs index 4bb45757f..f795ea469 100644 --- a/src/compile/ado_bundle.rs +++ b/src/compile/ado_bundle.rs @@ -53,6 +53,10 @@ pub enum Bundle { ExecContextRepo, ApprovalSummary, Conclusion, + /// Copilot process harness executed inside AWF. It receives only a + /// compiler-generated invocation document and the already-filtered Agent + /// environment, so it requires no ADO bearer. + CopilotInvoker, /// GitHub App installation-token minter/revoker (issue #1316). Runs before /// the Copilot invocation in the Agent and Detection jobs. It authenticates /// to the **GitHub** API (not ADO REST), so it needs no ADO bearer. @@ -155,6 +159,7 @@ impl Bundle { Bundle::ExecContextRepo, Bundle::ApprovalSummary, Bundle::Conclusion, + Bundle::CopilotInvoker, Bundle::GithubAppToken, Bundle::PreparePrBase, Bundle::AzureWifRefresh, @@ -183,6 +188,7 @@ impl Bundle { Bundle::ExecContextRepo => paths::EXEC_CONTEXT_REPO_PATH, Bundle::ApprovalSummary => paths::APPROVAL_SUMMARY_PATH, Bundle::Conclusion => paths::CONCLUSION_PATH, + Bundle::CopilotInvoker => paths::COPILOT_INVOKER_PATH, Bundle::GithubAppToken => paths::GITHUB_APP_TOKEN_PATH, Bundle::PreparePrBase => paths::PREPARE_PR_BASE_PATH, Bundle::AzureWifRefresh => paths::AZURE_WIF_REFRESH_PATH, @@ -212,6 +218,7 @@ impl Bundle { | Bundle::ExecContextManual | Bundle::ExecContextRepo | Bundle::ApprovalSummary + | Bundle::CopilotInvoker // Authenticates to the GitHub API with its own App JWT / minted // token, not the ADO bearer. | Bundle::GithubAppToken diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index 05fffae96..e5645a7a5 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -269,8 +269,8 @@ fn fanout_extension_declarations( /// function's cognitive complexity manageable — behaviour is unchanged. struct EngineSetup { compiler_version: String, - engine_run: String, - engine_run_detection: String, + agent_invocation_json: String, + detection_invocation_json: String, engine_install_steps_yaml: String, detection_engine_install_steps_yaml: String, engine_log_dir: String, @@ -293,19 +293,31 @@ fn build_engine_setup( let compiler_version = env!("CARGO_PKG_VERSION").to_string(); let detection_engine = crate::engine::get_engine(detection_engine_config.engine_id())?; - let engine_run = ctx.engine.invocation( + let agent_invocation = ctx.engine.invocation_document( ctx.front_matter, extension_declarations, - "/tmp/awf-tools/agent-prompt.md", - Some("/tmp/awf-tools/mcp-config.json"), + crate::engine::CopilotInvocationContext::new( + crate::engine::RuntimeModelRole::Agent, + "/tmp/awf-tools/agent-prompt.md", + Some("/tmp/awf-tools/mcp-config.json"), + "/tmp/awf-tools/copilot-invocation-result.json", + ), )?; - let engine_run_detection = detection_engine.detection_invocation_with_config( + let detection_invocation = detection_engine.invocation_document_with_config( detection_engine_config, ctx.front_matter, extension_declarations, - "/tmp/awf-tools/threat-analysis-prompt.md", - None, + crate::engine::CopilotInvocationContext::new( + crate::engine::RuntimeModelRole::Detection, + "/tmp/awf-tools/threat-analysis-prompt.md", + None, + "/tmp/awf-tools/copilot-invocation-result.json", + ), )?; + let agent_invocation_json = serde_json::to_string(&agent_invocation) + .context("failed to serialize Agent Copilot invocation document")?; + let detection_invocation_json = serde_json::to_string(&detection_invocation) + .context("failed to serialize Detection Copilot invocation document")?; let engine_install_steps_yaml = ctx.engine .install_steps(&front_matter.engine, &front_matter.target, ctx.ado_org())?; @@ -348,8 +360,8 @@ fn build_engine_setup( Ok(EngineSetup { compiler_version, - engine_run, - engine_run_detection, + agent_invocation_json, + detection_invocation_json, engine_install_steps_yaml, detection_engine_install_steps_yaml, engine_log_dir, @@ -440,8 +452,8 @@ pub(crate) fn build_pipeline_context( )?; let EngineSetup { compiler_version, - engine_run, - engine_run_detection, + agent_invocation_json, + detection_invocation_json, engine_install_steps_yaml, detection_engine_install_steps_yaml, engine_log_dir, @@ -573,8 +585,8 @@ pub(crate) fn build_pipeline_context( compiler_version: compiler_version.clone(), engine_install_steps_yaml, detection_engine_install_steps_yaml, - engine_run, - engine_run_detection, + agent_invocation_json, + detection_invocation_json, detection_engine_config, threat_detection, engine_env, @@ -828,8 +840,8 @@ pub(crate) struct StandaloneCtx { /// to typed steps. pub(crate) engine_install_steps_yaml: String, pub(crate) detection_engine_install_steps_yaml: String, - pub(crate) engine_run: String, - pub(crate) engine_run_detection: String, + pub(crate) agent_invocation_json: String, + pub(crate) detection_invocation_json: String, pub(crate) detection_engine_config: EngineConfig, pub(crate) threat_detection: ThreatDetectionConfig, /// Composed engine env block — `KEY: VALUE` lines, one per line. @@ -1328,8 +1340,7 @@ fn build_agent_job( // `checkout: self` (step 1) so the clone exists, and before the Copilot // run so the refs are present when the agent proposes a PR. The // `prepare-pr-base.js` bundle is staged by the ado-script extension's - // agent-prepare steps (`prepare_pr_base_active` is OR'd into that - // extension's Agent-job download predicate), so it is guaranteed present. + // always-on Agent preparation. if front_matter.create_pr_config().is_some() { // The prepare step deepens every checkout dir the SafeOutputs MCP server // may generate a patch from — see `create_pr_prepare_repos`. The @@ -1364,11 +1375,7 @@ fn build_agent_job( // step sets. Never runs for SafeOutputs/user steps. // // The ado-script bundle is staged by the ado-script extension's - // agent-prepare steps: `github_app_token_active` is OR'd into that - // extension's Agent-job download predicate (mirroring - // `safe_outputs_summary_active`), so the bundle is guaranteed present by - // the time we reach this step — no need to inspect emitted steps or - // re-download here. + // always-on Agent preparation, so no feature-specific download is needed. if let Some(app_token) = front_matter.engine.github_app_token() { steps.push(super::extensions::ado_script::github_app_token_step_typed( app_token, @@ -1389,9 +1396,8 @@ fn build_agent_job( &cfg.allowed_domains, &cfg.awf_mounts, &cfg.working_directory, - &cfg.engine_run, + &cfg.agent_invocation_json, &cfg.engine_env, - front_matter.engine.model().is_none(), &cfg.byom_exclude_keys, front_matter.supply_chain(), ado_proxy_enabled, @@ -1413,9 +1419,7 @@ fn build_agent_job( // emitted when any safe-output tool is enabled (transparency for every // run); when manual review is configured the reviewed proposals are listed // first. The ado-script bundle was delivered earlier in this job by the - // ado-script extension, gated on the SAME predicate - // (`has_any_safe_output_tool` → `safe_outputs_summary_active`), so the - // bundle is downloaded iff this step is emitted. + // ado-script extension's always-on Agent preparation. if front_matter.has_any_safe_output_tool() { let (_, reviewed_summary_tools) = front_matter.partition_safe_outputs_by_approval(); steps.push(Step::Bash(safe_outputs_summary_step( @@ -1547,17 +1551,6 @@ fn agent_job_variables_hoist( /// (see `AdoScriptExtension::build_agent_conditions` for today's /// only contributor — synth-PR-skip, PR-filter gate, pipeline-filter /// gate, and user `expression:` escape hatches). -/// Whether the Detection job must stage the `ado-script` bundle. The Detection -/// job has no extension-prepare phase (unlike the Agent job, whose bundle -/// download is contributed by `AdoScriptExtension`), so it stages the bundle -/// itself — but gated on this single predicate so exactly one download is -/// emitted. Today only the GitHub App token step needs it; future -/// detection-only bundle consumers should `||` their own condition in here -/// rather than adding a second `install_and_download_steps_typed` call. -fn detection_job_needs_ado_script_bundle(engine_config: &EngineConfig) -> bool { - engine_config.github_app_token().is_some() -} - fn build_detection_job( front_matter: &FrontMatter, cfg: &StandaloneCtx, @@ -1613,15 +1606,14 @@ fn build_detection_job( )?)); steps.push(Step::Bash(setup_compiler_step())); - // Stage auth support before custom pre-steps, but mint credentials only - // after them so trusted setup code receives the least privilege needed. - if detection_job_needs_ado_script_bundle(&cfg.detection_engine_config) { - steps.extend( - super::extensions::ado_script::install_and_download_steps_typed( - front_matter.supply_chain(), - ), - ); - } + // Detection always executes the Copilot invoker bundle. Stage it before + // custom pre-steps; mint optional credentials only after those steps so + // trusted setup code receives the least privilege needed. + steps.extend( + super::extensions::ado_script::install_and_download_steps_typed( + front_matter.supply_chain(), + ), + ); for user_step in &cfg.threat_detection.steps { steps.push(Step::RawYaml(step_to_raw_yaml_string(user_step)?)); } @@ -1641,7 +1633,7 @@ fn build_detection_job( steps.push(Step::Bash(run_threat_analysis_step( &cfg.detection_allowed_domains, &cfg.working_directory, - &cfg.engine_run_detection, + &cfg.detection_invocation_json, &cfg.detection_byom_exclude_keys, &cfg.detection_engine_env, crate::engine::github_token_source_var(&cfg.detection_engine_config), @@ -4460,61 +4452,40 @@ shell_script! { /// - `image_flags` — `--image-tag` plus optional `--image-registry` /// - `exclude_env` — provider credentials and internal MCP identity keys /// - `awf_mounts` — the compiler-supplied chain of `--mount "…"` args - /// - `routed_engine_run` — the single-quoted `NO_PROXY` prefix + engine - /// command that AWF invokes inside the sandbox + /// - `routed_invoker` — the fixed single-quoted `NO_PROXY` prefix + + /// compiler-owned invoker command that AWF runs inside the sandbox RUN_AGENT { interpreter: Bash, - bindings: [AGENT_TEMP, PIPELINE_WORKSPACE, ALLOWED_DOMAINS], - externals: [ - WORKING_DIRECTORY, - ADO_AW_MODEL_AGENT_COPILOT, - ADO_AW_DEFAULT_MODEL_COPILOT, - ADO_AW_EFFECTIVE_MODEL, - COPILOT_MODEL + bindings: [ + AGENT_TEMP, + PIPELINE_WORKSPACE, + ALLOWED_DOMAINS, + INVOCATION_DOCUMENT, + INVOCATION_DOCUMENT_PATH, + INVOCATION_RESULT_PATH, + COPILOT_INVOKER_PATH ], + externals: [WORKING_DIRECTORY], fragments: [ - resolve_runtime_model, append_aw_info_field, topology_attach, image_flags, exclude_env, awf_mounts, - routed_engine_run + routed_invoker ], phases: [append_aw_info_field = super::extensions::APPEND_AW_INFO_FIELD], - fragment_uses: [ - resolve_runtime_model => [ - ADO_AW_MODEL_AGENT_COPILOT, - ADO_AW_DEFAULT_MODEL_COPILOT, - ADO_AW_EFFECTIVE_MODEL, - COPILOT_MODEL, - ], - ], body: r###" set -eo pipefail AGENT_OUTPUT_FILE="$AGENT_TEMP/staging/logs/agent-output.txt" mkdir -p "$AGENT_TEMP/staging/logs" AGENT_EXIT_CODE=0 +printf '%s\n' "$INVOCATION_DOCUMENT" > "$INVOCATION_DOCUMENT_PATH" +rm -f "$INVOCATION_RESULT_PATH" echo "=== Running AI agent with AWF network isolation ===" echo "Allowed domains: $ALLOWED_DOMAINS" - -# Resolve the task-scoped runtime model after all user-authored Agent steps. -set -e -# ado-aw:fragment resolve_runtime_model - -# The marker extension always creates this file for production compile contexts. -ADO_AW_INFO_JSON="$AGENT_TEMP/staging/aw_info.json" -if [ -f "$ADO_AW_INFO_JSON" ]; then - # ado-aw:fragment append_aw_info_field - ado_aw_append_info_field \ - "model" \ - "${COPILOT_MODEL:-}" \ - "$ADO_AW_INFO_JSON" -else - echo "Warning: Agent metadata file not found at $ADO_AW_INFO_JSON; model metadata was not recorded" >&2 -fi set +e # AWF provides L7 domain whitelisting via a rootless Docker topology. @@ -4539,7 +4510,7 @@ AWF_ARGS+=( --log-level info --proxy-logs-dir "$AGENT_TEMP/staging/logs/firewall" ) -# ado-aw:fragment routed_engine_run +# ado-aw:fragment routed_invoker # Stream agent output in real-time while filtering VSO commands. # sed -u = unbuffered (line-by-line) so output appears immediately. @@ -4551,6 +4522,27 @@ AWF_ARGS+=( | tee "$AGENT_OUTPUT_FILE" \ || AGENT_EXIT_CODE=$? +MODEL_RESULT_STATUS=0 +REQUESTED_MODEL=$(node "$COPILOT_INVOKER_PATH" read-result "$INVOCATION_RESULT_PATH" agent) \ + || MODEL_RESULT_STATUS=$? +if [ "$MODEL_RESULT_STATUS" -ne 0 ]; then + echo "ERROR: Agent Copilot invocation result is missing or malformed" >&2 + if [ "$AGENT_EXIT_CODE" -eq 0 ]; then + AGENT_EXIT_CODE="$MODEL_RESULT_STATUS" + fi +else + ADO_AW_INFO_JSON="$AGENT_TEMP/staging/aw_info.json" + if [ -f "$ADO_AW_INFO_JSON" ]; then + # ado-aw:fragment append_aw_info_field + ado_aw_append_info_field \ + "model" \ + "$REQUESTED_MODEL" \ + "$ADO_AW_INFO_JSON" + else + echo "Warning: Agent metadata file not found at $ADO_AW_INFO_JSON; model metadata was not recorded" >&2 + fi +fi + # Print firewall summary if available if [ -x "$PIPELINE_WORKSPACE/awf/awf" ]; then echo "=== Firewall Summary ===" @@ -4567,9 +4559,8 @@ fn run_agent_step( allowed_domains: &str, awf_mounts: &str, working_directory: &str, - engine_run: &str, + invocation_document: &str, engine_env: &str, - runtime_model_enabled: bool, byom_exclude_keys: &[String], supply_chain: Option<&SupplyChainConfig>, ado_proxy_enabled: bool, @@ -4610,12 +4601,6 @@ fn run_agent_step( }; let image_flags_block = awf_image_flags(supply_chain); let exclude_env_block = awf_exclude_env_flags(byom_exclude_keys); - let resolve_runtime_model = if runtime_model_enabled { - crate::engine::runtime_model_preamble(crate::engine::RuntimeModelRole::Agent) - } else { - ":".to_string() - }; - // AWF attaches externally-launched trusted containers to its internal // network by name. The flag is repeatable, which is what lets the policy // engine join alongside MCPG. Attaching also gives the agent an @@ -4670,9 +4655,10 @@ fn run_agent_step( } else { MCPG_CONTAINER_NAME.to_string() }; - let routed_engine_run = format!( + let routed_invoker = format!( "AWF_ARGS+=(-- 'export NO_PROXY=\"${{NO_PROXY:+$NO_PROXY,}}{no_proxy_peers}\"; \ - export no_proxy=\"$NO_PROXY\"; {engine_run}')" + export no_proxy=\"$NO_PROXY\"; exec node {} run /tmp/awf-tools/copilot-invocation.json')", + super::extensions::ado_script::COPILOT_INVOKER_PATH ); let mut step = ShellScript::new(&RUN_AGENT) @@ -4682,7 +4668,22 @@ fn run_agent_step( Binding::ado_macro("Pipeline.Workspace"), ) .bind_text("ALLOWED_DOMAINS", allowed_domains) - .fragment("resolve_runtime_model", resolve_runtime_model) + .bind( + "INVOCATION_DOCUMENT", + Binding::document(invocation_document), + ) + .bind_text( + "INVOCATION_DOCUMENT_PATH", + "/tmp/awf-tools/copilot-invocation.json", + ) + .bind_text( + "INVOCATION_RESULT_PATH", + "/tmp/awf-tools/copilot-invocation-result.json", + ) + .bind_text( + "COPILOT_INVOKER_PATH", + super::extensions::ado_script::COPILOT_INVOKER_PATH, + ) .fragment( "append_aw_info_field", phase_body(&super::extensions::APPEND_AW_INFO_FIELD), @@ -4691,7 +4692,7 @@ fn run_agent_step( .fragment("image_flags", image_flags_line) .fragment("exclude_env", exclude_env_line) .fragment("awf_mounts", awf_mounts_block) - .fragment("routed_engine_run", routed_engine_run) + .fragment("routed_invoker", routed_invoker) .into_step("Run copilot (AWF network isolated)"); step.working_directory = Some(working_directory.to_string()); // Engine env comes as a multi-line YAML env block — `KEY: VALUE` lines @@ -4852,8 +4853,7 @@ node "$APPROVAL_SUMMARY_PATH" || echo "##vso[task.logissue type=warning]approval /// Emitted at the **end of the Agent job** (after `collect_safe_outputs_step` /// has staged `safe_outputs.ndjson`), never in the Detection/threat-analysis /// job. The ado-script bundle is delivered earlier in the same job by the -/// ado-script extension's agent-prepare steps (gated on -/// `safe_outputs_summary_active`). +/// ado-script extension's always-on Agent preparation. /// /// `reviewed` is the compiler-resolved set of approval-gated tool names; when /// non-empty the bundle lists those proposals first under a "Pending approval" @@ -6260,33 +6260,25 @@ shell_script! { /// verbatim. RUN_THREAT_ANALYSIS { interpreter: Bash, - bindings: [AGENT_TEMP, PIPELINE_WORKSPACE, ALLOWED_DOMAINS], - externals: [ - WORKING_DIRECTORY, - ADO_AW_MODEL_DETECTION_COPILOT, - ADO_AW_DEFAULT_MODEL_COPILOT, - ADO_AW_EFFECTIVE_MODEL, - COPILOT_MODEL - ], - fragments: [capture_runtime_model, image_flags, exclude_env, engine_run_detection], - fragment_uses: [ - capture_runtime_model => [ - ADO_AW_MODEL_DETECTION_COPILOT, - ADO_AW_DEFAULT_MODEL_COPILOT, - ADO_AW_EFFECTIVE_MODEL, - COPILOT_MODEL, - ], + bindings: [ + AGENT_TEMP, + PIPELINE_WORKSPACE, + ALLOWED_DOMAINS, + INVOCATION_DOCUMENT, + INVOCATION_DOCUMENT_PATH, + INVOCATION_RESULT_PATH, + COPILOT_INVOKER_PATH ], + externals: [WORKING_DIRECTORY], + fragments: [image_flags, exclude_env, run_invoker], body: r###" set -o pipefail # Run threat analysis with AWF network isolation THREAT_OUTPUT_FILE="$AGENT_TEMP/threat-analysis-output.txt" AGENT_EXIT_CODE=0 - -# Capture the model selected from this task's Detection-scoped environment -# before trusted post-steps can change pipeline variables. -# ado-aw:fragment capture_runtime_model +printf '%s\n' "$INVOCATION_DOCUMENT" > "$INVOCATION_DOCUMENT_PATH" +rm -f "$INVOCATION_RESULT_PATH" # The argument list is assembled into an array so runtime-supplied # fragments splice in as ordinary shell statements (`AWF_ARGS+=(...)`) @@ -6303,7 +6295,7 @@ AWF_ARGS+=( --log-level info --proxy-logs-dir "$AGENT_TEMP/threat-analysis-logs/firewall" ) -# ado-aw:fragment engine_run_detection +# ado-aw:fragment run_invoker # Stream threat analysis output in real-time with VSO command filtering # shellcheck disable=SC2016 # The single-quoted engine command inside AWF_ARGS is intentionally expanded by AWF inside the sandbox @@ -6312,6 +6304,18 @@ AWF_ARGS+=( | tee "$THREAT_OUTPUT_FILE" \ || AGENT_EXIT_CODE=$? +MODEL_RESULT_STATUS=0 +REQUESTED_MODEL=$(node "$COPILOT_INVOKER_PATH" read-result "$INVOCATION_RESULT_PATH" detection) \ + || MODEL_RESULT_STATUS=$? +if [ "$MODEL_RESULT_STATUS" -ne 0 ]; then + echo "ERROR: Detection Copilot invocation result is missing or malformed" >&2 + if [ "$AGENT_EXIT_CODE" -eq 0 ]; then + AGENT_EXIT_CODE="$MODEL_RESULT_STATUS" + fi +else + printf '%s' "$REQUESTED_MODEL" > "$AGENT_TEMP/detection-runtime-model" +fi + exit "$AGENT_EXIT_CODE" "###, } @@ -6320,7 +6324,7 @@ exit "$AGENT_EXIT_CODE" fn run_threat_analysis_step( allowed_domains: &str, working_directory: &str, - engine_run_detection: &str, + invocation_document: &str, byom_exclude_keys: &[String], detection_engine_env: &[(String, String)], github_token_var: &str, @@ -6356,18 +6360,10 @@ fn run_threat_analysis_step( format!("AWF_ARGS+=({})", parts.join(" ")) } }; - let engine_run_detection_line = format!("AWF_ARGS+=(-- '{engine_run_detection}')"); - let runtime_model_enabled = detection_engine_env - .iter() - .any(|(key, _)| key == crate::engine::ADO_AW_MODEL_DETECTION_COPILOT); - let capture_runtime_model = if runtime_model_enabled { - format!( - "{}\nprintf '%s' \"${{COPILOT_MODEL:-}}\" > \"$AGENT_TEMP/detection-runtime-model\"", - crate::engine::runtime_model_preamble(crate::engine::RuntimeModelRole::Detection) - ) - } else { - ":".to_string() - }; + let run_invoker = format!( + "AWF_ARGS+=(-- 'exec node {} run /tmp/awf-tools/copilot-invocation.json')", + super::extensions::ado_script::COPILOT_INVOKER_PATH + ); let mut step = ShellScript::new(&RUN_THREAT_ANALYSIS) .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) @@ -6376,10 +6372,25 @@ fn run_threat_analysis_step( Binding::ado_macro("Pipeline.Workspace"), ) .bind_text("ALLOWED_DOMAINS", allowed_domains) - .fragment("capture_runtime_model", capture_runtime_model) + .bind( + "INVOCATION_DOCUMENT", + Binding::document(invocation_document), + ) + .bind_text( + "INVOCATION_DOCUMENT_PATH", + "/tmp/awf-tools/copilot-invocation.json", + ) + .bind_text( + "INVOCATION_RESULT_PATH", + "/tmp/awf-tools/copilot-invocation-result.json", + ) + .bind_text( + "COPILOT_INVOKER_PATH", + super::extensions::ado_script::COPILOT_INVOKER_PATH, + ) .fragment("image_flags", image_flags_line) .fragment("exclude_env", exclude_env_line) - .fragment("engine_run_detection", engine_run_detection_line) + .fragment("run_invoker", run_invoker) .into_step("Run threat analysis (AWF network isolated)"); step.working_directory = Some(working_directory.to_string()); // env block: GITHUB_TOKEN + GITHUB_READ_ONLY — emit the latter as @@ -7118,8 +7129,8 @@ mod tests { compiler_version: "0.0.0-test".to_string(), engine_install_steps_yaml: String::new(), detection_engine_install_steps_yaml: String::new(), - engine_run: "echo agent".to_string(), - engine_run_detection: "echo detection".to_string(), + agent_invocation_json: "{}".to_string(), + detection_invocation_json: "{}".to_string(), detection_engine_config: EngineConfig::default(), threat_detection: ThreatDetectionConfig::default(), engine_env: "GITHUB_READ_ONLY: 1".to_string(), @@ -7790,9 +7801,8 @@ safe-outputs: "example.com", "\\", "/work", - "copilot -p prompt", + r#"{"schema_version":1,"role":"agent"}"#, "FOO: bar", - false, &[], None, ado_proxy_enabled, @@ -7806,9 +7816,8 @@ safe-outputs: "example.com", "\\", "/work", - "copilot -p prompt", + r#"{"schema_version":1,"role":"agent"}"#, "ADO_AW_MODEL_AGENT_COPILOT: $(ADO_AW_MODEL_AGENT_COPILOT)\nADO_AW_DEFAULT_MODEL_COPILOT: $(ADO_AW_DEFAULT_MODEL_COPILOT)", - true, &[], None, false, @@ -7852,7 +7861,7 @@ safe-outputs: } #[test] - fn agent_runtime_model_is_resolved_recorded_and_exported_in_run_task() { + fn agent_runtime_model_is_delegated_to_invoker_and_recorded_after_awf() { let step = runtime_agent_step_for_test(); assert!(matches!( step.env @@ -7866,20 +7875,23 @@ safe-outputs: Some(EnvValue::PipelineVar(name)) if name == crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT )); - assert!(step.script.contains("export COPILOT_MODEL=\"$ADO_AW_EFFECTIVE_MODEL\"")); + assert!(step.script.contains("copilot-invoker.js run")); + assert!(step + .script + .contains("node \"$COPILOT_INVOKER_PATH\" read-result")); assert!(step.script.contains("\"model\"")); - assert!(step.script.contains("\"${COPILOT_MODEL:-}\"")); let metadata_index = step .script - .find("ado_aw_append_info_field") + .rfind("ado_aw_append_info_field") .expect("metadata append"); let awf_index = step .script .find("\"$PIPELINE_WORKSPACE/awf/awf\"") .expect("AWF invocation"); - assert!(metadata_index < awf_index); + assert!(awf_index < metadata_index); assert!(!step.script.contains("$(ADO_AW_MODEL_AGENT_COPILOT)")); assert!(!step.script.contains("$(ADO_AW_DEFAULT_MODEL_COPILOT)")); + assert!(!step.script.contains("ADO_AW_EFFECTIVE_MODEL")); assert!(!step.script.contains("--model")); } @@ -7901,30 +7913,26 @@ safe-outputs: .script .find("\"$PIPELINE_WORKSPACE/awf/awf\"") .expect("AWF invocation"); - assert!(warning_index < awf_index); + assert!(awf_index < warning_index); } #[test] - fn prefixed_user_env_key_does_not_unset_static_model() { + fn prefixed_user_env_key_does_not_change_fixed_invoker_command() { let step = run_agent_step( "example.com", "\\", "/work", - "copilot -p prompt", - "COPILOT_MODEL: static-model\nADO_AW_MODEL_AGENT_COPILOT_X: harmless", - false, + r#"{"schema_version":1,"role":"agent","explicit_model":"static-model"}"#, + "ADO_AW_MODEL_AGENT_COPILOT_X: harmless", &[], None, false, ) .expect("run_agent_step should build"); - assert!(matches!( - step.env.get(crate::engine::COPILOT_MODEL), - Some(EnvValue::Literal(value)) if value == "static-model" - )); assert!(!step.script.contains("ADO_AW_EFFECTIVE_MODEL")); assert!(!step.script.contains("unset COPILOT_MODEL")); + assert!(step.script.contains("copilot-invoker.js run")); } #[test] @@ -8483,8 +8491,8 @@ safe-outputs: compiler_version: "0.0.0-test".to_string(), engine_install_steps_yaml: String::new(), detection_engine_install_steps_yaml: String::new(), - engine_run: String::new(), - engine_run_detection: String::new(), + agent_invocation_json: "{}".to_string(), + detection_invocation_json: "{}".to_string(), detection_engine_config, threat_detection, engine_env: "env:\n GITHUB_TOKEN: $(GITHUB_TOKEN)\n".to_string(), @@ -8753,22 +8761,22 @@ safe-outputs: .script .contains("$AGENT_TEMP/detection-runtime-model") ); - assert!( - run_step - .script - .contains("export COPILOT_MODEL=\"$ADO_AW_EFFECTIVE_MODEL\"") - ); + assert!(run_step.script.contains("copilot-invoker.js run")); + assert!(run_step + .script + .contains("node \"$COPILOT_INVOKER_PATH\" read-result")); + assert!(!run_step.script.contains("ADO_AW_EFFECTIVE_MODEL")); assert!(!run_step.script.contains("--model")); assert!( run_step .script - .find("$AGENT_TEMP/detection-runtime-model") + .find("\"$PIPELINE_WORKSPACE/awf/awf\"") .unwrap() < run_step .script - .find("\"$PIPELINE_WORKSPACE/awf/awf\"") + .find("$AGENT_TEMP/detection-runtime-model") .unwrap(), - "runtime model must be captured before the Detection engine runs" + "the trusted host must consume the invoker result after Detection runs" ); assert!( !run_step diff --git a/src/compile/extensions/ado_script.rs b/src/compile/extensions/ado_script.rs index 26bd43c07..9cc39371a 100644 --- a/src/compile/extensions/ado_script.rs +++ b/src/compile/extensions/ado_script.rs @@ -220,6 +220,8 @@ node "$BUNDLE" pub(crate) const GATE_EVAL_PATH: &str = "/tmp/ado-aw-scripts/ado-script/gate.js"; pub(crate) const IMPORT_EVAL_PATH: &str = "/tmp/ado-aw-scripts/ado-script/import.js"; +pub(crate) const COPILOT_INVOKER_PATH: &str = + "/tmp/ado-aw-scripts/ado-script/copilot-invoker.js"; /// Path to the ado-proxy bundle inside the unpacked `ado-script.zip`. /// /// Unlike every other bundle this one is not executed by a pipeline step. It @@ -308,83 +310,6 @@ pub struct AdoScriptExtension { pub pr_filters: Option, pub pipeline_filters: Option, pub inlined_imports: bool, - /// Whether the PR-context contributor will activate. When true, - /// the Agent-job install/download must fire even if - /// `runtime_imports_active()` is false (i.e. the user has - /// `inlined-imports: true` but a PR trigger configured), so that - /// `exec-context-pr.js` is present for the `pr.rs` invocation. - /// - /// Populated at construction by `collect_extensions` using the - /// shared `exec_context_pr_active` predicate so this stays in - /// lock-step with `ExecContextExtension`'s own activation gate. - pub exec_context_pr_active: bool, - /// Whether the Manual-context contributor (Stage 1 of the - /// exec-context contributor build-out — see plan.md) will - /// activate. When true, the Agent-job install/download must - /// fire so that `exec-context-manual.js` is present. - /// - /// Populated at construction by `collect_extensions` using the - /// shared `manual_contributor_will_activate` predicate so this - /// stays in lock-step with the contributor's `should_activate`. - pub exec_context_manual_active: bool, - /// Whether the Pipeline-context contributor (Stage 2 of the - /// exec-context contributor build-out — see plan.md) will - /// activate. When true, the Agent-job install/download must - /// fire so that `exec-context-pipeline.js` is present. - /// - /// Populated at construction by `collect_extensions` using the - /// shared `pipeline_contributor_will_activate` predicate so this - /// stays in lock-step with the contributor's `should_activate`. - pub exec_context_pipeline_active: bool, - /// Whether the CI-push-context contributor (Stage 3 of the - /// exec-context contributor build-out — see plan.md) will - /// activate. Default-off opt-in feature; when true the - /// install/download must fire so that - /// `exec-context-ci-push.js` is present. - pub exec_context_ci_push_active: bool, - /// Whether the Workitem-context contributor (Stage 4 of the - /// exec-context contributor build-out — see plan.md) will - /// activate. Activates whenever the PR contributor activates - /// unless explicitly disabled. **Crosses an untrusted-prose - /// boundary** — see workitem.rs. - pub exec_context_workitem_active: bool, - /// Whether the Schedule-context contributor (Stage 5 of the - /// exec-context contributor build-out — see plan.md) will - /// activate. Opt-in (default OFF). - pub exec_context_schedule_active: bool, - /// Whether the PR-checks extension (Stage 6 of the build-out — - /// see plan.md) will activate. Opt-in (default OFF) AND - /// requires the PR contributor to activate. - pub exec_context_pr_checks_active: bool, - /// Whether the Repo-context contributor (Stage 7 of the - /// build-out — see plan.md) will activate. Always-on capability, - /// default OFF (opt-in). - pub exec_context_repo_active: bool, - /// Whether the safe-outputs approval-summary step will run at the - /// end of the Agent job. True whenever the workflow enables any - /// safe-output tool. When true the Agent-job install/download must - /// fire so that `approval-summary.js` is present for the - /// end-of-job render step (emitted by `build_agent_job`). - pub safe_outputs_summary_active: bool, - /// Whether GitHub App-backed Copilot auth is configured - /// (`engine.github-app-token`, issue #1316). When true the Agent-job - /// install/download must fire so that `github-app-token.js` is present for - /// the mint (and revoke) steps that `build_agent_job` emits immediately - /// around the Copilot run. Mirrors `safe_outputs_summary_active`: the - /// consuming steps are emitted by `build_agent_job`, not this extension, so - /// the flag drives the shared bundle download — the builder never has to - /// inspect emitted steps to decide whether to download. - pub github_app_token_active: bool, - /// Whether `create-pull-request` is configured (issue #1413). When true the - /// Agent-job install/download must fire so that `prepare-pr-base.js` is - /// present for the base-ref prepare step that `build_agent_job` emits before - /// the Copilot run. Mirrors `github_app_token_active`: the consuming step is - /// emitted by `build_agent_job`, not this extension, so the flag drives the - /// shared bundle download. - pub prepare_pr_base_active: bool, - /// Whether any user-defined stdio MCP server configures `azure-auth`. - /// Drives Agent-job bundle delivery for `azure-wif-refresh.js`. - pub azure_mcp_auth_active: bool, /// PR trigger config required to build `PR_SYNTH_SPEC`. `Some(_)` /// is the single source of truth for "synthetic-from-ci path is /// active for this agent" — `is_some()` replaces what used to be a @@ -1156,25 +1081,9 @@ impl CompilerExtension for AdoScriptExtension { // ─── Agent job ───────────────────────────────────────── let mut agent_prepare_steps: Vec = Vec::new(); let import_active = self.runtime_imports_active(); - if import_active - || self.exec_context_pr_active - || self.exec_context_manual_active - || self.exec_context_pipeline_active - || self.exec_context_ci_push_active - || self.exec_context_workitem_active - || self.exec_context_schedule_active - || self.exec_context_pr_checks_active - || self.exec_context_repo_active - || self.safe_outputs_summary_active - || self.github_app_token_active - || self.prepare_pr_base_active - || self.azure_mcp_auth_active - { - agent_prepare_steps - .extend(install_and_download_steps_typed(self.supply_chain.as_ref())); - if import_active { - agent_prepare_steps.push(resolver_step_typed()); - } + agent_prepare_steps.extend(install_and_download_steps_typed(self.supply_chain.as_ref())); + if import_active { + agent_prepare_steps.push(resolver_step_typed()); } // ─── Agent-job condition contribution ────────────────── @@ -1365,18 +1274,6 @@ mod tests { pr_filters: pr, pipeline_filters: pipeline, inlined_imports: inlined, - exec_context_pr_active: false, - exec_context_manual_active: false, - exec_context_pipeline_active: false, - exec_context_ci_push_active: false, - exec_context_workitem_active: false, - exec_context_schedule_active: false, - exec_context_pr_checks_active: false, - exec_context_repo_active: false, - safe_outputs_summary_active: false, - github_app_token_active: false, - prepare_pr_base_active: false, - azure_mcp_auth_active: false, pr_trigger_for_synth: None, supply_chain: None, } @@ -1445,18 +1342,6 @@ mod tests { pr_filters: None, pipeline_filters: None, inlined_imports: true, - exec_context_pr_active: false, - exec_context_manual_active: false, - exec_context_pipeline_active: false, - exec_context_ci_push_active: false, - exec_context_workitem_active: false, - exec_context_schedule_active: false, - exec_context_pr_checks_active: false, - exec_context_repo_active: false, - safe_outputs_summary_active: false, - github_app_token_active: false, - prepare_pr_base_active: false, - azure_mcp_auth_active: false, pr_trigger_for_synth: Some(PrTriggerConfig { branches: Some(BranchFilter { include: vec!["main".into()], @@ -1505,18 +1390,6 @@ mod tests { pr_filters: Some(filters), pipeline_filters: None, inlined_imports: true, - exec_context_pr_active: false, - exec_context_manual_active: false, - exec_context_pipeline_active: false, - exec_context_ci_push_active: false, - exec_context_workitem_active: false, - exec_context_schedule_active: false, - exec_context_pr_checks_active: false, - exec_context_repo_active: false, - safe_outputs_summary_active: false, - github_app_token_active: false, - prepare_pr_base_active: false, - azure_mcp_auth_active: false, pr_trigger_for_synth: Some(PrTriggerConfig { branches: Some(BranchFilter { include: vec!["main".into()], @@ -2090,14 +1963,13 @@ mod tests { } #[test] - fn declarations_agent_prepare_download_fires_when_only_prepare_pr_base_active() { - let mut ext = ext_with(None, None, true); - ext.prepare_pr_base_active = true; + fn declarations_agent_prepare_always_stages_bundle() { + let ext = ext_with(None, None, true); let fm: FrontMatter = serde_yaml::from_str("name: t\ndescription: t").unwrap(); let ctx = CompileContext::for_test(&fm); let steps = ext.declarations(&ctx).unwrap().agent_prepare_steps; - // Install + download fire (so prepare-pr-base.js is staged), but no - // runtime-import resolver (inlined_imports: true). + // The invoker is required by every Agent job, so install + download + // fire even when no other ado-script consumer is active. assert_eq!(steps.len(), 2, "install + download only"); assert!(matches!(&steps[0], Step::Task(t) if t.task == "UseNode@1")); assert!( @@ -2266,18 +2138,6 @@ mod tests { pr_filters: pr, pipeline_filters: pipeline, inlined_imports: true, - exec_context_pr_active: false, - exec_context_manual_active: false, - exec_context_pipeline_active: false, - exec_context_ci_push_active: false, - exec_context_workitem_active: false, - exec_context_schedule_active: false, - exec_context_pr_checks_active: false, - exec_context_repo_active: false, - safe_outputs_summary_active: false, - github_app_token_active: false, - prepare_pr_base_active: false, - azure_mcp_auth_active: false, pr_trigger_for_synth: Some(PrTriggerConfig { branches: Some(BranchFilter { include: vec!["main".into()], @@ -2740,30 +2600,19 @@ mod tests { // ── Typed-IR declarations (port-ado-script) ───────────────────── - /// `declarations()` returns empty step lists when neither - /// runtime-import nor exec-context-pr nor any gate / synth path - /// is active. + /// Setup remains empty when no gate / synth path is active, while Agent + /// preparation always stages the Copilot invoker bundle. #[test] - fn declarations_empty_when_nothing_active() { + fn declarations_stages_agent_bundle_when_nothing_else_active() { let ext = ext_with(None, None, true); let fm: FrontMatter = serde_yaml::from_str("name: t\ndescription: t").unwrap(); let ctx = CompileContext::for_test(&fm); let decl = ext.declarations(&ctx).unwrap(); assert!(decl.setup_steps.is_empty()); - assert!(decl.agent_prepare_steps.is_empty()); - } - - #[test] - fn declarations_agent_prepare_download_fires_for_azure_mcp_auth() { - let mut ext = ext_with(None, None, true); - ext.azure_mcp_auth_active = true; - let fm: FrontMatter = serde_yaml::from_str("name: t\ndescription: t").unwrap(); - let ctx = CompileContext::for_test(&fm); - let steps = ext.declarations(&ctx).unwrap().agent_prepare_steps; - assert_eq!(steps.len(), 2, "install + download only"); - assert!(matches!(&steps[0], Step::Task(t) if t.task == "UseNode@1")); + assert_eq!(decl.agent_prepare_steps.len(), 2, "install + download"); + assert!(matches!(&decl.agent_prepare_steps[0], Step::Task(t) if t.task == "UseNode@1")); assert!( - matches!(&steps[1], Step::Bash(b) if b.display_name.contains("Download ado-aw scripts")) + matches!(&decl.agent_prepare_steps[1], Step::Bash(b) if b.display_name.contains("Download ado-aw scripts")) ); } @@ -2816,18 +2665,6 @@ mod tests { pr_filters: None, pipeline_filters: None, inlined_imports: true, - exec_context_pr_active: false, - exec_context_manual_active: false, - exec_context_pipeline_active: false, - exec_context_ci_push_active: false, - exec_context_workitem_active: false, - exec_context_schedule_active: false, - exec_context_pr_checks_active: false, - exec_context_repo_active: false, - safe_outputs_summary_active: false, - github_app_token_active: false, - prepare_pr_base_active: false, - azure_mcp_auth_active: false, pr_trigger_for_synth: Some(PrTriggerConfig { branches: Some(BranchFilter { include: vec!["main".into()], diff --git a/src/compile/extensions/exec_context/mod.rs b/src/compile/extensions/exec_context/mod.rs index 82d949013..4d1e5bf93 100644 --- a/src/compile/extensions/exec_context/mod.rs +++ b/src/compile/extensions/exec_context/mod.rs @@ -55,118 +55,6 @@ use repo::RepoContextContributor; use schedule::ScheduleContextContributor; use workitem::WorkitemContextContributor; -/// Returns `true` iff the PR-context contributor will activate for the -/// given front matter. Shared between `ExecContextExtension::new` (for -/// its own `any_contributor_active` precomputation) and -/// `collect_extensions` (which passes it to `AdoScriptExtension` so -/// the Agent-job install/download fires whenever the bundle is needed). -/// -/// MAINTENANCE: this MUST match `PrContextContributor::should_activate` -/// (in `pr.rs`). The duplication is intentional — `should_activate` -/// takes a `CompileContext` that includes both front matter and target, -/// while this helper only needs the front matter (because `target` is -/// not relevant to PR activation today). -pub fn pr_contributor_will_activate(front_matter: &FrontMatter) -> bool { - // Borrow the embedded config when present; fall back to a stack- - // local default. Avoids the per-call clone — this helper is called - // on every `collect_extensions` invocation, which is hot during - // compile. - let default_cfg = ExecutionContextConfig::default(); - let cfg = front_matter - .execution_context - .as_ref() - .unwrap_or(&default_cfg); - pr_contributor_will_activate_with_cfg(cfg, front_matter) -} - -/// Returns `true` iff the Manual-context contributor will activate -/// for the given front matter. Shared between `ExecContextExtension::new` -/// (for its own `any_contributor_active` aggregate) and -/// `collect_extensions` (which passes it to `AdoScriptExtension` so -/// the Agent-job install/download fires whenever the bundle is needed). -/// -/// MAINTENANCE: this MUST match -/// `ManualContextContributor::should_activate` (in `manual.rs`). -/// Tests in `tests::manual` exercise both paths. -pub fn manual_contributor_will_activate(front_matter: &FrontMatter) -> bool { - let default_cfg = ExecutionContextConfig::default(); - let cfg = front_matter - .execution_context - .as_ref() - .unwrap_or(&default_cfg); - manual_contributor_will_activate_with_cfg(cfg, front_matter) -} - -/// Returns `true` iff the Pipeline-context contributor will activate -/// for the given front matter. Same pattern as the helpers above. -/// -/// MAINTENANCE: this MUST match -/// `PipelineContextContributor::should_activate` (in `pipeline.rs`). -pub fn pipeline_contributor_will_activate(front_matter: &FrontMatter) -> bool { - let default_cfg = ExecutionContextConfig::default(); - let cfg = front_matter - .execution_context - .as_ref() - .unwrap_or(&default_cfg); - pipeline_contributor_will_activate_with_cfg(cfg, front_matter) -} - -/// Returns `true` iff the CI-push-context contributor will activate -/// for the given front matter. Purely config-driven (opt-in, -/// default OFF). -pub fn ci_push_contributor_will_activate(front_matter: &FrontMatter) -> bool { - let default_cfg = ExecutionContextConfig::default(); - let cfg = front_matter - .execution_context - .as_ref() - .unwrap_or(&default_cfg); - ci_push_contributor_will_activate_with_cfg(cfg, front_matter) -} - -/// Returns `true` iff the Workitem contributor will activate. -/// PR-linked mode only — depends on the PR trigger being configured. -pub fn workitem_contributor_will_activate(front_matter: &FrontMatter) -> bool { - let default_cfg = ExecutionContextConfig::default(); - let cfg = front_matter - .execution_context - .as_ref() - .unwrap_or(&default_cfg); - workitem_contributor_will_activate_with_cfg(cfg, front_matter) -} - -/// Returns `true` iff the Schedule contributor will activate. Opt-in -/// (default OFF) AND requires `on.schedule` to be declared. -pub fn schedule_contributor_will_activate(front_matter: &FrontMatter) -> bool { - let default_cfg = ExecutionContextConfig::default(); - let cfg = front_matter - .execution_context - .as_ref() - .unwrap_or(&default_cfg); - schedule_contributor_will_activate_with_cfg(cfg, front_matter) -} - -/// Returns `true` iff the PR-checks extension will activate. Opt-in -/// (default OFF) AND requires the PR contributor to activate. -pub fn pr_checks_contributor_will_activate(front_matter: &FrontMatter) -> bool { - let default_cfg = ExecutionContextConfig::default(); - let cfg = front_matter - .execution_context - .as_ref() - .unwrap_or(&default_cfg); - pr_checks_contributor_will_activate_with_cfg(cfg, front_matter) -} - -/// Returns `true` iff the Repo contributor will activate. Pure -/// config-driven (opt-in, default OFF). -pub fn repo_contributor_will_activate(front_matter: &FrontMatter) -> bool { - let default_cfg = ExecutionContextConfig::default(); - let cfg = front_matter - .execution_context - .as_ref() - .unwrap_or(&default_cfg); - repo_contributor_will_activate_with_cfg(cfg, front_matter) -} - /// Variant that takes the resolved `ExecutionContextConfig` explicitly. /// Used by [`ExecContextExtension::new`] so its internal /// `any_contributor_active` precomputation tracks the config it was diff --git a/src/compile/extensions/exec_context/pr.rs b/src/compile/extensions/exec_context/pr.rs index 21d50c44e..bff38c33d 100644 --- a/src/compile/extensions/exec_context/pr.rs +++ b/src/compile/extensions/exec_context/pr.rs @@ -138,13 +138,6 @@ impl ContextContributor for PrContextContributor { } fn should_activate(&self, ctx: &CompileContext) -> bool { - // MAINTENANCE: this MUST stay in lock-step with - // `super::pr_contributor_will_activate` (the shared helper used - // by `collect_extensions` to populate - // `AdoScriptExtension::exec_context_pr_active`). The divergence- - // trap tests in `super::tests` exercise the helper path; this - // method is the runtime-context-aware version used by the - // declarations path. if ctx.front_matter.pr_trigger().is_none() { return false; } diff --git a/src/compile/extensions/mod.rs b/src/compile/extensions/mod.rs index a40aca34a..2970f2c1a 100644 --- a/src/compile/extensions/mod.rs +++ b/src/compile/extensions/mod.rs @@ -689,12 +689,7 @@ pub use ado_aw_marker::AdoAwMarkerExtension; pub(crate) use ado_aw_marker::APPEND_AW_INFO_FIELD; pub use ado_script::AdoScriptExtension; pub use azure_cli::AzureCliExtension; -pub use exec_context::{ - ExecContextExtension, ci_push_contributor_will_activate, manual_contributor_will_activate, - pipeline_contributor_will_activate, pr_checks_contributor_will_activate, - pr_contributor_will_activate, repo_contributor_will_activate, - schedule_contributor_will_activate, workitem_contributor_will_activate, -}; +pub use exec_context::ExecContextExtension; pub use github::GitHubExtension; pub use safe_outputs::SafeOutputsExtension; @@ -774,59 +769,6 @@ pub fn collect_extensions(front_matter: &FrontMatter) -> Vec { pr_filters: front_matter.pr_filters().cloned(), pipeline_filters: front_matter.pipeline_filters().cloned(), inlined_imports: front_matter.inlined_imports, - // Tell the ado-script extension whether the PR-context - // contributor will activate so it can fire the Agent-job - // install/download even when `inlined-imports: true` (no - // import.js needed). The two extensions stay loosely - // coupled: ExecContextExtension owns invoking the bundle; - // AdoScriptExtension owns installing it. Shared helper - // keeps the activation predicate in lock-step. - exec_context_pr_active: pr_contributor_will_activate(front_matter), - // Same loose-coupling pattern for the Manual contributor - // (Stage 1 of the exec-context contributor build-out — - // see plan.md). Activates whenever any `parameters:` - // block is declared and the contributor isn't explicitly - // disabled. - exec_context_manual_active: manual_contributor_will_activate(front_matter), - // Same loose-coupling pattern for the Pipeline contributor - // (Stage 2 of the exec-context contributor build-out — - // see plan.md). Activates whenever `on.pipeline` is - // configured and the contributor isn't explicitly - // disabled. - exec_context_pipeline_active: pipeline_contributor_will_activate(front_matter), - // CI-push contributor (Stage 3 — opt-in, default OFF). - exec_context_ci_push_active: ci_push_contributor_will_activate(front_matter), - // Workitem contributor (Stage 4 — PR-linked mode only). - // Activates whenever the PR contributor activates and - // workitem isn't explicitly disabled. - exec_context_workitem_active: workitem_contributor_will_activate(front_matter), - // Schedule contributor (Stage 5 — opt-in, default OFF). - exec_context_schedule_active: schedule_contributor_will_activate(front_matter), - // PR-checks extension (Stage 6 — opt-in, default OFF). - exec_context_pr_checks_active: pr_checks_contributor_will_activate(front_matter), - // Repo contributor (Stage 7 — opt-in, default OFF, no - // bearer / no REST, pure git). - exec_context_repo_active: repo_contributor_will_activate(front_matter), - // True whenever any safe-output tool is enabled — drives the - // Agent-job bundle install/download so `approval-summary.js` - // is present for the end-of-job render step that - // `build_agent_job` emits. MUST use the same predicate as that - // step (see `FrontMatter::has_any_safe_output_tool`). - safe_outputs_summary_active: front_matter.has_any_safe_output_tool(), - // True when `engine.github-app-token` is configured — drives the - // Agent-job bundle install/download so `github-app-token.js` is - // present for the mint/revoke steps that `build_agent_job` emits - // around the Copilot run. Same loose-coupling pattern as - // `safe_outputs_summary_active`: the consuming steps live in - // `build_agent_job`, not this extension. - github_app_token_active: front_matter.engine.github_app_token().is_some(), - // True when `create-pull-request` is configured (issue #1413) — - // drives the Agent-job bundle download so `prepare-pr-base.js` - // is present for the base-ref prepare step `build_agent_job` - // emits before the Copilot run. Same loose-coupling pattern as - // `github_app_token_active`. - prepare_pr_base_active: front_matter.create_pr_config().is_some(), - azure_mcp_auth_active: front_matter.has_azure_authenticated_mcp_servers(), pr_trigger_for_synth, supply_chain: front_matter.supply_chain().cloned(), } diff --git a/src/compile/types.rs b/src/compile/types.rs index 08eb3afe4..219d8d4d4 100644 --- a/src/compile/types.rs +++ b/src/compile/types.rs @@ -1626,15 +1626,6 @@ impl FrontMatter { servers } - pub fn has_azure_authenticated_mcp_servers(&self) -> bool { - self.mcp_servers.values().any(|config| { - matches!( - config, - McpConfig::WithOptions(options) - if options.enabled.unwrap_or(true) && options.azure_auth.is_some() - ) - }) - } } /// Compile-time source for a remote reusable import. @@ -2115,14 +2106,7 @@ impl FrontMatter { /// Whether the workflow enables **any** safe-output tool. /// - /// Single source of truth for the safe-outputs-summary feature gate: it - /// drives BOTH the ado-script bundle download - /// (`AdoScriptExtension::safe_outputs_summary_active`, set in - /// `collect_extensions`) and the end-of-Agent-job render step emission - /// (`build_agent_job`). Both call sites MUST go through this so the bundle - /// is downloaded iff the step that runs it is emitted — a drift between two - /// independent copies of this predicate would make the step invoke a bundle - /// that was never downloaded. + /// Single source of truth for the end-of-Agent-job summary step. pub fn has_any_safe_output_tool(&self) -> bool { self.safe_output_tool_names().next().is_some() || !self.custom_safe_output_tool_names().is_empty() diff --git a/src/engine.rs b/src/engine.rs index b101608b7..0b85a302a 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use anyhow::Result; +use serde::Serialize; use crate::compile::extensions::Declarations; use crate::compile::shell::{Binding, ShellScript}; @@ -88,7 +89,8 @@ pub const ADO_AW_MODEL_DETECTION_COPILOT: &str = "ADO_AW_MODEL_DETECTION_COPILOT /// Shared runtime pipeline-variable fallback for Copilot models. pub const ADO_AW_DEFAULT_MODEL_COPILOT: &str = "ADO_AW_DEFAULT_MODEL_COPILOT"; -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] pub(crate) enum RuntimeModelRole { Agent, Detection, @@ -103,6 +105,42 @@ impl RuntimeModelRole { } } +#[derive(Debug, Clone, Serialize)] +pub(crate) struct CopilotInvocationDocument { + pub(crate) schema_version: u32, + pub(crate) role: RuntimeModelRole, + pub(crate) command: String, + pub(crate) prompt_path: String, + pub(crate) mcp_config_path: Option, + pub(crate) args: Vec, + pub(crate) explicit_model: Option, + pub(crate) result_path: String, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct CopilotInvocationContext<'a> { + role: RuntimeModelRole, + prompt_path: &'a str, + mcp_config_path: Option<&'a str>, + result_path: &'a str, +} + +impl<'a> CopilotInvocationContext<'a> { + pub(crate) const fn new( + role: RuntimeModelRole, + prompt_path: &'a str, + mcp_config_path: Option<&'a str>, + result_path: &'a str, + ) -> Self { + Self { + role, + prompt_path, + mcp_config_path, + result_path, + } + } +} + /// Returns true when `key` is an allowlisted BYOM/BYOK provider env-var key that /// may carry ADO macro/runtime expressions in `engine.env`. Case-sensitive: see /// [`COPILOT_PROVIDER_EXPR_ENV_KEYS`] for why exact case is required. @@ -323,17 +361,24 @@ impl Engine { } } - /// Generate CLI arguments for the engine invocation. + /// Generate the legacy display form of the Copilot CLI arguments. + /// + /// Runtime execution consumes [`Self::args_with_config`] as a typed argv + /// vector; this joined form remains useful for diagnostics and focused + /// validation tests. + #[allow(dead_code)] pub fn args( &self, front_matter: &FrontMatter, extension_declarations: &[Declarations], ) -> Result { - self.args_with_config( - &front_matter.engine, - front_matter, - extension_declarations, - ) + Ok(self + .args_with_config( + &front_matter.engine, + front_matter, + extension_declarations, + )? + .join(" ")) } /// Generate CLI arguments using an explicit engine configuration while @@ -343,7 +388,7 @@ impl Engine { engine_config: &EngineConfig, front_matter: &FrontMatter, extension_declarations: &[Declarations], - ) -> Result { + ) -> Result> { match self { Engine::Copilot => copilot_args(engine_config, front_matter, extension_declarations), } @@ -437,55 +482,37 @@ impl Engine { } } - /// Generate the full AWF `--` command string for running the engine. - /// - /// Returns the content for the AWF `-- ''` argument, including the - /// binary path, prompt delivery flag, MCP config flag, and all CLI arguments. - /// The engine controls how the prompt is provided (e.g., `--prompt="$(cat ...)"` - /// for Copilot) and how MCP config is referenced. - /// - /// `prompt_path` is the path to the prompt file inside the AWF container. - /// `mcp_config_path` is optionally the path to the MCP config file - /// (Some for Agent job, None for Detection job which has no MCP). - pub fn invocation( + pub(crate) fn invocation_document( &self, front_matter: &FrontMatter, extension_declarations: &[Declarations], - prompt_path: &str, - mcp_config_path: Option<&str>, - ) -> Result { - let args = self.args(front_matter, extension_declarations)?; - self.invocation_with_args(&front_matter.engine, prompt_path, mcp_config_path, &args) + invocation: CopilotInvocationContext<'_>, + ) -> Result { + self.invocation_document_with_config( + &front_matter.engine, + front_matter, + extension_declarations, + invocation, + ) } - /// Generate a Detection-job invocation using an explicit engine configuration. - pub fn detection_invocation_with_config( + pub(crate) fn invocation_document_with_config( &self, engine_config: &EngineConfig, front_matter: &FrontMatter, extension_declarations: &[Declarations], - prompt_path: &str, - mcp_config_path: Option<&str>, - ) -> Result { + invocation: CopilotInvocationContext<'_>, + ) -> Result { let args = self.args_with_config(engine_config, front_matter, extension_declarations)?; - self.invocation_with_args(engine_config, prompt_path, mcp_config_path, &args) - } - - fn invocation_with_args( - &self, - engine_config: &EngineConfig, - prompt_path: &str, - mcp_config_path: Option<&str>, - args: &str, - ) -> Result { match self { Engine::Copilot => { let command_path = match engine_config.command() { Some(cmd) => { if !is_valid_command_path(cmd) { anyhow::bail!( - "engine.command '{}' contains invalid characters. \ - Only ASCII alphanumerics, '.', '_', '/', and '-' are allowed.", + "engine.command '{}' is invalid. Use a bare executable name or an \ + absolute container path with ASCII alphanumerics, '.', '_', '/', \ + and '-' and no dot, empty, or traversal segments.", cmd ); } @@ -493,12 +520,19 @@ impl Engine { } None => "/tmp/awf-tools/copilot".to_string(), }; - Ok(copilot_invocation( - &command_path, - prompt_path, - mcp_config_path, + if let Some(model) = engine_config.model() { + validate_model_name(model)?; + } + Ok(CopilotInvocationDocument { + schema_version: 1, + role: invocation.role, + command: command_path, + prompt_path: invocation.prompt_path.to_string(), + mcp_config_path: invocation.mcp_config_path.map(str::to_string), args, - )) + explicit_model: engine_config.model().map(str::to_string), + result_path: invocation.result_path.to_string(), + }) } } } @@ -653,7 +687,7 @@ fn copilot_args( engine_config: &EngineConfig, front_matter: &FrontMatter, extension_declarations: &[Declarations], -) -> Result { +) -> Result> { // Check if bash triggers --allow-all-tools. This happens when: // 1. Bash has an explicit wildcard entry (":*" or "*"), OR // 2. Bash is not specified at all (None) — ado-aw agents always run in AWF sandbox, @@ -709,7 +743,8 @@ fn copilot_args( agent ); } - params.push(format!("--agent {}", agent)); + params.push("--agent".to_string()); + params.push(agent.to_string()); } // Wire engine.api-target — sets the GHES/GHEC API endpoint hostname @@ -721,7 +756,8 @@ fn copilot_args( api_target ); } - params.push(format!("--api-target {}", api_target)); + params.push("--api-target".to_string()); + params.push(api_target.to_string()); } params.push("--disable-builtin-mcps".to_string()); @@ -732,13 +768,8 @@ fn copilot_args( } for tool in allowed_tools { - if tool.contains('(') || tool.contains(')') || tool.contains(' ') { - // Use double quotes - the copilot_params are embedded inside a single-quoted - // bash string in the AWF command, so single quotes would break quoting. - params.push(format!("--allow-tool \"{}\"", tool)); - } else { - params.push(format!("--allow-tool {}", tool)); - } + params.push("--allow-tool".to_string()); + params.push(tool); } // --allow-all-paths when edit is enabled — lets the agent write to any file path. @@ -754,13 +785,13 @@ fn copilot_args( params.push(arg.to_string()); } - Ok(params.join(" ")) + Ok(params) } fn validate_model_name(model: &str) -> Result<()> { - // Validate model names before they become task environment values or + // Validate model names before they become invocation-document values or // runtime-selected COPILOT_MODEL values. Keep this character set in sync - // with runtime_model_preamble. + // with the Copilot invoker. if model.is_empty() || !model .chars() @@ -872,7 +903,6 @@ fn copilot_env(engine_config: &EngineConfig) -> Result { ]; if let Some(model) = engine_config.model() { validate_model_name(model)?; - lines.push(format!("{COPILOT_MODEL}: \"{model}\"")); } else { add_runtime_model_env_lines(&mut lines, RuntimeModelRole::Agent); } @@ -988,7 +1018,6 @@ pub fn copilot_detection_env(engine_config: &EngineConfig) -> Result, - args: &str, -) -> String { - let mut parts = vec![ - command_path.to_string(), - format!("--prompt=\"$(cat {prompt_path})\""), - ]; - - if let Some(mcp_path) = mcp_config_path { - parts.push(format!("--additional-mcp-config @{mcp_path}")); - } - if !args.is_empty() { - parts.push(args.to_string()); - } - parts.join(" ") -} - -pub(crate) fn runtime_model_preamble(role: RuntimeModelRole) -> String { - let specific = role.specific_var(); - format!( - r#"ADO_AW_EFFECTIVE_MODEL="" -ADO_AW_MACRO_PREFIX='$' -ADO_AW_UNRESOLVED_SPECIFIC_MODEL="${{ADO_AW_MACRO_PREFIX}}({specific})" -ADO_AW_UNRESOLVED_DEFAULT_MODEL="${{ADO_AW_MACRO_PREFIX}}({ADO_AW_DEFAULT_MODEL_COPILOT})" -for ADO_AW_CANDIDATE_MODEL in "${{{specific}:-}}" "${{{ADO_AW_DEFAULT_MODEL_COPILOT}:-}}"; do - # Azure DevOps leaves an undefined macro as the literal $(VAR); treat that as unset. - if [ -z "$ADO_AW_CANDIDATE_MODEL" ] || [ "$ADO_AW_CANDIDATE_MODEL" = "$ADO_AW_UNRESOLVED_SPECIFIC_MODEL" ] || [ "$ADO_AW_CANDIDATE_MODEL" = "$ADO_AW_UNRESOLVED_DEFAULT_MODEL" ]; then - continue - fi - case "$ADO_AW_CANDIDATE_MODEL" in - # Keep this character set in sync with validate_model_name and EMIT_AW_INFO. - *[!A-Za-z0-9._:-]*) - echo "ERROR: runtime Copilot model from {specific}/{ADO_AW_DEFAULT_MODEL_COPILOT} contains invalid characters. Only ASCII alphanumerics, ., _, :, and - are allowed." >&2 - exit 1 - ;; - esac - ADO_AW_EFFECTIVE_MODEL="$ADO_AW_CANDIDATE_MODEL" - break -done -if [ -n "$ADO_AW_EFFECTIVE_MODEL" ]; then - export COPILOT_MODEL="$ADO_AW_EFFECTIVE_MODEL" -else - unset COPILOT_MODEL -fi"# - ) -} - #[cfg(test)] mod tests { use super::{ ADO_AW_DEFAULT_MODEL_COPILOT, ADO_AW_MODEL_AGENT_COPILOT, ADO_AW_MODEL_DETECTION_COPILOT, - COPILOT_MODEL, Engine, GITHUB_APP_TOKEN_VAR, copilot_byom_active, - copilot_byom_credential_keys, copilot_detection_env, copilot_provider_env, get_engine, - github_app_token_secrecy_advisory, github_token_source_var, normalize_version_tag, - validate_engine_feature_support, + COPILOT_MODEL, CopilotInvocationContext, Engine, GITHUB_APP_TOKEN_VAR, RuntimeModelRole, + copilot_byom_active, copilot_byom_credential_keys, copilot_detection_env, + copilot_provider_env, get_engine, github_app_token_secrecy_advisory, + github_token_source_var, normalize_version_tag, validate_engine_feature_support, }; - #[cfg(unix)] - use super::{RuntimeModelRole, runtime_model_preamble}; use crate::compile::{ extensions::{CompileContext, CompilerExtension, Declarations, collect_extensions}, parse_markdown, }; - #[cfg(unix)] - use std::process::Command; fn declarations_for(fm: &crate::compile::types::FrontMatter) -> Vec { let extensions = collect_extensions(fm); @@ -1488,23 +1460,6 @@ mod tests { .collect() } - #[cfg(unix)] - fn run_runtime_model_preamble( - role: RuntimeModelRole, - envs: &[(&str, &str)], - ) -> std::process::Output { - let script = format!( - "{}\nprintf '%s' \"$ADO_AW_EFFECTIVE_MODEL\"", - runtime_model_preamble(role) - ); - let mut command = Command::new("bash"); - command.arg("-c").arg(script).env_clear(); - for (key, value) in envs { - command.env(key, value); - } - command.output().expect("bash should run") - } - #[test] fn copilot_engine_command() { assert_eq!(Engine::Copilot.command(), "copilot"); @@ -1533,67 +1488,92 @@ mod tests { .unwrap(); assert!(!params.contains("--model")); let env = Engine::Copilot.env(&front_matter.engine).unwrap(); - assert!(env.contains("COPILOT_MODEL: \"gpt-5\""), "{env}"); + assert!(!env.contains(COPILOT_MODEL), "{env}"); + let invocation = Engine::Copilot + .invocation_document( + &front_matter, + &declarations_for(&front_matter), + CopilotInvocationContext::new( + RuntimeModelRole::Agent, + "/tmp/prompt.md", + None, + "/tmp/result.json", + ), + ) + .unwrap(); + assert_eq!(invocation.explicit_model.as_deref(), Some("gpt-5")); } #[test] - fn copilot_invocation_does_not_embed_runtime_model_resolution() { + fn copilot_invocation_document_defers_runtime_model_resolution() { let (front_matter, _) = parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); let invocation = Engine::Copilot - .invocation( + .invocation_document( &front_matter, &declarations_for(&front_matter), - "/tmp/prompt.md", - Some("/tmp/mcp.json"), + CopilotInvocationContext::new( + RuntimeModelRole::Agent, + "/tmp/prompt.md", + Some("/tmp/mcp.json"), + "/tmp/result.json", + ), ) .unwrap(); - assert!(!invocation.contains("ADO_AW_MODEL_AGENT_COPILOT")); - assert!(!invocation.contains("ADO_AW_DEFAULT_MODEL_COPILOT")); - assert!(!invocation.contains("--model")); - assert!(!invocation.contains("$(ADO_AW_MODEL_AGENT_COPILOT)")); - assert!(!invocation.contains("$(ADO_AW_DEFAULT_MODEL_COPILOT)")); + assert_eq!(invocation.role, RuntimeModelRole::Agent); + assert_eq!(invocation.explicit_model, None); + assert_eq!( + invocation.mcp_config_path.as_deref(), + Some("/tmp/mcp.json") + ); + assert!(!invocation.args.iter().any(|arg| arg.starts_with("--model"))); } #[test] - fn copilot_invocation_keeps_explicit_model_static() { + fn copilot_invocation_document_keeps_explicit_model_static() { let (front_matter, _) = parse_markdown( "---\nname: test\ndescription: test\nengine:\n id: copilot\n model: gpt-5\n---\n", ) .unwrap(); let invocation = Engine::Copilot - .invocation( + .invocation_document( &front_matter, &declarations_for(&front_matter), - "/tmp/prompt.md", - None, + CopilotInvocationContext::new( + RuntimeModelRole::Agent, + "/tmp/prompt.md", + None, + "/tmp/result.json", + ), ) .unwrap(); - assert!(!invocation.contains("--model")); - assert!(!invocation.contains("ADO_AW_MODEL_AGENT_COPILOT")); - assert!(!invocation.contains("ADO_AW_EFFECTIVE_MODEL")); + assert_eq!(invocation.explicit_model.as_deref(), Some("gpt-5")); + assert!(!invocation.args.iter().any(|arg| arg.starts_with("--model"))); } #[test] - fn copilot_detection_invocation_uses_independent_runtime_model() { + fn copilot_detection_invocation_document_uses_independent_runtime_model() { let (front_matter, _) = parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); let invocation = Engine::Copilot - .detection_invocation_with_config( + .invocation_document_with_config( &front_matter.engine, &front_matter, &declarations_for(&front_matter), - "/tmp/threat.md", - None, + CopilotInvocationContext::new( + RuntimeModelRole::Detection, + "/tmp/threat.md", + None, + "/tmp/result.json", + ), ) .unwrap(); - assert!(!invocation.contains("ADO_AW_MODEL_DETECTION_COPILOT")); - assert!(!invocation.contains("ADO_AW_DEFAULT_MODEL_COPILOT")); - assert!(!invocation.contains("ADO_AW_MODEL_AGENT_COPILOT")); - assert!(!invocation.contains("--model")); + assert_eq!(invocation.role, RuntimeModelRole::Detection); + assert_eq!(invocation.explicit_model, None); + assert!(!invocation.args.iter().any(|arg| arg.starts_with("--model"))); let env = copilot_detection_env(&front_matter.engine).unwrap(); assert!( env.iter() @@ -1605,61 +1585,6 @@ mod tests { ); } - #[test] - #[cfg(unix)] - fn runtime_model_preamble_uses_role_specific_before_default() { - let output = run_runtime_model_preamble( - RuntimeModelRole::Agent, - &[ - (ADO_AW_MODEL_AGENT_COPILOT, "agent-model"), - (ADO_AW_DEFAULT_MODEL_COPILOT, "default-model"), - ], - ); - - assert!(output.status.success(), "{output:?}"); - assert_eq!(String::from_utf8(output.stdout).unwrap(), "agent-model"); - } - - #[test] - #[cfg(unix)] - fn runtime_model_preamble_uses_default_when_role_specific_missing() { - let output = run_runtime_model_preamble( - RuntimeModelRole::Detection, - &[(ADO_AW_DEFAULT_MODEL_COPILOT, "default-model")], - ); - - assert!(output.status.success(), "{output:?}"); - assert_eq!(String::from_utf8(output.stdout).unwrap(), "default-model"); - } - - #[test] - #[cfg(unix)] - fn runtime_model_preamble_treats_unexpanded_ado_macro_as_missing() { - let output = run_runtime_model_preamble( - RuntimeModelRole::Agent, - &[ - (ADO_AW_MODEL_AGENT_COPILOT, "$(ADO_AW_MODEL_AGENT_COPILOT)"), - (ADO_AW_DEFAULT_MODEL_COPILOT, "default-model"), - ], - ); - - assert!(output.status.success(), "{output:?}"); - assert_eq!(String::from_utf8(output.stdout).unwrap(), "default-model"); - } - - #[test] - #[cfg(unix)] - fn runtime_model_preamble_rejects_invalid_runtime_model() { - let output = run_runtime_model_preamble( - RuntimeModelRole::Detection, - &[(ADO_AW_MODEL_DETECTION_COPILOT, "gpt-5 && curl evil.example")], - ); - - assert!(!output.status.success(), "{output:?}"); - let stderr = String::from_utf8(output.stderr).unwrap(); - assert!(stderr.contains("invalid characters"), "{stderr}"); - } - #[test] fn engine_args_reject_model_flag() { for args in ["[--model, gpt-5]", "[--model=gpt-5]"] { @@ -1887,29 +1812,36 @@ mod tests { "---\nname: test\ndescription: test\nengine:\n id: copilot\n command: /usr/local/bin/my-copilot\n---\n", ).unwrap(); let result = Engine::Copilot - .invocation( + .invocation_document( &fm, &declarations_for(&fm), - "/tmp/prompt.md", - Some("/tmp/mcp.json"), + CopilotInvocationContext::new( + RuntimeModelRole::Agent, + "/tmp/prompt.md", + Some("/tmp/mcp.json"), + "/tmp/result.json", + ), ) .unwrap(); - assert_eq!(result.matches("/usr/local/bin/my-copilot ").count(), 1); - assert!(!result.contains("/tmp/awf-tools/copilot")); + assert_eq!(result.command, "/usr/local/bin/my-copilot"); } #[test] fn engine_command_default_uses_awf_path() { let (fm, _) = parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); let result = Engine::Copilot - .invocation( + .invocation_document( &fm, &declarations_for(&fm), - "/tmp/prompt.md", - Some("/tmp/mcp.json"), + CopilotInvocationContext::new( + RuntimeModelRole::Agent, + "/tmp/prompt.md", + Some("/tmp/mcp.json"), + "/tmp/result.json", + ), ) .unwrap(); - assert_eq!(result.matches("/tmp/awf-tools/copilot ").count(), 1); + assert_eq!(result.command, "/tmp/awf-tools/copilot"); } #[test] @@ -1917,14 +1849,22 @@ mod tests { let (fm, _) = parse_markdown( "---\nname: test\ndescription: test\nengine:\n id: copilot\n command: \"/tmp/copilot; rm -rf /\"\n---\n", ).unwrap(); - let result = - Engine::Copilot.invocation(&fm, &declarations_for(&fm), "/tmp/prompt.md", None); + let result = Engine::Copilot.invocation_document( + &fm, + &declarations_for(&fm), + CopilotInvocationContext::new( + RuntimeModelRole::Agent, + "/tmp/prompt.md", + None, + "/tmp/result.json", + ), + ); assert!(result.is_err()); assert!( result .unwrap_err() .to_string() - .contains("invalid characters") + .contains("is invalid") ); } @@ -1933,8 +1873,16 @@ mod tests { let (fm, _) = parse_markdown( "---\nname: test\ndescription: test\nengine:\n id: copilot\n command: \"/tmp/co'pilot\"\n---\n", ).unwrap(); - let result = - Engine::Copilot.invocation(&fm, &declarations_for(&fm), "/tmp/prompt.md", None); + let result = Engine::Copilot.invocation_document( + &fm, + &declarations_for(&fm), + CopilotInvocationContext::new( + RuntimeModelRole::Agent, + "/tmp/prompt.md", + None, + "/tmp/result.json", + ), + ); assert!(result.is_err()); } diff --git a/src/validate.rs b/src/validate.rs index a12fb5c7e..a3059478e 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -42,12 +42,24 @@ pub fn is_safe_path_segment(s: &str) -> bool { .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) } -/// Characters allowed in engine.command paths (absolute path chars only). -/// Prevents shell injection when the path is embedded in AWF single-quoted commands. +/// Validate an engine command as either a bare executable name or an absolute +/// container path with no empty, dot, or traversal segments. pub fn is_valid_command_path(s: &str) -> bool { - !s.is_empty() - && s.chars() + if s.is_empty() + || !s + .chars() .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '/' | '-')) + { + return false; + } + if !s.contains('/') { + return s != "." && s != ".."; + } + s.starts_with('/') + && !s.ends_with('/') + && s[1..] + .split('/') + .all(|segment| !segment.is_empty() && segment != "." && segment != "..") } /// Characters allowed in engine.agent and engine.model identifiers. @@ -1003,6 +1015,12 @@ mod tests { assert!(is_valid_command_path("/tmp/awf-tools/copilot")); assert!(is_valid_command_path("copilot")); assert!(is_valid_command_path("/usr/local/bin/my-tool_v2")); + assert!(!is_valid_command_path("bin/copilot")); + assert!(!is_valid_command_path(".")); + assert!(!is_valid_command_path("..")); + assert!(!is_valid_command_path("/tmp/../copilot")); + assert!(!is_valid_command_path("/tmp//copilot")); + assert!(!is_valid_command_path("/tmp/copilot/")); assert!(!is_valid_command_path("")); assert!(!is_valid_command_path("/tmp/copilot; rm -rf /")); assert!(!is_valid_command_path("/tmp/copilot'")); diff --git a/tests/awf-copilot-safeoutputs/run.sh b/tests/awf-copilot-safeoutputs/run.sh index b25c15a08..5710a6937 100644 --- a/tests/awf-copilot-safeoutputs/run.sh +++ b/tests/awf-copilot-safeoutputs/run.sh @@ -6,6 +6,7 @@ umask 077 : "${ADO_AW_BIN:?ADO_AW_BIN is required}" : "${AWF_BIN:?AWF_BIN is required}" : "${COPILOT_BIN:?COPILOT_BIN is required}" +: "${COPILOT_INVOKER_BUNDLE:?COPILOT_INVOKER_BUNDLE is required}" : "${AWF_VERSION:?AWF_VERSION is required}" : "${MCPG_VERSION:?MCPG_VERSION is required}" : "${ADO_AW_COPILOT_CLI_ARTIFACT_DIR:?ADO_AW_COPILOT_CLI_ARTIFACT_DIR is required}" @@ -71,6 +72,10 @@ for binary in "${ADO_AW_BIN}" "${AWF_BIN}" "${COPILOT_BIN}"; do exit 1 } done +[[ -f "${COPILOT_INVOKER_BUNDLE}" ]] || { + echo "Copilot invoker bundle is missing: ${COPILOT_INVOKER_BUNDLE}" >&2 + exit 1 +} MCP_GATEWAY_API_KEY="$(openssl rand -base64 45 | tr -d '/+=')" install -m 0755 "${ADO_AW_BIN}" "${TOOLS_DIR}/ado-aw" @@ -223,15 +228,41 @@ jq \ chmod 600 "${TOOLS_DIR}/mcp-config.json" install -m 0755 "${COPILOT_BIN}" "${TOOLS_DIR}/copilot" +mkdir -p /tmp/ado-aw-scripts/ado-script +install -m 0644 "${COPILOT_INVOKER_BUNDLE}" \ + /tmp/ado-aw-scripts/ado-script/copilot-invoker.js cat >"${TOOLS_DIR}/agent-prompt.md" <"${TOOLS_DIR}/copilot-invocation.json" +chmod 600 "${TOOLS_DIR}/copilot-invocation.json" readonly ALLOWED_DOMAINS="api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,config.edge.skype.com,copilot-proxy.githubusercontent.com,github.com,telemetry.enterprise.githubcopilot.com,*.copilot.github.com,*.githubcopilot.com" # shellcheck disable=SC2016 # AWF expands the engine command inside the sandbox. -readonly ENGINE_RUN='export NO_PROXY="${NO_PROXY:+$NO_PROXY,}awmg-mcpg"; export no_proxy="$NO_PROXY"; /tmp/awf-tools/copilot --prompt="$(cat /tmp/awf-tools/agent-prompt.md)" --additional-mcp-config @/tmp/awf-tools/mcp-config.json --model gpt-5-mini --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-tool safeoutputs --allow-all-paths' +readonly ENGINE_RUN='export NO_PROXY="${NO_PROXY:+$NO_PROXY,}awmg-mcpg"; export no_proxy="$NO_PROXY"; exec node /tmp/ado-aw-scripts/ado-script/copilot-invoker.js run /tmp/awf-tools/copilot-invocation.json' set +e "${AWF_BIN}" \ @@ -254,6 +285,15 @@ if [[ "${AWF_STATUS}" -ne 0 ]]; then exit "${AWF_STATUS}" fi +REQUESTED_MODEL="$( + node /tmp/ado-aw-scripts/ado-script/copilot-invoker.js \ + read-result "${TOOLS_DIR}/copilot-invocation-result.json" agent +)" +[[ "${REQUESTED_MODEL}" == "gpt-5-mini" ]] || { + echo "Unexpected requested model from invoker: ${REQUESTED_MODEL}" >&2 + exit 1 +} + NDJSON_PATH="${SAFE_OUTPUTS_DIR}/safe_outputs.ndjson" for _ in $(seq 1 30); do [[ -s "${NDJSON_PATH}" ]] && break diff --git a/tests/compiler_tests.rs b/tests/compiler_tests.rs index 2e2db3a15..b5080cf73 100644 --- a/tests/compiler_tests.rs +++ b/tests/compiler_tests.rs @@ -2049,33 +2049,31 @@ Call the noop tool exactly once. let detection = extract_job_block(&compiled, "Detection").expect("Detection job should exist"); assert!( - agent.contains( - "/tmp/awf-tools/copilot --prompt=\"$(cat /tmp/awf-tools/agent-prompt.md)\" \ - --additional-mcp-config @/tmp/awf-tools/mcp-config.json" - ), - "agent job should pass compiler-emitted MCP config to Copilot CLI: {agent}" + agent.contains(r#""prompt_path":"/tmp/awf-tools/agent-prompt.md""#) + && agent.contains(r#""mcp_config_path":"/tmp/awf-tools/mcp-config.json""#), + "agent invocation document should reference the prompt and MCP config: {agent}" ); assert!( - !agent.contains("--prompt \"$(cat "), - "agent job should not pass prompt as a separate option value: {agent}" + agent.contains("copilot-invoker.js run") + && !agent.contains("/tmp/awf-tools/copilot --prompt"), + "agent job should execute only the fixed invoker command: {agent}" ); assert!( - detection.contains( - "/tmp/awf-tools/copilot --prompt=\"$(cat \ - /tmp/awf-tools/threat-analysis-prompt.md)\"" - ), - "detection job should pass prompt using attached form: {detection}" + detection.contains(r#""prompt_path":"/tmp/awf-tools/threat-analysis-prompt.md""#), + "detection invocation document should reference the threat prompt: {detection}" ); assert!( - !detection.contains("--prompt \"$(cat "), - "detection job should not pass prompt as a separate option value: {detection}" + detection.contains("copilot-invoker.js run") + && !detection.contains("/tmp/awf-tools/copilot --prompt"), + "detection job should execute only the fixed invoker command: {detection}" ); assert!( agent.contains("--allow-all-tools"), "default unrestricted tools path should emit --allow-all-tools: {agent}" ); assert!( - !detection.contains("--additional-mcp-config"), + detection.contains(r#""mcp_config_path":null"#) + && !detection.contains(r#""mcp_config_path":"/tmp/awf-tools/mcp-config.json""#), "detection job should not receive the SafeOutputs MCP config: {detection}" ); assert!( @@ -2119,19 +2117,17 @@ fn test_runtime_import_frontmatter_prompt_uses_attached_copilot_prompt_flag() { "agent prompt should runtime-import the full markdown fixture: {compiled}" ); assert!( - agent.contains("/tmp/awf-tools/copilot --prompt=\"$(cat /tmp/awf-tools/agent-prompt.md)\""), - "agent job should pass prompt using attached form: {agent}" + agent.contains(r#""prompt_path":"/tmp/awf-tools/agent-prompt.md""#), + "agent invocation document should reference the resolved prompt file: {agent}" ); assert!( - detection.contains( - "/tmp/awf-tools/copilot --prompt=\"$(cat \ - /tmp/awf-tools/threat-analysis-prompt.md)\"" - ), - "detection job should pass prompt using attached form: {detection}" + detection.contains(r#""prompt_path":"/tmp/awf-tools/threat-analysis-prompt.md""#), + "detection invocation document should reference the threat prompt: {detection}" ); assert!( - !compiled.contains("--prompt \"$(cat "), - "compiled pipeline should not pass prompt as a separate option value: {compiled}" + compiled.contains("copilot-invoker.js run") + && !compiled.contains("/tmp/awf-tools/copilot --prompt"), + "compiled pipeline should use the fixed invoker command: {compiled}" ); exercise_attached_prompt_with_pinned_copilot_cli(&fixture); @@ -2171,19 +2167,19 @@ Call the noop tool exactly once. "restricted bash path should not emit --allow-all-tools: {agent}" ); assert!( - agent.contains("--allow-tool safeoutputs"), + agent.contains(r#""--allow-tool","safeoutputs""#), "restricted bash path must explicitly allow the SafeOutputs MCP server: {agent}" ); assert!( - agent.contains("--allow-tool \"shell(echo)\""), + agent.contains(r#""--allow-tool","shell(echo)""#), "restricted bash path must emit the configured bash allowlist: {agent}" ); assert!( - agent.contains("--agent my-custom-agent"), + agent.contains(r#""--agent","my-custom-agent""#), "engine.agent should flow through the compiled Copilot CLI invocation: {agent}" ); assert!( - agent.contains("--api-target api.example.com"), + agent.contains(r#""--api-target","api.example.com""#), "engine.api-target should flow through the compiled Copilot CLI invocation: {agent}" ); assert!( @@ -2191,7 +2187,7 @@ Call the noop tool exactly once. "engine.args should append additive Copilot CLI arguments: {agent}" ); assert!( - agent.contains("--additional-mcp-config @/tmp/awf-tools/mcp-config.json"), + agent.contains(r#""mcp_config_path":"/tmp/awf-tools/mcp-config.json""#), "restricted tools path should still use the compiler-emitted MCP config: {agent}" ); } @@ -2279,7 +2275,7 @@ fn permissions_read_enables_proxy_and_wrapped_az_without_mcp() { "displayName: Install az wrapper (ado-proxy)", "displayName: Detect Azure CLI on host (for AWF mount)", "--topology-attach \"awmg-ado-proxy\"", - "--allow-tool \"shell(az)\"", + r#""--allow-tool","shell(az)""#, ] { assert!( compiled.contains(required), @@ -2490,7 +2486,7 @@ fn test_fixture_azure_devops_mcp_compiled_output() { "MCPG config should have entrypointArgs field" ); assert!( - !compiled.contains("\"command\""), + !compiled.contains("\"command\": "), "MCPG config should NOT use command field" ); @@ -2604,7 +2600,7 @@ fn test_mcpg_config_container_based_mcp() { assert!(compiled.contains("/host/data:/app/data:ro")); assert!(compiled.contains("\"API_KEY\": \"test-key\"")); assert!(compiled.contains("\"tool_a\"")); - assert!(!compiled.contains("\"command\"")); + assert!(!compiled.contains("\"command\": ")); let _ = fs::remove_dir_all(&temp_dir); } @@ -2728,7 +2724,7 @@ fn test_mcpg_config_http_based_mcp() { assert!(compiled.contains("\"url\": \"https://mcp.dev.azure.com/myorg\"")); assert!(compiled.contains("\"X-MCP-Toolsets\": \"repos,wit\"")); assert!(compiled.contains("\"wit_get_work_item\"")); - assert!(!compiled.contains("\"command\"")); + assert!(!compiled.contains("\"command\": ")); let _ = fs::remove_dir_all(&temp_dir); } @@ -5310,8 +5306,8 @@ fn test_1es_compiled_output_is_valid_yaml() { "1ES output should contain SafeOutputs references" ); assert!( - compiled.contains("copilot --prompt="), - "1ES output should contain copilot invocation (engine_run substituted)" + compiled.contains("copilot-invoker.js run"), + "1ES output should contain the fixed Copilot invoker command" ); assert!( compiled.contains("threat-analysis"), @@ -5849,11 +5845,10 @@ fn extract_job_block<'a>(yaml: &'a str, name: &str) -> Option<&'a str> { Some(&yaml[start..end]) } -/// Per-job download placement: gate-only pipeline must put the download in -/// Setup and NOT in Agent. ADO jobs run on isolated VMs, so the gate's -/// install/download has to land in the same job as the gate step. +/// Gate-only pipelines stage the bundle in Setup for the gate and in both +/// Copilot jobs for the invoker. #[test] -fn test_gate_only_pipeline_downloads_bundle_in_setup_job_not_agent() { +fn test_gate_only_pipeline_downloads_bundle_in_all_consuming_jobs() { let yaml = compile_fixture("dedupe_gate_only.md"); let setup = extract_job_block(&yaml, "Setup").expect("Setup job should exist"); let agent = extract_job_block(&yaml, "Agent").expect("Agent job should exist"); @@ -5862,9 +5857,8 @@ fn test_gate_only_pipeline_downloads_bundle_in_setup_job_not_agent() { "Setup job is missing the script bundle download (gate consumer lives here)" ); assert!( - !agent.contains("Download ado-aw scripts"), - "Agent job should NOT have the script bundle download (gate-only, no runtime imports). \ - Agent block contents: {}", + agent.contains("Download ado-aw scripts"), + "Agent job must stage the Copilot invoker bundle. Agent block contents: {}", agent ); } @@ -5890,9 +5884,8 @@ fn test_imports_only_pipeline_downloads_bundle_in_agent_job_not_setup() { } } -/// Per-job download placement: when both gate and runtime imports are active, -/// the bundle is downloaded twice — once per consuming job. ADO's VM -/// isolation makes this correct architecture, not duplication waste. +/// When both gate and runtime imports are active, each isolated consuming job +/// stages the bundle: Setup, Agent, and Detection. #[test] fn test_both_features_active_downloads_bundle_in_both_jobs() { let yaml = compile_fixture("dedupe_both.md"); @@ -5908,23 +5901,19 @@ fn test_both_features_active_downloads_bundle_in_both_jobs() { ); assert_eq!( yaml.matches("Download ado-aw scripts").count(), - 2, - "Expected exactly two downloads — one per consuming job (Setup + Agent)" + 3, + "Expected exactly three downloads — Setup, Agent, and Detection" ); } -/// Per-job download placement: with neither gate nor runtime imports active, -/// no Node install or script-bundle download should appear anywhere. +/// Even with no gate or runtime imports, Agent and Detection stage the invoker. #[test] -fn test_neither_feature_active_emits_no_node_or_download_anywhere() { +fn test_neither_feature_active_stages_invoker_in_copilot_jobs() { let yaml = compile_fixture("dedupe_neither.md"); - assert!( - !yaml.contains("UseNode@1"), - "No UseNode@1 expected when neither gate nor runtime imports are active" - ); - assert!( - !yaml.contains("Download ado-aw scripts"), - "No script bundle download expected when neither gate nor runtime imports are active" + assert_eq!( + yaml.matches("Download ado-aw scripts").count(), + 2, + "Agent and Detection must each stage the invoker bundle" ); } @@ -5993,12 +5982,11 @@ fn test_node_runtime_install_orders_after_ado_script_so_user_version_wins() { ado-script idx = {ado_script_install_idx}, user idx = {user_runtime_install_idx}" ); - // Both downloads of ado-script.zip remain unaffected (still exactly one - // in the Agent job in this fixture — no filters, so no Setup-side download). + // Agent and Detection each stage ado-script.zip; no Setup download exists. assert_eq!( yaml.matches("Download ado-aw scripts").count(), - 1, - "Expected exactly one ado-script.zip download (Agent job only; no gate active)" + 2, + "Expected exactly two ado-script.zip downloads (Agent + Detection)" ); } @@ -8175,14 +8163,20 @@ safe-outputs: let agent = job_block(&compiled, "Agent"); let detection = job_block(&compiled, "Detection"); - assert!(agent.contains("COPILOT_MODEL: agent-model"), "{agent}"); + assert!( + agent.contains(r#""explicit_model":"agent-model""#), + "{agent}" + ); assert!(agent.contains("--reasoning-effort=high"), "{agent}"); assert!(!agent.contains("--model"), "{agent}"); - assert!(!agent.contains("COPILOT_MODEL: detection-model"), "{agent}"); + assert!( + !agent.contains(r#""explicit_model":"detection-model""#), + "{agent}" + ); assert!(!agent.contains("DETECTION_ENV"), "{agent}"); assert!( - detection.contains("COPILOT_MODEL: detection-model"), + detection.contains(r#""explicit_model":"detection-model""#), "{detection}" ); assert!(!detection.contains("--model"), "{detection}"); @@ -8237,8 +8231,17 @@ fn runtime_model_controls_compile_across_all_targets() { "{target}: missing Detection runtime model env mapping" ); assert!( - compiled.contains("export COPILOT_MODEL=\"$ADO_AW_EFFECTIVE_MODEL\""), - "{target}: runtime resolver must export COPILOT_MODEL" + compiled.contains("copilot-invoker.js run"), + "{target}: fixed Copilot invoker command must be emitted" + ); + assert!( + compiled.contains(r#""role":"agent""#) + && compiled.contains(r#""role":"detection""#), + "{target}: Agent and Detection invocation documents must be emitted" + ); + assert!( + !compiled.contains("ADO_AW_EFFECTIVE_MODEL"), + "{target}: runtime model shell resolver must be absent" ); assert!( !compiled.contains("--model"), @@ -8247,6 +8250,39 @@ fn runtime_model_controls_compile_across_all_targets() { } } +#[test] +fn runtime_model_control_resolves_after_prior_agent_step() { + let source = r###"--- +name: Runtime Model Set Variable +description: Runtime model task-scope resolution +steps: + - bash: | + echo "##vso[task.setvariable variable=ADO_AW_MODEL_AGENT_COPILOT]gpt-runtime" +safe-outputs: + threat-detection: false +--- + +## Agent +"###; + let (ok, compiled, stderr) = compile_inline_source("runtime-model-set-variable", source); + assert!(ok, "pipeline should compile: {stderr}"); + let agent = job_block(&compiled, "Agent"); + let producer = agent + .find("task.setvariable variable=ADO_AW_MODEL_AGENT_COPILOT") + .expect("model variable producer should be emitted"); + let consumer = agent + .find("Run copilot (AWF network isolated)") + .expect("Copilot invoker step should be emitted"); + assert!( + producer < consumer, + "trusted variable producer must run before the invoker task: {agent}" + ); + assert!( + agent.contains("ADO_AW_MODEL_AGENT_COPILOT: $(ADO_AW_MODEL_AGENT_COPILOT)"), + "invoker task must resolve the model through its typed env mapping: {agent}" + ); +} + #[test] fn threat_detection_disabled_preserves_outputs_artifacts_and_manual_review() { let source = r#"--- @@ -10167,10 +10203,8 @@ fn test_github_app_token_hyphenated_private_key_variable() { /// When another ado-script bundle feature is active in the Agent job (here a /// safe-output activates the approval-summary bundle download), the mint step -/// must NOT trigger a second bundle download in that job — it reuses the -/// already-staged bundle. Proven by a delta: adding `github-app-token` to an -/// otherwise-identical workflow adds exactly ONE bundle download (the -/// Detection job, which has no extension-prepare phase), never two. +/// must NOT trigger another bundle download in either Copilot job because both +/// already stage the invoker bundle. #[test] fn test_github_app_token_reuses_staged_bundle_in_agent() { fn count_downloads(compiled: &str) -> usize { @@ -10193,14 +10227,11 @@ fn test_github_app_token_reuses_staged_bundle_in_agent() { ); assert_github_app_token_wiring(&with); - // Adding github-app-token stages the bundle only in Detection (Agent - // reuses its already-staged copy), so the download count grows by exactly 1. + // Adding github-app-token reuses the always-staged Agent and Detection bundles. assert_eq!( count_downloads(&with), - count_downloads(&without) + 1, - "github-app-token must add exactly one bundle download (Detection), \ - proving the Agent job reuses its staged bundle rather than \ - double-downloading. without={}, with={}", + count_downloads(&without), + "github-app-token must not add bundle downloads. without={}, with={}", count_downloads(&without), count_downloads(&with), ); From a12235c29431627423be89c7ec2536251d73565b Mon Sep 17 00:00:00 2001 From: James Devine Date: Thu, 1 Oct 2026 12:56:58 +0100 Subject: [PATCH 13/24] fix(test): repair copilot invoker smoke fixtures Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- .../test/azure-wif-isolation.test.ts | 19 +++++++++++++++++-- scripts/ado-script/test/smoke.test.ts | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/ado-script/test/azure-wif-isolation.test.ts b/scripts/ado-script/test/azure-wif-isolation.test.ts index 0823661ea..d8ae17c2b 100644 --- a/scripts/ado-script/test/azure-wif-isolation.test.ts +++ b/scripts/ado-script/test/azure-wif-isolation.test.ts @@ -209,10 +209,18 @@ describe.skipIf(!awfEnabled)("Azure WIF real AWF boundary", () => { const workspace = join(directory, "workspace"); const temp = join(directory, "runner-temp"); const tools = join(directory, "tools"); + const awfTools = join(tools, "awf-tools"); + const adoScripts = join(tools, "ado-aw-scripts"); const home = join(directory, "home"); const auth = join(temp, "ado-aw-azure-auth", "fixture"); mkdirSync(workspace, { recursive: true }); mkdirSync(home); + mkdirSync(awfTools, { recursive: true }); + mkdirSync(join(adoScripts, "ado-script"), { recursive: true }); + copyFileSync( + resolve(testDir, "../copilot-invoker.js"), + join(adoScripts, "ado-script/copilot-invoker.js"), + ); mkdirSync(join(auth, "token.d"), { recursive: true }); chmodSync(join(temp, "ado-aw-azure-auth"), 0o700); chmodSync(auth, 0o700); @@ -224,15 +232,22 @@ describe.skipIf(!awfEnabled)("Azure WIF real AWF boundary", () => { const runStep = pipeline.jobs.find((job) => job.job === "Agent")?.steps .find((step) => step.bash?.includes("AWF_ARGS+=(--skip-pull --env-all)")); if (!runStep?.bash) throw new Error("compiled AWF invocation is missing"); + expect(runStep.bash).toContain( + "copilot-invoker.js run /tmp/awf-tools/copilot-invocation.json", + ); const capture = join(directory, "awf-args"); + const invocationResult = join(awfTools, "copilot-invocation-result.json"); writeFileSync(join(tools, "awf/awf"), `#!/bin/sh if [ "$1" = logs ]; then exit 0; fi printf '%s\\0' "$@" > '${capture}' +printf '%s\\n' '{"schema_version":1,"role":"agent","requested_model":null}' > '${invocationResult}' `, { mode: 0o755 }); const script = runStep.bash .replaceAll("$(Agent.TempDirectory)", temp) .replaceAll("$(Pipeline.Workspace)", tools) - .replaceAll("$(Build.SourcesDirectory)", workspace); + .replaceAll("$(Build.SourcesDirectory)", workspace) + .replaceAll("/tmp/awf-tools", awfTools) + .replaceAll("/tmp/ado-aw-scripts", adoScripts); const env: NodeJS.ProcessEnv = { PATH: process.env.PATH, HOME: home, @@ -261,7 +276,7 @@ printf '%s\\0' "$@" > '${capture}' const commandIndex = captured.indexOf("--"); expect(commandIndex).toBeGreaterThan(0); expect(captured[commandIndex + 1]).toContain( - "copilot-invoker.js run /tmp/awf-tools/copilot-invocation.json", + `copilot-invoker.js run ${join(awfTools, "copilot-invocation.json")}`, ); const args: string[] = []; for (let i = 0; i < commandIndex; i++) { diff --git a/scripts/ado-script/test/smoke.test.ts b/scripts/ado-script/test/smoke.test.ts index f95905657..f03b5c188 100644 --- a/scripts/ado-script/test/smoke.test.ts +++ b/scripts/ado-script/test/smoke.test.ts @@ -144,7 +144,7 @@ describe.skipIf(process.platform === "win32")("copilot-invoker.js smoke", () => writeFileSync( command, `#!/bin/sh -node -e 'require("node:fs").writeFileSync(process.env.CAPTURE_PATH, JSON.stringify({ argv: process.argv.slice(1), model: process.env.COPILOT_MODEL }))' "$@" +node -e 'require("node:fs").writeFileSync(process.env.CAPTURE_PATH, JSON.stringify({ argv: process.argv.slice(1), model: process.env.COPILOT_MODEL }))' -- "$@" exit 7 `, ); From 22768d55274ebbe62ed729e39233fa9bf07d1a08 Mon Sep 17 00:00:00 2001 From: James Devine Date: Thu, 1 Oct 2026 13:10:00 +0100 Subject: [PATCH 14/24] fix(engine): harden copilot invoker boundaries Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- .../src/copilot-invoker/index.test.ts | 92 ++++++++++++++++++- .../ado-script/src/copilot-invoker/index.ts | 24 ++++- src/engine.rs | 34 +------ 3 files changed, 115 insertions(+), 35 deletions(-) diff --git a/scripts/ado-script/src/copilot-invoker/index.test.ts b/scripts/ado-script/src/copilot-invoker/index.test.ts index 5b7e77e76..d04688f19 100644 --- a/scripts/ado-script/src/copilot-invoker/index.test.ts +++ b/scripts/ado-script/src/copilot-invoker/index.test.ts @@ -1,5 +1,5 @@ import { EventEmitter } from "node:events"; -import { mkdtempSync, readFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -7,6 +7,7 @@ import { describe, expect, it, vi } from "vitest"; import { buildChildEnvironment, buildCopilotArgs, + main, parseInvocationDocument, parseInvocationResult, resolveRequestedModel, @@ -70,7 +71,10 @@ describe("copilot invoker document", () => { [{ ...document(), role: "other" }, "must be 'agent' or 'detection'"], [{ ...document(), command: "copilot;sh" }, "field 'command' is invalid"], [{ ...document(), command: ".." }, "field 'command' is invalid"], + [{ ...document(), command: "bin/copilot" }, "field 'command' is invalid"], [{ ...document(), command: "/tmp/../copilot" }, "field 'command' is invalid"], + [{ ...document(), command: "/tmp//copilot" }, "field 'command' is invalid"], + [{ ...document(), command: "/tmp/copilot/" }, "field 'command' is invalid"], [{ ...document(), prompt_path: "relative.md" }, "field 'prompt_path' is invalid"], [{ ...document(), prompt_path: "/tmp/../prompt.md" }, "field 'prompt_path' is invalid"], [{ ...document(), result_path: "/" }, "field 'result_path' is invalid"], @@ -141,6 +145,12 @@ describe("argv and child environment", () => { ]); }); + it("rejects NUL bytes in prompt content", () => { + expect(() => buildCopilotArgs(document(), "before\0after")).toThrow( + "prompt contains an invalid NUL byte", + ); + }); + it("sets or removes only the child COPILOT_MODEL", () => { const original = { KEEP: "yes", COPILOT_MODEL: "old" }; expect(buildChildEnvironment(original, "selected")).toEqual({ @@ -152,6 +162,86 @@ describe("argv and child environment", () => { }); }); +describe("read-result command", () => { + it("writes the requested model after validating the result", async () => { + const directory = mkdtempSync(join(tmpdir(), "copilot-invoker-result-")); + const resultPath = join(directory, "result.json"); + writeFileSync( + resultPath, + JSON.stringify({ + schema_version: 1, + role: "agent", + requested_model: "gpt-test", + }), + ); + const write = vi.spyOn(process.stdout, "write").mockImplementation( + ((_chunk: unknown, callback?: (error?: Error | null) => void) => { + callback?.(); + return true; + }) as typeof process.stdout.write, + ); + + await expect(main(["read-result", resultPath, "agent"])).resolves.toBe(0); + expect(write).toHaveBeenCalledWith("gpt-test", expect.any(Function)); + write.mockRestore(); + }); + + it("rejects a result for the wrong role", async () => { + const directory = mkdtempSync(join(tmpdir(), "copilot-invoker-result-")); + const resultPath = join(directory, "result.json"); + writeFileSync( + resultPath, + JSON.stringify({ + schema_version: 1, + role: "agent", + requested_model: "gpt-test", + }), + ); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(main(["read-result", resultPath, "detection"])).resolves.toBe(1); + expect(error).toHaveBeenCalledWith( + "copilot-invoker: invocation result role 'agent' does not match expected role 'detection'", + ); + error.mockRestore(); + }); + + it("reports a missing or malformed result", async () => { + const directory = mkdtempSync(join(tmpdir(), "copilot-invoker-result-")); + const resultPath = join(directory, "missing.json"); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(main(["read-result", resultPath, "agent"])).resolves.toBe(1); + expect(error).toHaveBeenCalledWith(expect.stringContaining("copilot-invoker:")); + error.mockRestore(); + }); + + it("reports stdout write failures", async () => { + const directory = mkdtempSync(join(tmpdir(), "copilot-invoker-result-")); + const resultPath = join(directory, "result.json"); + writeFileSync( + resultPath, + JSON.stringify({ + schema_version: 1, + role: "agent", + requested_model: "gpt-test", + }), + ); + const write = vi.spyOn(process.stdout, "write").mockImplementation( + ((_chunk: unknown, callback?: (error?: Error | null) => void) => { + callback?.(new Error("EPIPE")); + return false; + }) as typeof process.stdout.write, + ); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(main(["read-result", resultPath, "agent"])).resolves.toBe(1); + expect(error).toHaveBeenCalledWith("copilot-invoker: EPIPE"); + write.mockRestore(); + error.mockRestore(); + }); +}); + describe("process lifecycle", () => { it("publishes the result before spawning and preserves the child exit code", async () => { const events: string[] = []; diff --git a/scripts/ado-script/src/copilot-invoker/index.ts b/scripts/ado-script/src/copilot-invoker/index.ts index da5ec7358..e6a6d4450 100644 --- a/scripts/ado-script/src/copilot-invoker/index.ts +++ b/scripts/ado-script/src/copilot-invoker/index.ts @@ -244,6 +244,9 @@ export function buildCopilotArgs( document: InvocationDocument, prompt: string, ): string[] { + if (prompt.includes("\0")) { + throw new Error("prompt contains an invalid NUL byte"); + } const args = [`--prompt=${prompt}`]; if (document.mcp_config_path !== null) { args.push("--additional-mcp-config", `@${document.mcp_config_path}`); @@ -256,6 +259,8 @@ export function buildChildEnvironment( env: NodeJS.ProcessEnv, requestedModel: string | null, ): NodeJS.ProcessEnv { + // AWF filters host-only credentials before launching the invoker. Preserve + // the remaining environment because Copilot providers and MCPs consume it. const childEnv = { ...env }; if (requestedModel === null) { delete childEnv.COPILOT_MODEL; @@ -338,7 +343,12 @@ export async function main(argv: string[]): Promise { `invocation result role '${result.role}' does not match expected role '${argv[2]}'`, ); } - process.stdout.write(result.requested_model ?? ""); + await new Promise((resolveWrite, rejectWrite) => { + process.stdout.write(result.requested_model ?? "", (error) => { + if (error) rejectWrite(error); + else resolveWrite(); + }); + }); return 0; } catch (error) { const message = error instanceof Error ? error.message : "unknown error"; @@ -364,7 +374,13 @@ export async function main(argv: string[]): Promise { const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; if (invokedPath === fileURLToPath(import.meta.url)) { - void main(process.argv.slice(2)).then((code) => { - process.exitCode = code; - }); + void main(process.argv.slice(2)) + .then((code) => { + process.exitCode = code; + }) + .catch((error: unknown) => { + const message = error instanceof Error ? error.message : "unknown error"; + console.error(`copilot-invoker: unexpected failure: ${message}`); + process.exitCode = 1; + }); } diff --git a/src/engine.rs b/src/engine.rs index 0b85a302a..92a9155d2 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -348,25 +348,16 @@ pub fn get_engine(engine_id: &str) -> Result { } impl Engine { - /// The default engine binary name (e.g., "copilot"). - /// - /// Currently scaffolding — the pipeline templates hard-code the binary path - /// (`/tmp/awf-tools/copilot`). This will be wired into template substitution - /// when additional engines are added. Can be overridden per-agent via - /// `engine.command` in front matter. - #[allow(dead_code)] + /// Test-only legacy display of the default engine binary name. + #[cfg(test)] pub fn command(&self) -> &str { match self { Engine::Copilot => "copilot", } } - /// Generate the legacy display form of the Copilot CLI arguments. - /// - /// Runtime execution consumes [`Self::args_with_config`] as a typed argv - /// vector; this joined form remains useful for diagnostics and focused - /// validation tests. - #[allow(dead_code)] + /// Test-only legacy display form of the Copilot CLI arguments. + #[cfg(test)] pub fn args( &self, front_matter: &FrontMatter, @@ -1460,23 +1451,6 @@ mod tests { .collect() } - #[test] - fn copilot_engine_command() { - assert_eq!(Engine::Copilot.command(), "copilot"); - } - - #[test] - fn copilot_engine_args() { - let (front_matter, _) = - parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); - let params = Engine::Copilot - .args(&front_matter, &declarations_for(&front_matter)) - .unwrap(); - // Default engine (copilot) lets the Copilot CLI choose its default model. - assert!(!params.contains("--model ")); - assert!(params.contains("--disable-builtin-mcps")); - } - #[test] fn copilot_engine_with_explicit_model() { let (front_matter, _) = parse_markdown( From 2274fef5ff6d40c94036ee86a30e0d1754013b13 Mon Sep 17 00:00:00 2001 From: James Devine Date: Sun, 4 Oct 2026 22:16:49 +0100 Subject: [PATCH 15/24] fix(compile): isolate Copilot control plane from AWF Split Copilot invocation into a host-private controller and sandbox-only runner, move authoritative model metadata outside sandbox-writable /tmp, and add tamper regressions across compiler and AWF contracts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- .github/workflows/copilot-cli-safeoutputs.yml | 9 +- AGENTS.md | 13 +- docs/ado-script.md | 129 ++++-- docs/engine.md | 30 +- scripts/ado-script/.gitignore | 3 +- scripts/ado-script/package.json | 9 +- .../src/__tests__/bundle-coverage.test.ts | 1 + .../src/copilot-controller/index.test.ts | 112 +++++ .../src/copilot-controller/index.ts | 68 +++ .../src/copilot-invoker/index.test.ts | 359 ---------------- .../ado-script/src/copilot-invoker/index.ts | 386 ------------------ .../src/copilot-runner/index.test.ts | 150 +++++++ .../ado-script/src/copilot-runner/index.ts | 118 ++++++ .../src/copilot-shared/protocol.test.ts | 245 +++++++++++ .../ado-script/src/copilot-shared/protocol.ts | 385 +++++++++++++++++ .../test/azure-wif-isolation.test.ts | 34 +- scripts/ado-script/test/smoke.test.ts | 39 +- src/compile/ado_bundle.rs | 18 +- src/compile/agentic_pipeline.rs | 189 +++++---- src/compile/extensions/ado_script.rs | 16 +- src/engine.rs | 57 ++- tests/awf-copilot-safeoutputs/run.sh | 53 ++- tests/compiler_tests.rs | 66 ++- 23 files changed, 1508 insertions(+), 981 deletions(-) create mode 100644 scripts/ado-script/src/copilot-controller/index.test.ts create mode 100644 scripts/ado-script/src/copilot-controller/index.ts delete mode 100644 scripts/ado-script/src/copilot-invoker/index.test.ts delete mode 100644 scripts/ado-script/src/copilot-invoker/index.ts create mode 100644 scripts/ado-script/src/copilot-runner/index.test.ts create mode 100644 scripts/ado-script/src/copilot-runner/index.ts create mode 100644 scripts/ado-script/src/copilot-shared/protocol.test.ts create mode 100644 scripts/ado-script/src/copilot-shared/protocol.ts diff --git a/.github/workflows/copilot-cli-safeoutputs.yml b/.github/workflows/copilot-cli-safeoutputs.yml index 71967b4b3..13eb68503 100644 --- a/.github/workflows/copilot-cli-safeoutputs.yml +++ b/.github/workflows/copilot-cli-safeoutputs.yml @@ -63,11 +63,12 @@ jobs: mcpg:MCPG_VERSION:src/compile/common.rs VERSIONS - - name: Build Copilot invoker bundle + - name: Build Copilot controller and runner bundles run: | set -euo pipefail npm --prefix scripts/ado-script ci - npm --prefix scripts/ado-script run build:copilot-invoker + npm --prefix scripts/ado-script run build:copilot-controller + npm --prefix scripts/ado-script run build:copilot-runner - name: Install compiler-pinned GitHub Copilot CLI run: | @@ -115,10 +116,12 @@ jobs: - name: Run handwritten AWF + Copilot + SafeOutputs contract env: ADO_AW_COPILOT_CLI_ARTIFACT_DIR: ${{ runner.temp }}/copilot-cli-safeoutputs + ADO_AW_COPILOT_CLI_CONTROL_DIR: ${{ runner.temp }}/copilot-cli-control ADO_AW_BIN: ${{ github.workspace }}/target/debug/ado-aw AWF_BIN: ${{ runner.temp }}/bin/awf COPILOT_BIN: ${{ runner.temp }}/bin/copilot - COPILOT_INVOKER_BUNDLE: ${{ github.workspace }}/scripts/ado-script/copilot-invoker.js + COPILOT_CONTROLLER_BUNDLE: ${{ github.workspace }}/scripts/ado-script/copilot-controller.js + COPILOT_RUNNER_BUNDLE: ${{ github.workspace }}/scripts/ado-script/copilot-runner.js AWF_VERSION: ${{ steps.versions.outputs.awf }} MCPG_VERSION: ${{ steps.versions.outputs.mcpg }} run: bash tests/awf-copilot-safeoutputs/run.sh diff --git a/AGENTS.md b/AGENTS.md index ed63bfe48..ed8368a24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -319,7 +319,9 @@ fail-closed and only pauses when the agent actually proposed a reviewed output. │ ├── conclusion/ # Conclusion-job reporter source (bundled to conclusion.js) │ ├── approval-summary/ # Safe-outputs summary renderer (bundled to approval-summary.js; end-of-Agent-job summary tab) │ ├── github-app-token/ # GitHub App token minter (bundled to github-app-token.js; mints installation token in Agent + Detection when engine.github-app-token is set) -│ ├── copilot-invoker/ # Sandboxed Copilot process harness (bundled to copilot-invoker.js): strict versioned invocation/result documents, runtime model resolution, typed argv, signal forwarding, exact exit propagation +│ ├── copilot-shared/ # Strict schema-v2 request/prepared/result protocol, model resolution, typed argv/env, atomic writes shared by the controller and runner +│ ├── copilot-controller/ # Trusted host control plane (bundled to copilot-controller.js): prepare + read-result only; never mounted into AWF +│ ├── copilot-runner/ # Sandbox-only process harness (bundled to copilot-runner.js): run only, self-removal, typed argv, signal forwarding, exact exit propagation │ ├── executor-e2e/ # Stage 3 safe-output E2E test harness (not a bundle; runs deterministic scenarios against a real ADO project and files a GitHub issue on failure) │ ├── compiler-smoke-e2e/ # Smoke E2E orchestrator (not a bundle): stages each case in `tests/smoke/cases.json` to the fixed `.smoke/pipeline.yml` path on its own per-case `ado-aw-mirror` ref, queues it against its credential *lane* definition, and asserts they go green. Two modes via `SMOKE_COMPILER_SOURCE`: `candidate` (compiler built from this commit, pinned pipeline-artifact) and `released` (latest release asset, release URLs required). Built to `test-bin/` by `build:compiler-smoke-e2e`, listed in `NON_BUNDLE_DIRS`. │ ├── prepare-pr-base/ # create-pull-request preparer (bundled to prepare-pr-base.js): Agent mode uses ADO diff metadata + bounded fallback; SafeOutputs fetches the target tip; cross-org targets use isolated credentials + exact remote matching @@ -464,7 +466,8 @@ index to jump to the right page. (`gate.js`, `import.js`, the execution-context `exec-context-*.js` bundles, `conclusion.js`, `approval-summary.js`, `github-app-token.js`, `prepare-pr-base.js`, and - `azure-wif-refresh.js`, `copilot-invoker.js`), schemars-driven + `azure-wif-refresh.js`, `copilot-controller.js`, `copilot-runner.js`), + schemars-driven type codegen, the A2 design decision, the bundle env contract modelled in `src/compile/ado_bundle.rs`, and the `trigger-e2e/` gate-spec drift guard (kept in sync via `export-fact-catalog`). @@ -534,7 +537,11 @@ Following the gh-aw security model: assume deletion will make the exchange safe. This trap has caused repeated incorrect designs in credential-bearing work. Stream private material over stdin or use a container-private volume; publish only intentionally public - files (for example the interception CA certificate) under `/tmp`. + files (for example the interception CA certificate) under `/tmp`. The same + boundary applies to integrity, not only secrecy: after AWF starts, never + execute host-side code from `/tmp` or treat `/tmp` metadata as authoritative. + Copy trusted executables and results beneath `$(Agent.TempDirectory)` before + AWF and consume only those private copies afterward. 3. **Tool Allow-listing**: Agents have access to a limited, controlled set of tools — see [`docs/tools.md`](docs/tools.md) and [`docs/mcp.md`](docs/mcp.md). diff --git a/docs/ado-script.md b/docs/ado-script.md index 9c932e97a..77db181ab 100644 --- a/docs/ado-script.md +++ b/docs/ado-script.md @@ -98,13 +98,17 @@ pipeline** as runtime helpers. Today it produces the following shipped bundles: never enter the agent, MCP environment, Docker arguments, logs, status documents, or artifacts. See [`mcp.md`](mcp.md#renewable-azure-workload-identity). -- `copilot-invoker.js` — the sandboxed Copilot process harness used by every - Agent job and every enabled Detection job. It validates a versioned, - compiler-emitted invocation document, resolves role-specific runtime model - variables, constructs Copilot argv without a shell, forwards termination - signals, publishes the requested-model result, and preserves Copilot's exit - status. It does not own AWF topology, credentials, retries, or OTel - interpretation. +- `copilot-controller.js` — the trusted host control plane used by every Agent + job and every enabled Detection job. Before AWF starts, it validates the + compiler request, resolves role-specific runtime model variables once, and + writes both the prepared sandbox invocation and an authoritative + host-private result. After AWF, its private copy validates that private + result. +- `copilot-runner.js` — the sandbox-only Copilot process harness. It validates + a prepared invocation, removes that document and its own bundle before + starting Copilot, constructs argv without a shell, forwards termination + signals, and preserves Copilot's exit status. It has no controller modes and + writes no authoritative metadata. > **Internal-only.** `ado-script` is not a user-facing front-matter > feature. Authors never write an `ado-script:` block in their agent @@ -673,9 +677,15 @@ scripts/ado-script/ │ ├── azure-wif-refresh/ # azure-wif-refresh.js entry point + renewable WIF assertion sidecar │ │ ├── index.ts # main(): rotate a private token file for user-defined stdio MCP servers │ │ └── __tests__/ # unit tests for rotation and isolation behaviour -│ ├── copilot-invoker/ # copilot-invoker.js entry point + versioned Copilot process harness -│ │ ├── index.ts # document/result validation, model resolution, argv, signals, exact exit -│ │ └── index.test.ts # schema, precedence, argv, environment, result, and process tests +│ ├── copilot-shared/ # strict schema-v2 request/prepared/result protocol shared by both bundles +│ │ ├── protocol.ts # validation, model resolution, argv/environment construction, atomic JSON writes +│ │ └── protocol.test.ts # schema, precedence, argv, environment, and result tests +│ ├── copilot-controller/ # copilot-controller.js trusted-host entry point +│ │ ├── index.ts # prepare + read-result modes only +│ │ └── index.test.ts # controller CLI and trusted-result tests +│ ├── copilot-runner/ # copilot-runner.js sandbox entry point +│ │ ├── index.ts # run mode, self-removal, signals, exact exit +│ │ └── index.test.ts # process, lifecycle, and mode-isolation tests │ ├── trigger-e2e/ # test-only: FACT_META gate-spec table + trigger-evaluation E2E scenarios (not a bundle) │ │ ├── gate-spec.ts # FACT_META mirror of Rust Fact::ALL; drift-guarded by export-fact-catalog + fact-catalog.gen.json │ │ ├── fact-catalog.gen.json # generated by `cargo run -- export-fact-catalog`; deep-compared by gate-spec.test.ts @@ -700,7 +710,8 @@ scripts/ado-script/ ├── prepare-pr-base.js # ncc bundle output (gitignored) ├── ado-proxy.js # ncc bundle output (gitignored) ├── azure-wif-refresh.js # ncc bundle output (gitignored) -└── copilot-invoker.js # ncc bundle output (gitignored) +├── copilot-controller.js # ncc bundle output (gitignored) +└── copilot-runner.js # ncc bundle output (gitignored) ``` The release workflow (`.github/workflows/release.yml`) runs @@ -712,7 +723,7 @@ captures every bundle, including `gate.js`, `import.js`, `exec-context-schedule.js`, `exec-context-pr-checks.js`, `exec-context-repo.js`, `conclusion.js`, `approval-summary.js`, `github-app-token.js`, `prepare-pr-base.js`, `ado-proxy.js`, -`azure-wif-refresh.js`, and `copilot-invoker.js` — into the +`azure-wif-refresh.js`, `copilot-controller.js`, and `copilot-runner.js` — into the `ado-script.zip` release asset. Pipelines download that asset at runtime by URL pinned to the compiler's `CARGO_PKG_VERSION`, verify its SHA-256 against the `checksums.txt` asset, then extract. @@ -772,11 +783,11 @@ returns three typed `Declarations::setup_steps` entries for the Setup job: runs the gate with `GATE_SPEC` and the env-var contract documented above. -### Agent job (Copilot invoker + optional helpers) +### Agent job (Copilot controller/runner + optional helpers) Every Agent job receives the install + download pair in `Declarations::agent_prepare_steps`, because every Copilot run uses -`copilot-invoker.js`: +the controller and runner: 1. **`UseNode@1`** — same shape as above. 2. **`curl` download + verify + extract** — same artefact, same @@ -786,11 +797,19 @@ Every Agent job receives the install + download pair in `/tmp/awf-tools/agent-prompt.md` in place. See [`runtime-imports.md`](runtime-imports.md) for marker syntax. **Only emitted when `inlined-imports: false`.** -4. The AWF run executes the fixed command - `node /tmp/ado-aw-scripts/ado-script/copilot-invoker.js run - /tmp/awf-tools/copilot-invocation.json`. Author values, prompt contents, and - Azure DevOps model macros are data in the invocation document or typed - environment mappings, never shell fragments. +4. The Agent task copies the verified controller into a mode-0700 directory + beneath `$(Agent.TempDirectory)`, writes the schema-v2 request there, and + runs `prepare`. Preparation atomically writes the host-private result and + the non-authoritative prepared invocation under `/tmp/awf-tools`. +5. The AWF run executes the fixed command + `node /tmp/ado-aw-scripts/ado-script/copilot-runner.js run + /tmp/awf-tools/copilot-invocation.json`. Author values and prompt contents + are data in the prepared document, never shell fragments. The private + controller copy and its request/result paths are not mounted or passed into + AWF. +6. After AWF, only the private controller copy reads the private result. The + controller source staged under `/tmp` is removed before AWF, and no + sandbox-writable code or metadata is trusted on the host afterward. The PR-context precompute step (`node exec-context-pr.js`) is owned by `ExecContextExtension` (not `AdoScriptExtension`) and emitted through @@ -802,26 +821,46 @@ exec-context invocation runs. ### Detection job Every enabled Detection job installs Node and stages `ado-script.zip`, then -uses the same fixed invoker command with a Detection-role document. Detection -does not receive the Agent MCP configuration. Workflows that explicitly disable -threat detection emit no Detection download, document, model mappings, or -invoker command. +uses the same trusted controller plus fixed runner command with a +Detection-role request. Detection does not receive the Agent MCP configuration. +Workflows that explicitly disable threat detection emit no Detection download, +documents, model mappings, controller, or runner command. ### Copilot invocation and result protocol -The compiler writes schema version 1 JSON under -`/tmp/awf-tools/copilot-invocation.json`. The document contains only -compiler-validated, non-secret data: role, command, prompt path, optional MCP -config path, argv entries, optional explicit model, and result path. The -invoker rejects unknown fields and unsupported versions, reads the prompt as -UTF-8, spawns Copilot without a shell, and sets or removes `COPILOT_MODEL` only -in the child environment. - -Before spawning Copilot, the invoker atomically writes -`/tmp/awf-tools/copilot-invocation-result.json` with the role and requested -model. Trusted host code validates that document with the invoker's -`read-result` mode and merges it into `aw_info.json`. The invoker never edits -trusted metadata and never receives Stage 3 credentials. +Schema version 2 has three strict document kinds: + +- **request** — compiler-emitted under the private + `$(Agent.TempDirectory)` controller directory. It contains role, command, + prompt path, optional MCP config path, argv entries, and optional explicit + model. +- **prepared** — controller-emitted under + `/tmp/awf-tools/copilot-invocation.json`. It contains the same execution data + plus the resolved `requested_model`; it has no result path and performs no + ADO-variable lookup inside AWF. +- **result** — controller-emitted beside the private request. It contains the + role and the same `requested_model` written into the prepared document. + +The controller supports only `prepare ` and +`read-result `. The runner supports only +`run `. Both reject unknown fields, wrong document kinds, unsupported +versions, unsafe paths/commands, invalid models, and NUL bytes. + +Preparation happens after Azure DevOps resolves the task's typed environment +mappings, so YAML variables, UI variables, variable groups, and earlier +same-job `task.setvariable` values retain their task-start semantics. The model +is resolved once and atomically committed to both execution and audit metadata. +The runner ignores runtime selector variables, uses only the prepared model, +reads the prompt as UTF-8, spawns Copilot without a shell, and sets or removes +`COPILOT_MODEL` only in the child environment. + +AWF exposes host `/tmp` read-write to the sandbox. Therefore every file beneath +`/tmp`, including the runner and prepared document, is untrusted after AWF +starts. The runner removes itself and the prepared document before giving +Copilot control, but that is defense in depth rather than a single-process +security boundary. The authoritative controller and result remain outside +`/tmp`; host code never executes or trusts sandbox-writable code or metadata +afterward. ### Per-job download (NOT a duplication bug) @@ -838,14 +877,14 @@ The rows below assume the synthetic-PR resolver is **not** active | Setup consumer | Agent optional consumers | Setup-job steps | Agent-job steps | |---|---|---|---| -| no gate | none | (none) | install + download + invoker | -| no gate | runtime imports | (none) | install + download + resolver + invoker | -| no gate | execution-context contributor(s) | (none) | install + download + exec-context bundle(s) + invoker | -| gate | none | install + download + gate | install + download + invoker | -| gate | resolver / exec-context | install + download + gate | install + download + optional helpers + invoker | +| no gate | none | (none) | install + download + controller/runner | +| no gate | runtime imports | (none) | install + download + resolver + controller/runner | +| no gate | execution-context contributor(s) | (none) | install + download + exec-context bundle(s) + controller/runner | +| gate | none | install + download + gate | install + download + controller/runner | +| gate | resolver / exec-context | install + download + gate | install + download + optional helpers + controller/runner | Every row also has an enabled Detection job with its own install + download + -invoker unless threat detection is explicitly disabled. +controller/runner unless threat detection is explicitly disabled. When the synthetic-PR resolver **is** active (`pr_trigger_for_synth = Some(_)`, i.e. `synthetic_pr_active()` is @@ -855,9 +894,9 @@ emitted even with no gate: | Setup consumer | Setup-job steps | Agent-job steps | |---|---|---| -| synth-PR (no gate) | install + download + synth-PR | install + download + optional helpers + invoker | -| gate (no synth-PR) | install + download + gate | install + download + optional helpers + invoker | -| synth-PR + gate | install + download + synth-PR + gate | install + download + optional helpers + invoker | +| synth-PR (no gate) | install + download + synth-PR | install + download + optional helpers + controller/runner | +| gate (no synth-PR) | install + download + gate | install + download + optional helpers + controller/runner | +| synth-PR + gate | install + download + synth-PR + gate | install + download + optional helpers + controller/runner | The "Setup consumer" column is gated on `filters:` lowering to non-empty checks **or** `synthetic_pr_active()` being true. The Agent download is diff --git a/docs/engine.md b/docs/engine.md index 0c39757bb..4562a5c43 100644 --- a/docs/engine.md +++ b/docs/engine.md @@ -22,7 +22,7 @@ engine: | Field | Type | Default | Description | |-------|------|---------|-------------| | `id` | string | `copilot` | Engine identifier. Currently only `copilot` (GitHub Copilot CLI) is supported. | -| `model` | string | *(none)* | AI model to use (e.g., `gpt-5-mini`). When set, the compiler records it in the versioned Copilot invocation document and the invoker sets Copilot CLI's native `COPILOT_MODEL` environment variable. When omitted, runtime model controls can select a model; if no runtime control is set, `COPILOT_MODEL` remains unset and the Copilot CLI chooses its own default. | +| `model` | string | *(none)* | AI model to use (e.g., `gpt-5-mini`). When set, the compiler records it in the versioned Copilot request; the trusted controller preserves it in the prepared invocation, and the sandbox runner sets Copilot CLI's native `COPILOT_MODEL` environment variable. When omitted, runtime model controls can select a model; if no runtime control is set, `COPILOT_MODEL` remains unset and the Copilot CLI chooses its own default. | | `timeout-minutes` | integer | *(none)* | Maximum time in minutes the agent job is allowed to run. Sets `timeoutInMinutes` on the `Agent` job in the generated pipeline. | | `version` | string | *(none)* | Engine CLI version to install (e.g., `"1.0.70"`, `"latest"`). Overrides the pinned `COPILOT_CLI_VERSION`. Set to `"latest"` to use the newest available version. | | `agent` | string | *(none)* | Custom agent file identifier (Copilot only). Adds `--agent ` to the CLI invocation, selecting a custom agent from `.github/agents/`. | @@ -65,19 +65,25 @@ recompilation. Runtime values are passed through typed step environment mappings, so Azure DevOps YAML variables, UI variables, variable groups, and variables set by an earlier trusted `##vso[task.setvariable]` step all resolve at task start. Inside -AWF, the compiler-owned `copilot-invoker.js` validates the selected value and -sets Copilot CLI's native `COPILOT_MODEL` only in the child process environment. -When no value resolves, the invoker removes `COPILOT_MODEL` rather than -supplying a compiler default. Raw `engine.args --model` and +the trusted host task, `copilot-controller.js` validates and resolves the +selected value before AWF starts. It writes both a host-private result and a +prepared sandbox invocation containing that same model decision. Inside AWF, +the run-only `copilot-runner.js` sets Copilot CLI's native `COPILOT_MODEL` only +in the child process environment. When no value resolves, the runner removes +`COPILOT_MODEL` rather than supplying a compiler default. Raw +`engine.args --model` and `engine.env.COPILOT_MODEL` are rejected so they cannot bypass this precedence. -The invoker writes a strict result document before starting Copilot. The trusted -Agent host task reads that result after AWF returns and records the requested -session model in `aw_info.json`; Detection uses the same result contract and -later enriches the copied metadata in `analyzed_outputs_` from its own -job scope. A prior trusted step can therefore set a variable and both execution -and metadata see the same task-start value. `ado-aw audit` merges those -job-owned fields. +The authoritative result and controller copy live beneath +`$(Agent.TempDirectory)`, outside AWF's automatic host `/tmp` mount. After AWF +returns, the host validates that private result with the private controller and +records the requested session model in `aw_info.json`; Detection uses the same +contract and later enriches the copied metadata in +`analyzed_outputs_` from its own job scope. No JavaScript or result +file exposed through sandbox-writable `/tmp` is executed or trusted afterward. +A prior trusted step can therefore set a variable and both execution and +metadata see the same task-start value. `ado-aw audit` merges those job-owned +fields. When `engine.agent` selects a custom agent whose definition declares `model` or `models`, Copilot CLI may use that agent-pinned model instead of the requested diff --git a/scripts/ado-script/.gitignore b/scripts/ado-script/.gitignore index 196e2b199..db750d1f3 100644 --- a/scripts/ado-script/.gitignore +++ b/scripts/ado-script/.gitignore @@ -17,7 +17,8 @@ github-app-token.js prepare-pr-base.js ado-proxy.js azure-wif-refresh.js -copilot-invoker.js +copilot-controller.js +copilot-runner.js schema *.tsbuildinfo test-bin diff --git a/scripts/ado-script/package.json b/scripts/ado-script/package.json index 049ae9d8a..0e611488a 100644 --- a/scripts/ado-script/package.json +++ b/scripts/ado-script/package.json @@ -7,8 +7,8 @@ "node": ">=20.0.0" }, "scripts": { - "build": "npm run codegen && npm run clean && npm run build:gate && npm run build:import && npm run build:exec-context-pr && npm run build:exec-context-pr-synth && npm run build:exec-context-manual && npm run build:exec-context-pipeline && npm run build:exec-context-ci-push && npm run build:exec-context-workitem && npm run build:exec-context-schedule && npm run build:exec-context-pr-checks && npm run build:exec-context-repo && npm run build:conclusion && npm run build:approval-summary && npm run build:github-app-token && npm run build:prepare-pr-base && npm run build:ado-proxy && npm run build:azure-wif-refresh && npm run build:copilot-invoker", - "clean": "node -e \"const fs=require('node:fs'); fs.rmSync('.ado-build',{recursive:true,force:true}); for (const n of ['gate','import','exec-context-pr','exec-context-pr-synth','exec-context-manual','exec-context-pipeline','exec-context-ci-push','exec-context-workitem','exec-context-schedule','exec-context-pr-checks','exec-context-repo','conclusion','approval-summary','github-app-token','prepare-pr-base','ado-proxy','azure-wif-refresh','copilot-invoker']) fs.rmSync(n+'.js',{force:true});\"", + "build": "npm run codegen && npm run clean && npm run build:gate && npm run build:import && npm run build:exec-context-pr && npm run build:exec-context-pr-synth && npm run build:exec-context-manual && npm run build:exec-context-pipeline && npm run build:exec-context-ci-push && npm run build:exec-context-workitem && npm run build:exec-context-schedule && npm run build:exec-context-pr-checks && npm run build:exec-context-repo && npm run build:conclusion && npm run build:approval-summary && npm run build:github-app-token && npm run build:prepare-pr-base && npm run build:ado-proxy && npm run build:azure-wif-refresh && npm run build:copilot-controller && npm run build:copilot-runner", + "clean": "node -e \"const fs=require('node:fs'); fs.rmSync('.ado-build',{recursive:true,force:true}); for (const n of ['gate','import','exec-context-pr','exec-context-pr-synth','exec-context-manual','exec-context-pipeline','exec-context-ci-push','exec-context-workitem','exec-context-schedule','exec-context-pr-checks','exec-context-repo','conclusion','approval-summary','github-app-token','prepare-pr-base','ado-proxy','azure-wif-refresh','copilot-invoker','copilot-controller','copilot-runner']) fs.rmSync(n+'.js',{force:true});\"", "build:gate": "ncc build src/gate/index.ts -o .ado-build/gate -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/gate/index.js','gate.js'); fs.rmSync('.ado-build/gate',{recursive:true,force:true});\"", "build:import": "ncc build src/import/index.ts -o .ado-build/import -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/import/index.js','import.js'); fs.rmSync('.ado-build/import',{recursive:true,force:true});\"", "build:exec-context-pr": "ncc build src/exec-context-pr/index.ts -o .ado-build/exec-context-pr -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/exec-context-pr/index.js','exec-context-pr.js'); fs.rmSync('.ado-build/exec-context-pr',{recursive:true,force:true});\"", @@ -26,14 +26,15 @@ "build:prepare-pr-base": "ncc build src/prepare-pr-base/index.ts -o .ado-build/prepare-pr-base -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/prepare-pr-base/index.js','prepare-pr-base.js'); fs.rmSync('.ado-build/prepare-pr-base',{recursive:true,force:true});\"", "build:ado-proxy": "ncc build src/ado-proxy/index.ts -o .ado-build/ado-proxy -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/ado-proxy/index.js','ado-proxy.js'); fs.rmSync('.ado-build/ado-proxy',{recursive:true,force:true});\"", "build:azure-wif-refresh": "ncc build src/azure-wif-refresh/index.ts -o .ado-build/azure-wif-refresh -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/azure-wif-refresh/index.js','azure-wif-refresh.js'); fs.rmSync('.ado-build/azure-wif-refresh',{recursive:true,force:true});\"", - "build:copilot-invoker": "ncc build src/copilot-invoker/index.ts -o .ado-build/copilot-invoker -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/copilot-invoker/index.js','copilot-invoker.js'); fs.rmSync('.ado-build/copilot-invoker',{recursive:true,force:true});\"", + "build:copilot-controller": "ncc build src/copilot-controller/index.ts -o .ado-build/copilot-controller -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/copilot-controller/index.js','copilot-controller.js'); fs.rmSync('.ado-build/copilot-controller',{recursive:true,force:true});\"", + "build:copilot-runner": "ncc build src/copilot-runner/index.ts -o .ado-build/copilot-runner -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/copilot-runner/index.js','copilot-runner.js'); fs.rmSync('.ado-build/copilot-runner',{recursive:true,force:true});\"", "build:executor-e2e": "ncc build src/executor-e2e/index.ts -o .ado-build/executor-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/executor-e2e/index.js','test-bin/executor-e2e.js'); fs.rmSync('.ado-build/executor-e2e',{recursive:true,force:true});\"", "build:trigger-e2e": "ncc build src/trigger-e2e/index.ts -o .ado-build/trigger-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/trigger-e2e/index.js','test-bin/trigger-e2e.js'); fs.rmSync('.ado-build/trigger-e2e',{recursive:true,force:true});\"", "build:compiler-smoke-e2e": "ncc build src/compiler-smoke-e2e/index.ts -o .ado-build/compiler-smoke-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/compiler-smoke-e2e/index.js','test-bin/compiler-smoke-e2e.js'); fs.rmSync('.ado-build/compiler-smoke-e2e',{recursive:true,force:true});\"", "build:check": "ls -lh gate.js && wc -c gate.js", "codegen": "node -e \"require('node:fs').mkdirSync('schema', { recursive: true })\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-gate-schema --output schema/gate-spec.schema.json && npx json2ts schema/gate-spec.schema.json -o src/shared/types.gen.ts --bannerComment \"// AUTO-GENERATED from Rust IR via cargo run -- export-gate-schema. Do not edit; run npm run codegen.\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-fact-catalog --output src/trigger-e2e/fact-catalog.gen.json && cargo run --quiet --manifest-path ../../Cargo.toml -- export-ado-proxy-catalog-schema --output schema/ado-proxy-catalog.schema.json && npx json2ts schema/ado-proxy-catalog.schema.json -o src/shared/ado-proxy-catalog.types.gen.ts --bannerComment \"// AUTO-GENERATED from Rust via cargo run -- export-ado-proxy-catalog-schema. Do not edit; run npm run codegen.\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-ado-proxy-catalog --output src/ado-proxy/catalog.gen.json", "test": "vitest run", - "test:smoke": "npm run build:gate && npm run build:import && npm run build:exec-context-pr && npm run build:exec-context-pr-synth && npm run build:exec-context-manual && npm run build:exec-context-pipeline && npm run build:exec-context-ci-push && npm run build:exec-context-workitem && npm run build:exec-context-schedule && npm run build:exec-context-pr-checks && npm run build:exec-context-repo && npm run build:conclusion && npm run build:approval-summary && npm run build:github-app-token && npm run build:prepare-pr-base && npm run build:ado-proxy && npm run build:azure-wif-refresh && npm run build:copilot-invoker && vitest run -c vitest.config.smoke.ts", + "test:smoke": "npm run build:gate && npm run build:import && npm run build:exec-context-pr && npm run build:exec-context-pr-synth && npm run build:exec-context-manual && npm run build:exec-context-pipeline && npm run build:exec-context-ci-push && npm run build:exec-context-workitem && npm run build:exec-context-schedule && npm run build:exec-context-pr-checks && npm run build:exec-context-repo && npm run build:conclusion && npm run build:approval-summary && npm run build:github-app-token && npm run build:prepare-pr-base && npm run build:ado-proxy && npm run build:azure-wif-refresh && npm run build:copilot-controller && npm run build:copilot-runner && vitest run -c vitest.config.smoke.ts", "lint": "echo TODO", "typecheck": "tsc --noEmit" }, diff --git a/scripts/ado-script/src/__tests__/bundle-coverage.test.ts b/scripts/ado-script/src/__tests__/bundle-coverage.test.ts index 053207945..4c9861590 100644 --- a/scripts/ado-script/src/__tests__/bundle-coverage.test.ts +++ b/scripts/ado-script/src/__tests__/bundle-coverage.test.ts @@ -44,6 +44,7 @@ const packageJsonPath = join(here, "..", "..", "package.json"); */ const NON_BUNDLE_DIRS = new Set([ "shared", + "copilot-shared", "__tests__", "executor-e2e", "trigger-e2e", diff --git a/scripts/ado-script/src/copilot-controller/index.test.ts b/scripts/ado-script/src/copilot-controller/index.test.ts new file mode 100644 index 000000000..bcc8fe108 --- /dev/null +++ b/scripts/ado-script/src/copilot-controller/index.test.ts @@ -0,0 +1,112 @@ +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { main } from "./index.js"; + +function writeRequest(directory: string): string { + const path = join(directory, "request.json"); + writeFileSync( + path, + JSON.stringify({ + schema_version: 2, + document_kind: "request", + role: "agent", + command: "/tmp/awf-tools/copilot", + prompt_path: "/tmp/awf-tools/agent-prompt.md", + mcp_config_path: null, + args: ["--no-ask-user"], + explicit_model: null, + }), + ); + return path; +} + +afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +describe("copilot controller", () => { + it("prepares sandbox execution and host result from the same model decision", async () => { + const directory = mkdtempSync(join(tmpdir(), "copilot-controller-")); + const requestPath = writeRequest(directory); + const preparedPath = join(directory, "prepared.json"); + const resultPath = join(directory, "result.json"); + vi.stubEnv("ADO_AW_MODEL_AGENT_COPILOT", "runtime-model"); + + await expect( + main(["prepare", requestPath, preparedPath, resultPath]), + ).resolves.toBe(0); + const prepared = JSON.parse(readFileSync(preparedPath, "utf8")); + const result = JSON.parse(readFileSync(resultPath, "utf8")); + expect(prepared).toMatchObject({ + schema_version: 2, + document_kind: "prepared", + requested_model: "runtime-model", + }); + expect(result).toEqual({ + schema_version: 2, + document_kind: "result", + role: "agent", + requested_model: prepared.requested_model, + }); + }); + + it("reads only a matching strict result", async () => { + const directory = mkdtempSync(join(tmpdir(), "copilot-controller-")); + const resultPath = join(directory, "result.json"); + writeFileSync( + resultPath, + JSON.stringify({ + schema_version: 2, + document_kind: "result", + role: "agent", + requested_model: "gpt-test", + }), + ); + const write = vi.spyOn(process.stdout, "write").mockImplementation( + ((_chunk: unknown, callback?: (error?: Error | null) => void) => { + callback?.(); + return true; + }) as typeof process.stdout.write, + ); + await expect(main(["read-result", resultPath, "agent"])).resolves.toBe(0); + expect(write).toHaveBeenCalledWith("gpt-test", expect.any(Function)); + }); + + it("rejects role mismatch, missing results, and stdout failures", async () => { + const directory = mkdtempSync(join(tmpdir(), "copilot-controller-")); + const resultPath = join(directory, "result.json"); + writeFileSync( + resultPath, + JSON.stringify({ + schema_version: 2, + document_kind: "result", + role: "agent", + requested_model: "gpt-test", + }), + ); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + await expect(main(["read-result", resultPath, "detection"])).resolves.toBe(1); + await expect( + main(["read-result", join(directory, "missing.json"), "agent"]), + ).resolves.toBe(1); + + vi.spyOn(process.stdout, "write").mockImplementation( + ((_chunk: unknown, callback?: (error?: Error | null) => void) => { + callback?.(new Error("EPIPE")); + return false; + }) as typeof process.stdout.write, + ); + await expect(main(["read-result", resultPath, "agent"])).resolves.toBe(1); + expect(error).toHaveBeenCalledWith("copilot-controller: EPIPE"); + }); + + it("does not expose sandbox run mode", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + await expect(main(["run", "/tmp/prepared.json"])).resolves.toBe(2); + expect(error).toHaveBeenCalledWith(expect.stringContaining("usage:")); + }); +}); diff --git a/scripts/ado-script/src/copilot-controller/index.ts b/scripts/ado-script/src/copilot-controller/index.ts new file mode 100644 index 000000000..7f4ede8e8 --- /dev/null +++ b/scripts/ado-script/src/copilot-controller/index.ts @@ -0,0 +1,68 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + parseInvocationRequest, + parseInvocationResult, + prepareInvocation, + writeJsonAtomic, +} from "../copilot-shared/protocol.js"; + +async function writeStdout(value: string): Promise { + await new Promise((resolveWrite, rejectWrite) => { + process.stdout.write(value, (error) => { + if (error) rejectWrite(error); + else resolveWrite(); + }); + }); +} + +export async function main(argv: string[]): Promise { + if (argv[0] === "prepare" && argv.length === 4) { + try { + const request = parseInvocationRequest(readFileSync(argv[1]!, "utf8")); + const { prepared, result } = prepareInvocation(request, process.env); + writeJsonAtomic(argv[2]!, prepared); + writeJsonAtomic(argv[3]!, result); + return 0; + } catch (error) { + const message = error instanceof Error ? error.message : "unknown error"; + console.error(`copilot-controller: ${message}`); + return 1; + } + } + if (argv[0] === "read-result" && argv.length === 3) { + try { + const result = parseInvocationResult(readFileSync(argv[1]!, "utf8")); + if (result.role !== argv[2]) { + throw new Error( + `invocation result role '${result.role}' does not match expected role '${argv[2]}'`, + ); + } + await writeStdout(result.requested_model ?? ""); + return 0; + } catch (error) { + const message = error instanceof Error ? error.message : "unknown error"; + console.error(`copilot-controller: ${message}`); + return 1; + } + } + console.error( + "usage: copilot-controller prepare | read-result ", + ); + return 2; +} + +const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; +if (invokedPath === fileURLToPath(import.meta.url)) { + void main(process.argv.slice(2)) + .then((code) => { + process.exitCode = code; + }) + .catch((error: unknown) => { + const message = error instanceof Error ? error.message : "unknown error"; + console.error(`copilot-controller: unexpected failure: ${message}`); + process.exitCode = 1; + }); +} diff --git a/scripts/ado-script/src/copilot-invoker/index.test.ts b/scripts/ado-script/src/copilot-invoker/index.test.ts deleted file mode 100644 index d04688f19..000000000 --- a/scripts/ado-script/src/copilot-invoker/index.test.ts +++ /dev/null @@ -1,359 +0,0 @@ -import { EventEmitter } from "node:events"; -import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { describe, expect, it, vi } from "vitest"; - -import { - buildChildEnvironment, - buildCopilotArgs, - main, - parseInvocationDocument, - parseInvocationResult, - resolveRequestedModel, - runInvocation, - type InvocationDocument, - type InvocationResult, -} from "./index.js"; - -function document(overrides: Partial = {}): InvocationDocument { - return { - schema_version: 1, - role: "agent", - command: "/tmp/awf-tools/copilot", - prompt_path: "/tmp/awf-tools/agent-prompt.md", - mcp_config_path: "/tmp/awf-tools/mcp-config.json", - args: ["--disable-builtin-mcps", "--allow-tool", "shell(cat *)"], - explicit_model: null, - result_path: "/tmp/awf-tools/copilot-invocation-result.json", - ...overrides, - }; -} - -describe("copilot invoker document", () => { - it("parses a strict versioned document", () => { - expect(parseInvocationDocument(JSON.stringify(document()))).toEqual(document()); - }); - - describe("copilot invoker result", () => { - it("parses a strict result document", () => { - expect( - parseInvocationResult( - JSON.stringify({ - schema_version: 1, - role: "detection", - requested_model: "detector", - }), - ), - ).toEqual({ - schema_version: 1, - role: "detection", - requested_model: "detector", - }); - }); - - it.each([ - [{ schema_version: 2, role: "agent", requested_model: null }, "unsupported"], - [ - { schema_version: 1, role: "agent", requested_model: null, extra: true }, - "unknown field", - ], - [{ schema_version: 1, role: "other", requested_model: null }, "must be"], - [{ schema_version: 1, role: "agent", requested_model: "bad model" }, "invalid"], - ])("rejects malformed results %#", (value, message) => { - expect(() => parseInvocationResult(JSON.stringify(value))).toThrow(message); - }); - }); - - it.each([ - [{ ...document(), schema_version: 2 }, "unsupported invocation schema"], - [{ ...document(), unexpected: true }, "unknown field 'unexpected'"], - [{ ...document(), role: "other" }, "must be 'agent' or 'detection'"], - [{ ...document(), command: "copilot;sh" }, "field 'command' is invalid"], - [{ ...document(), command: ".." }, "field 'command' is invalid"], - [{ ...document(), command: "bin/copilot" }, "field 'command' is invalid"], - [{ ...document(), command: "/tmp/../copilot" }, "field 'command' is invalid"], - [{ ...document(), command: "/tmp//copilot" }, "field 'command' is invalid"], - [{ ...document(), command: "/tmp/copilot/" }, "field 'command' is invalid"], - [{ ...document(), prompt_path: "relative.md" }, "field 'prompt_path' is invalid"], - [{ ...document(), prompt_path: "/tmp/../prompt.md" }, "field 'prompt_path' is invalid"], - [{ ...document(), result_path: "/" }, "field 'result_path' is invalid"], - [{ ...document(), args: [1] }, "field 'args' must be an array of strings"], - [{ ...document(), explicit_model: "bad model" }, "field 'explicit_model' is invalid"], - ])("rejects malformed input %#", (value, message) => { - expect(() => parseInvocationDocument(JSON.stringify(value))).toThrow(message); - }); -}); - -describe("model resolution", () => { - it("keeps an explicit model authoritative", () => { - expect( - resolveRequestedModel(document({ explicit_model: "frontmatter-model" }), { - ADO_AW_MODEL_AGENT_COPILOT: "role-model", - ADO_AW_DEFAULT_MODEL_COPILOT: "default-model", - }), - ).toBe("frontmatter-model"); - }); - - it("uses role-specific then shared values", () => { - expect( - resolveRequestedModel(document(), { - ADO_AW_MODEL_AGENT_COPILOT: "role-model", - ADO_AW_DEFAULT_MODEL_COPILOT: "default-model", - }), - ).toBe("role-model"); - expect( - resolveRequestedModel(document({ role: "detection" }), { - ADO_AW_MODEL_DETECTION_COPILOT: "", - ADO_AW_DEFAULT_MODEL_COPILOT: "default-model", - }), - ).toBe("default-model"); - }); - - it("treats unresolved ADO macros as absent", () => { - expect( - resolveRequestedModel(document(), { - ADO_AW_MODEL_AGENT_COPILOT: "$(ADO_AW_MODEL_AGENT_COPILOT)", - ADO_AW_DEFAULT_MODEL_COPILOT: "$(ADO_AW_DEFAULT_MODEL_COPILOT)", - }), - ).toBeNull(); - }); - - it("rejects invalid runtime values without printing them", () => { - let message = ""; - try { - resolveRequestedModel(document(), { - ADO_AW_MODEL_AGENT_COPILOT: "secret value", - }); - } catch (error) { - message = error instanceof Error ? error.message : String(error); - } - expect(message).toContain("contains invalid characters"); - expect(message).not.toContain("secret value"); - }); -}); - -describe("argv and child environment", () => { - it("passes prompt, MCP config, and authored values as distinct argv elements", () => { - expect(buildCopilotArgs(document(), "line one\nline two")).toEqual([ - "--prompt=line one\nline two", - "--additional-mcp-config", - "@/tmp/awf-tools/mcp-config.json", - "--disable-builtin-mcps", - "--allow-tool", - "shell(cat *)", - ]); - }); - - it("rejects NUL bytes in prompt content", () => { - expect(() => buildCopilotArgs(document(), "before\0after")).toThrow( - "prompt contains an invalid NUL byte", - ); - }); - - it("sets or removes only the child COPILOT_MODEL", () => { - const original = { KEEP: "yes", COPILOT_MODEL: "old" }; - expect(buildChildEnvironment(original, "selected")).toEqual({ - KEEP: "yes", - COPILOT_MODEL: "selected", - }); - expect(buildChildEnvironment(original, null)).toEqual({ KEEP: "yes" }); - expect(original.COPILOT_MODEL).toBe("old"); - }); -}); - -describe("read-result command", () => { - it("writes the requested model after validating the result", async () => { - const directory = mkdtempSync(join(tmpdir(), "copilot-invoker-result-")); - const resultPath = join(directory, "result.json"); - writeFileSync( - resultPath, - JSON.stringify({ - schema_version: 1, - role: "agent", - requested_model: "gpt-test", - }), - ); - const write = vi.spyOn(process.stdout, "write").mockImplementation( - ((_chunk: unknown, callback?: (error?: Error | null) => void) => { - callback?.(); - return true; - }) as typeof process.stdout.write, - ); - - await expect(main(["read-result", resultPath, "agent"])).resolves.toBe(0); - expect(write).toHaveBeenCalledWith("gpt-test", expect.any(Function)); - write.mockRestore(); - }); - - it("rejects a result for the wrong role", async () => { - const directory = mkdtempSync(join(tmpdir(), "copilot-invoker-result-")); - const resultPath = join(directory, "result.json"); - writeFileSync( - resultPath, - JSON.stringify({ - schema_version: 1, - role: "agent", - requested_model: "gpt-test", - }), - ); - const error = vi.spyOn(console, "error").mockImplementation(() => undefined); - - await expect(main(["read-result", resultPath, "detection"])).resolves.toBe(1); - expect(error).toHaveBeenCalledWith( - "copilot-invoker: invocation result role 'agent' does not match expected role 'detection'", - ); - error.mockRestore(); - }); - - it("reports a missing or malformed result", async () => { - const directory = mkdtempSync(join(tmpdir(), "copilot-invoker-result-")); - const resultPath = join(directory, "missing.json"); - const error = vi.spyOn(console, "error").mockImplementation(() => undefined); - - await expect(main(["read-result", resultPath, "agent"])).resolves.toBe(1); - expect(error).toHaveBeenCalledWith(expect.stringContaining("copilot-invoker:")); - error.mockRestore(); - }); - - it("reports stdout write failures", async () => { - const directory = mkdtempSync(join(tmpdir(), "copilot-invoker-result-")); - const resultPath = join(directory, "result.json"); - writeFileSync( - resultPath, - JSON.stringify({ - schema_version: 1, - role: "agent", - requested_model: "gpt-test", - }), - ); - const write = vi.spyOn(process.stdout, "write").mockImplementation( - ((_chunk: unknown, callback?: (error?: Error | null) => void) => { - callback?.(new Error("EPIPE")); - return false; - }) as typeof process.stdout.write, - ); - const error = vi.spyOn(console, "error").mockImplementation(() => undefined); - - await expect(main(["read-result", resultPath, "agent"])).resolves.toBe(1); - expect(error).toHaveBeenCalledWith("copilot-invoker: EPIPE"); - write.mockRestore(); - error.mockRestore(); - }); -}); - -describe("process lifecycle", () => { - it("publishes the result before spawning and preserves the child exit code", async () => { - const events: string[] = []; - const child = new EventEmitter() as EventEmitter & { - killed: boolean; - kill: ReturnType; - }; - child.killed = false; - child.kill = vi.fn(); - const results: InvocationResult[] = []; - let spawnedArgs: readonly string[] | undefined; - const spawn = vi.fn((_command: string, args: readonly string[]) => { - spawnedArgs = args; - events.push("spawn"); - queueMicrotask(() => child.emit("close", 23, null)); - return child; - }); - - const exit = await runInvocation( - document(), - { ADO_AW_MODEL_AGENT_COPILOT: "runtime-model" }, - { - readFile: () => "prompt", - writeResult: (_path, result) => { - events.push("result"); - results.push(result); - }, - spawn: spawn as never, - }, - ); - - expect(events).toEqual(["result", "spawn"]); - expect(results).toEqual([ - { schema_version: 1, role: "agent", requested_model: "runtime-model" }, - ]); - expect(exit).toBe(23); - expect(spawnedArgs).toEqual([ - "--prompt=prompt", - "--additional-mcp-config", - "@/tmp/awf-tools/mcp-config.json", - "--disable-builtin-mcps", - "--allow-tool", - "shell(cat *)", - ]); - }); - - it("writes results atomically with private permissions", async () => { - const directory = mkdtempSync(join(tmpdir(), "copilot-invoker-")); - const resultPath = join(directory, "result.json").replaceAll("\\", "/"); - const child = new EventEmitter() as EventEmitter & { - killed: boolean; - kill: ReturnType; - }; - child.killed = false; - child.kill = vi.fn(); - const invocation = document({ result_path: resultPath }); - const promise = runInvocation(invocation, {}, { - readFile: () => "prompt", - writeResult: (await import("./index.js")).writeResultAtomic, - spawn: (() => { - queueMicrotask(() => child.emit("close", 0, null)); - return child; - }) as never, - }); - await expect(promise).resolves.toBe(0); - expect(JSON.parse(readFileSync(resultPath, "utf8"))).toEqual({ - schema_version: 1, - role: "agent", - requested_model: null, - }); - }); - - it("returns a deterministic failure when Copilot cannot start", async () => { - const child = new EventEmitter() as EventEmitter & { - killed: boolean; - kill: ReturnType; - }; - child.killed = false; - child.kill = vi.fn(); - const error = vi.spyOn(console, "error").mockImplementation(() => undefined); - const promise = runInvocation(document(), {}, { - readFile: () => "prompt", - writeResult: () => undefined, - spawn: (() => { - queueMicrotask(() => child.emit("error", new Error("ENOENT"))); - return child; - }) as never, - }); - await expect(promise).resolves.toBe(1); - expect(error).toHaveBeenCalledWith( - "copilot-invoker: failed to start Copilot: ENOENT", - ); - error.mockRestore(); - }); - - it("forwards termination signals and maps a signaled exit", async () => { - const child = new EventEmitter() as EventEmitter & { - killed: boolean; - kill: ReturnType; - }; - child.killed = false; - child.kill = vi.fn((signal: NodeJS.Signals) => { - queueMicrotask(() => child.emit("close", null, signal)); - return true; - }); - const promise = runInvocation(document(), {}, { - readFile: () => "prompt", - writeResult: () => undefined, - spawn: (() => child) as never, - }); - process.emit("SIGTERM", "SIGTERM"); - await expect(promise).resolves.toBe(143); - expect(child.kill).toHaveBeenCalledWith("SIGTERM"); - }); -}); diff --git a/scripts/ado-script/src/copilot-invoker/index.ts b/scripts/ado-script/src/copilot-invoker/index.ts deleted file mode 100644 index e6a6d4450..000000000 --- a/scripts/ado-script/src/copilot-invoker/index.ts +++ /dev/null @@ -1,386 +0,0 @@ -import { spawn, type ChildProcess } from "node:child_process"; -import { readFileSync, renameSync, writeFileSync } from "node:fs"; -import { constants } from "node:os"; -import { resolve } from "node:path"; -import { fileURLToPath } from "node:url"; - -const SCHEMA_VERSION = 1; -const MODEL_PATTERN = /^[A-Za-z0-9._:-]+$/; -const COMMAND_PATTERN = /^[A-Za-z0-9._/-]+$/; -const DOCUMENT_KEYS = new Set([ - "schema_version", - "role", - "command", - "prompt_path", - "mcp_config_path", - "args", - "explicit_model", - "result_path", -]); -const RESULT_KEYS = new Set(["schema_version", "role", "requested_model"]); - -export type InvocationRole = "agent" | "detection"; - -export interface InvocationDocument { - schema_version: 1; - role: InvocationRole; - command: string; - prompt_path: string; - mcp_config_path: string | null; - args: string[]; - explicit_model: string | null; - result_path: string; -} - -export interface InvocationResult { - schema_version: 1; - role: InvocationRole; - requested_model: string | null; -} - -interface SpawnLike { - ( - command: string, - args: readonly string[], - options: { - env: NodeJS.ProcessEnv; - stdio: "inherit"; - }, - ): ChildProcess; -} - -export interface RunDependencies { - spawn: SpawnLike; - readFile(path: string): string; - writeResult(path: string, result: InvocationResult): void; -} - -const DEFAULT_DEPENDENCIES: RunDependencies = { - spawn, - readFile: (path) => readFileSync(path, "utf8"), - writeResult: writeResultAtomic, -}; - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function requiredString( - value: Record, - key: string, - validator?: (input: string) => boolean, -): string { - const candidate = value[key]; - if (typeof candidate !== "string" || candidate.length === 0) { - throw new Error(`invocation document field '${key}' must be a non-empty string`); - } - if (candidate.includes("\0") || (validator && !validator(candidate))) { - throw new Error(`invocation document field '${key}' is invalid`); - } - return candidate; -} - -function nullableString( - value: Record, - key: string, - validator?: (input: string) => boolean, -): string | null { - const candidate = value[key]; - if (candidate === null) return null; - if (typeof candidate !== "string" || candidate.includes("\0")) { - throw new Error(`invocation document field '${key}' must be a string or null`); - } - if (validator && !validator(candidate)) { - throw new Error(`invocation document field '${key}' is invalid`); - } - return candidate; -} - -function hasSafePathSegments(value: string, allowBareCommand: boolean): boolean { - if ( - value.includes("\n") || - value.includes("\r") || - value.includes(":") || - value.endsWith("/") - ) { - return false; - } - if (allowBareCommand && !value.includes("/")) { - return value !== "." && value !== ".."; - } - if (!value.startsWith("/")) return false; - const segments = value.slice(1).split("/"); - return ( - segments.length > 0 && - segments.every((segment) => segment.length > 0 && segment !== "." && segment !== "..") - ); -} - -function isAbsoluteContainerPath(value: string): boolean { - return hasSafePathSegments(value, false); -} - -function isSafeCommand(value: string): boolean { - return COMMAND_PATTERN.test(value) && hasSafePathSegments(value, true); -} - -export function parseInvocationDocument(raw: string): InvocationDocument { - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch { - throw new Error("invocation document is not valid JSON"); - } - - if (!isRecord(parsed)) { - throw new Error("invocation document must be a JSON object"); - } - const unknown = Object.keys(parsed).filter((key) => !DOCUMENT_KEYS.has(key)); - if (unknown.length > 0) { - throw new Error(`invocation document contains unknown field '${unknown.sort()[0]}'`); - } - if (parsed.schema_version !== SCHEMA_VERSION) { - throw new Error( - `unsupported invocation schema version '${String(parsed.schema_version)}'`, - ); - } - if (parsed.role !== "agent" && parsed.role !== "detection") { - throw new Error("invocation document field 'role' must be 'agent' or 'detection'"); - } - if (!Array.isArray(parsed.args) || !parsed.args.every((arg) => typeof arg === "string")) { - throw new Error("invocation document field 'args' must be an array of strings"); - } - if (parsed.args.some((arg) => arg.includes("\0"))) { - throw new Error("invocation document field 'args' contains an invalid NUL byte"); - } - - return { - schema_version: SCHEMA_VERSION, - role: parsed.role, - command: requiredString(parsed, "command", isSafeCommand), - prompt_path: requiredString(parsed, "prompt_path", isAbsoluteContainerPath), - mcp_config_path: nullableString(parsed, "mcp_config_path", isAbsoluteContainerPath), - args: [...parsed.args], - explicit_model: nullableString(parsed, "explicit_model", (value) => - MODEL_PATTERN.test(value), - ), - result_path: requiredString(parsed, "result_path", isAbsoluteContainerPath), - }; -} - -export function parseInvocationResult(raw: string): InvocationResult { - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch { - throw new Error("invocation result is not valid JSON"); - } - if (!isRecord(parsed)) { - throw new Error("invocation result must be a JSON object"); - } - const unknown = Object.keys(parsed).filter((key) => !RESULT_KEYS.has(key)); - if (unknown.length > 0) { - throw new Error(`invocation result contains unknown field '${unknown.sort()[0]}'`); - } - if (parsed.schema_version !== SCHEMA_VERSION) { - throw new Error( - `unsupported invocation result schema version '${String(parsed.schema_version)}'`, - ); - } - if (parsed.role !== "agent" && parsed.role !== "detection") { - throw new Error("invocation result field 'role' must be 'agent' or 'detection'"); - } - const requestedModel = nullableString(parsed, "requested_model", (value) => - MODEL_PATTERN.test(value), - ); - return { - schema_version: SCHEMA_VERSION, - role: parsed.role, - requested_model: requestedModel, - }; -} - -function runtimeModelVariable(role: InvocationRole): string { - return role === "agent" - ? "ADO_AW_MODEL_AGENT_COPILOT" - : "ADO_AW_MODEL_DETECTION_COPILOT"; -} - -function isUnresolvedAdoMacro(value: string, variable: string): boolean { - return value === `$(${variable})`; -} - -export function resolveRequestedModel( - document: InvocationDocument, - env: NodeJS.ProcessEnv, -): string | null { - if (document.explicit_model !== null) { - return document.explicit_model; - } - const specific = runtimeModelVariable(document.role); - const candidates: Array<[string, string | undefined]> = [ - [specific, env[specific]], - ["ADO_AW_DEFAULT_MODEL_COPILOT", env.ADO_AW_DEFAULT_MODEL_COPILOT], - ]; - for (const [variable, candidate] of candidates) { - if ( - candidate === undefined || - candidate.length === 0 || - isUnresolvedAdoMacro(candidate, variable) - ) { - continue; - } - if (!MODEL_PATTERN.test(candidate)) { - throw new Error( - `runtime Copilot model from ${specific}/ADO_AW_DEFAULT_MODEL_COPILOT contains invalid characters`, - ); - } - return candidate; - } - return null; -} - -export function buildCopilotArgs( - document: InvocationDocument, - prompt: string, -): string[] { - if (prompt.includes("\0")) { - throw new Error("prompt contains an invalid NUL byte"); - } - const args = [`--prompt=${prompt}`]; - if (document.mcp_config_path !== null) { - args.push("--additional-mcp-config", `@${document.mcp_config_path}`); - } - args.push(...document.args); - return args; -} - -export function buildChildEnvironment( - env: NodeJS.ProcessEnv, - requestedModel: string | null, -): NodeJS.ProcessEnv { - // AWF filters host-only credentials before launching the invoker. Preserve - // the remaining environment because Copilot providers and MCPs consume it. - const childEnv = { ...env }; - if (requestedModel === null) { - delete childEnv.COPILOT_MODEL; - } else { - childEnv.COPILOT_MODEL = requestedModel; - } - return childEnv; -} - -export function writeResultAtomic(path: string, result: InvocationResult): void { - const temporary = `${path}.tmp-${process.pid}`; - writeFileSync(temporary, `${JSON.stringify(result)}\n`, { - encoding: "utf8", - mode: 0o600, - }); - renameSync(temporary, path); -} - -function signalExitCode(signal: NodeJS.Signals): number { - return 128 + (constants.signals[signal] ?? 0); -} - -export async function runInvocation( - document: InvocationDocument, - env: NodeJS.ProcessEnv = process.env, - dependencies: RunDependencies = DEFAULT_DEPENDENCIES, -): Promise { - const requestedModel = resolveRequestedModel(document, env); - dependencies.writeResult(document.result_path, { - schema_version: SCHEMA_VERSION, - role: document.role, - requested_model: requestedModel, - }); - - const prompt = dependencies.readFile(document.prompt_path); - const args = buildCopilotArgs(document, prompt); - const child = dependencies.spawn(document.command, args, { - env: buildChildEnvironment(env, requestedModel), - stdio: "inherit", - }); - - return await new Promise((resolveExit) => { - let settled = false; - const settle = (code: number) => { - if (settled) return; - settled = true; - for (const signal of forwardedSignals) { - process.off(signal, handlers[signal]); - } - resolveExit(code); - }; - const forwardedSignals: NodeJS.Signals[] = ["SIGINT", "SIGTERM", "SIGHUP"]; - const handlers = Object.fromEntries( - forwardedSignals.map((signal) => [ - signal, - () => { - if (!child.killed) child.kill(signal); - }, - ]), - ) as Record void>; - for (const signal of forwardedSignals) { - process.on(signal, handlers[signal]); - } - child.once("error", (error) => { - console.error(`copilot-invoker: failed to start Copilot: ${error.message}`); - settle(1); - }); - child.once("close", (code, signal) => { - settle(code ?? (signal ? signalExitCode(signal) : 1)); - }); - }); -} - -export async function main(argv: string[]): Promise { - if (argv[0] === "read-result" && argv.length === 3) { - try { - const result = parseInvocationResult(readFileSync(argv[1]!, "utf8")); - if (result.role !== argv[2]) { - throw new Error( - `invocation result role '${result.role}' does not match expected role '${argv[2]}'`, - ); - } - await new Promise((resolveWrite, rejectWrite) => { - process.stdout.write(result.requested_model ?? "", (error) => { - if (error) rejectWrite(error); - else resolveWrite(); - }); - }); - return 0; - } catch (error) { - const message = error instanceof Error ? error.message : "unknown error"; - console.error(`copilot-invoker: ${message}`); - return 1; - } - } - if (argv[0] !== "run" || argv.length !== 2) { - console.error( - "usage: copilot-invoker run | read-result ", - ); - return 2; - } - try { - const document = parseInvocationDocument(readFileSync(argv[1]!, "utf8")); - return await runInvocation(document); - } catch (error) { - const message = error instanceof Error ? error.message : "unknown error"; - console.error(`copilot-invoker: ${message}`); - return 1; - } -} - -const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; -if (invokedPath === fileURLToPath(import.meta.url)) { - void main(process.argv.slice(2)) - .then((code) => { - process.exitCode = code; - }) - .catch((error: unknown) => { - const message = error instanceof Error ? error.message : "unknown error"; - console.error(`copilot-invoker: unexpected failure: ${message}`); - process.exitCode = 1; - }); -} diff --git a/scripts/ado-script/src/copilot-runner/index.test.ts b/scripts/ado-script/src/copilot-runner/index.test.ts new file mode 100644 index 000000000..836cb8329 --- /dev/null +++ b/scripts/ado-script/src/copilot-runner/index.test.ts @@ -0,0 +1,150 @@ +import { EventEmitter } from "node:events"; +import { describe, expect, it, vi } from "vitest"; + +import { + main, + runInvocation, + type RunDependencies, +} from "./index.js"; +import type { PreparedInvocation } from "../copilot-shared/protocol.js"; + +function prepared( + overrides: Partial = {}, +): PreparedInvocation { + return { + schema_version: 2, + document_kind: "prepared", + role: "agent", + command: "/tmp/awf-tools/copilot", + prompt_path: "/tmp/awf-tools/agent-prompt.md", + mcp_config_path: null, + args: ["--no-ask-user"], + requested_model: "prepared-model", + ...overrides, + }; +} + +function childProcess() { + const child = new EventEmitter() as EventEmitter & { + killed: boolean; + kill: ReturnType; + }; + child.killed = false; + child.kill = vi.fn(); + return child; +} + +describe("copilot runner", () => { + it("removes runner and prepared document before spawning", async () => { + const events: string[] = []; + const child = childProcess(); + let spawnedEnv: NodeJS.ProcessEnv | undefined; + const dependencies: RunDependencies = { + readFile: () => "prompt", + removeFile: (path) => events.push(`remove:${path}`), + spawn: (( + _command: string, + _args: readonly string[], + options: { env: NodeJS.ProcessEnv; stdio: "inherit" }, + ) => { + events.push("spawn"); + spawnedEnv = options.env; + queueMicrotask(() => child.emit("close", 23, null)); + return child; + }) as never, + }; + + await expect( + runInvocation( + prepared(), + "/tmp/prepared.json", + "/tmp/copilot-runner.js", + { + ADO_AW_MODEL_AGENT_COPILOT: "conflicting-model", + ADO_AW_DEFAULT_MODEL_COPILOT: "conflicting-default", + }, + dependencies, + ), + ).resolves.toBe(23); + expect(events).toEqual([ + "remove:/tmp/prepared.json", + "remove:/tmp/copilot-runner.js", + "spawn", + ]); + expect(spawnedEnv).toEqual({ COPILOT_MODEL: "prepared-model" }); + }); + + it("fails closed when self-removal fails", async () => { + const spawn = vi.fn(); + await expect( + runInvocation( + prepared(), + "/tmp/prepared.json", + "/tmp/copilot-runner.js", + {}, + { + readFile: () => "prompt", + removeFile: () => { + throw new Error("EPERM"); + }, + spawn: spawn as never, + }, + ), + ).rejects.toThrow("EPERM"); + expect(spawn).not.toHaveBeenCalled(); + }); + + it("returns deterministic spawn failure and forwards termination signals", async () => { + const child = childProcess(); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const failure = runInvocation( + prepared(), + "/tmp/prepared.json", + "/tmp/copilot-runner.js", + {}, + { + readFile: () => "prompt", + removeFile: () => undefined, + spawn: (() => { + queueMicrotask(() => child.emit("error", new Error("ENOENT"))); + return child; + }) as never, + }, + ); + await expect(failure).resolves.toBe(1); + expect(error).toHaveBeenCalledWith( + "copilot-runner: failed to start Copilot: ENOENT", + ); + error.mockRestore(); + + const signaledChild = childProcess(); + signaledChild.kill = vi.fn((signal: NodeJS.Signals) => { + queueMicrotask(() => signaledChild.emit("close", null, signal)); + return true; + }); + const signaled = runInvocation( + prepared(), + "/tmp/prepared.json", + "/tmp/copilot-runner.js", + {}, + { + readFile: () => "prompt", + removeFile: () => undefined, + spawn: (() => signaledChild) as never, + }, + ); + process.emit("SIGTERM", "SIGTERM"); + await expect(signaled).resolves.toBe(143); + expect(signaledChild.kill).toHaveBeenCalledWith("SIGTERM"); + }); + + it("does not expose controller modes", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + await expect(main(["prepare", "a", "b", "c"], "/tmp/runner.js")).resolves.toBe(2); + await expect(main(["read-result", "a", "agent"], "/tmp/runner.js")).resolves.toBe(2); + expect(error).toHaveBeenCalledWith( + "usage: copilot-runner run ", + ); + error.mockRestore(); + }); +}); diff --git a/scripts/ado-script/src/copilot-runner/index.ts b/scripts/ado-script/src/copilot-runner/index.ts new file mode 100644 index 000000000..7dfbce04a --- /dev/null +++ b/scripts/ado-script/src/copilot-runner/index.ts @@ -0,0 +1,118 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { readFileSync, unlinkSync } from "node:fs"; +import { constants } from "node:os"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + buildChildEnvironment, + buildCopilotArgs, + parsePreparedInvocation, + type PreparedInvocation, +} from "../copilot-shared/protocol.js"; + +interface SpawnLike { + ( + command: string, + args: readonly string[], + options: { + env: NodeJS.ProcessEnv; + stdio: "inherit"; + }, + ): ChildProcess; +} + +export interface RunDependencies { + spawn: SpawnLike; + readFile(path: string): string; + removeFile(path: string): void; +} + +const DEFAULT_DEPENDENCIES: RunDependencies = { + spawn, + readFile: (path) => readFileSync(path, "utf8"), + removeFile: unlinkSync, +}; + +function signalExitCode(signal: NodeJS.Signals): number { + return 128 + (constants.signals[signal] ?? 0); +} + +export async function runInvocation( + document: PreparedInvocation, + preparedPath: string, + runnerPath: string, + env: NodeJS.ProcessEnv = process.env, + dependencies: RunDependencies = DEFAULT_DEPENDENCIES, +): Promise { + const prompt = dependencies.readFile(document.prompt_path); + const args = buildCopilotArgs(document, prompt); + dependencies.removeFile(preparedPath); + dependencies.removeFile(runnerPath); + const child = dependencies.spawn(document.command, args, { + env: buildChildEnvironment(env, document.requested_model), + stdio: "inherit", + }); + + return await new Promise((resolveExit) => { + let settled = false; + const settle = (code: number) => { + if (settled) return; + settled = true; + for (const signal of forwardedSignals) { + process.off(signal, handlers[signal]); + } + resolveExit(code); + }; + const forwardedSignals: NodeJS.Signals[] = ["SIGINT", "SIGTERM", "SIGHUP"]; + const handlers = Object.fromEntries( + forwardedSignals.map((signal) => [ + signal, + () => { + if (!child.killed) child.kill(signal); + }, + ]), + ) as Record void>; + for (const signal of forwardedSignals) { + process.on(signal, handlers[signal]); + } + child.once("error", (error) => { + console.error(`copilot-runner: failed to start Copilot: ${error.message}`); + settle(1); + }); + child.once("close", (code, signal) => { + settle(code ?? (signal ? signalExitCode(signal) : 1)); + }); + }); +} + +export async function main( + argv: string[], + runnerPath = fileURLToPath(import.meta.url), +): Promise { + if (argv[0] !== "run" || argv.length !== 2) { + console.error("usage: copilot-runner run "); + return 2; + } + try { + const document = parsePreparedInvocation(readFileSync(argv[1]!, "utf8")); + return await runInvocation(document, argv[1]!, runnerPath); + } catch (error) { + const message = error instanceof Error ? error.message : "unknown error"; + console.error(`copilot-runner: ${message}`); + return 1; + } +} + +const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; +if (invokedPath === fileURLToPath(import.meta.url)) { + void main(process.argv.slice(2), invokedPath) + .then((code) => { + process.exitCode = code; + }) + .catch((error: unknown) => { + const message = error instanceof Error ? error.message : "unknown error"; + console.error(`copilot-runner: unexpected failure: ${message}`); + process.exitCode = 1; + }); +} diff --git a/scripts/ado-script/src/copilot-shared/protocol.test.ts b/scripts/ado-script/src/copilot-shared/protocol.test.ts new file mode 100644 index 000000000..e637c2a3f --- /dev/null +++ b/scripts/ado-script/src/copilot-shared/protocol.test.ts @@ -0,0 +1,245 @@ +import { mkdtempSync, readdirSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + buildChildEnvironment, + buildCopilotArgs, + parseInvocationRequest, + parseInvocationResult, + parsePreparedInvocation, + prepareInvocation, + resolveRequestedModel, + writeJsonAtomic, + type InvocationRequest, + type PreparedInvocation, +} from "./protocol.js"; + +function request( + overrides: Partial = {}, +): InvocationRequest { + return { + schema_version: 2, + document_kind: "request", + role: "agent", + command: "/tmp/awf-tools/copilot", + prompt_path: "/tmp/awf-tools/agent-prompt.md", + mcp_config_path: "/tmp/awf-tools/mcp-config.json", + args: ["--disable-builtin-mcps", "--allow-tool", "shell(cat *)"], + explicit_model: null, + ...overrides, + }; +} + +function prepared( + overrides: Partial = {}, +): PreparedInvocation { + return { + schema_version: 2, + document_kind: "prepared", + role: "agent", + command: "/tmp/awf-tools/copilot", + prompt_path: "/tmp/awf-tools/agent-prompt.md", + mcp_config_path: "/tmp/awf-tools/mcp-config.json", + args: ["--disable-builtin-mcps", "--allow-tool", "shell(cat *)"], + requested_model: null, + ...overrides, + }; +} + +describe("Copilot invocation protocol", () => { + it("parses strict request, prepared, and result documents", () => { + expect(parseInvocationRequest(JSON.stringify(request()))).toEqual(request()); + expect(parsePreparedInvocation(JSON.stringify(prepared()))).toEqual(prepared()); + expect( + parseInvocationResult( + JSON.stringify({ + schema_version: 2, + document_kind: "result", + role: "detection", + requested_model: "detector", + }), + ), + ).toEqual({ + schema_version: 2, + document_kind: "result", + role: "detection", + requested_model: "detector", + }); + }); + + it.each([ + [{ ...request(), schema_version: 1 }, "unsupported invocation request schema"], + [{ ...request(), document_kind: "prepared" }, "must be 'request'"], + [{ ...request(), unexpected: true }, "unknown field 'unexpected'"], + [{ ...request(), role: "other" }, "must be 'agent' or 'detection'"], + [{ ...request(), command: "copilot;sh" }, "field 'command' is invalid"], + [{ ...request(), command: "bin/copilot" }, "field 'command' is invalid"], + [{ ...request(), command: "/tmp/../copilot" }, "field 'command' is invalid"], + [{ ...request(), command: "/tmp//copilot" }, "field 'command' is invalid"], + [{ ...request(), command: "/tmp/copilot/" }, "field 'command' is invalid"], + [{ ...request(), prompt_path: "relative.md" }, "field 'prompt_path' is invalid"], + [{ ...request(), args: [1] }, "field 'args' must be an array of strings"], + [{ ...request(), explicit_model: "bad model" }, "field 'explicit_model' is invalid"], + ])("rejects malformed requests %#", (value, message) => { + expect(() => parseInvocationRequest(JSON.stringify(value))).toThrow(message); + }); + + it("rejects request/prepared document-kind confusion", () => { + expect(() => parsePreparedInvocation(JSON.stringify(request()))).toThrow( + "unknown field 'explicit_model'", + ); + expect(() => parseInvocationRequest(JSON.stringify(prepared()))).toThrow( + "unknown field 'requested_model'", + ); + }); + + it.each([ + [{ ...prepared(), schema_version: 1 }, "unsupported prepared invocation schema"], + [{ ...prepared(), document_kind: "request" }, "must be 'prepared'"], + [{ ...prepared(), unexpected: true }, "unknown field 'unexpected'"], + [{ ...prepared(), requested_model: "bad model" }, "field 'requested_model' is invalid"], + ])("rejects malformed prepared invocations %#", (value, message) => { + expect(() => parsePreparedInvocation(JSON.stringify(value))).toThrow(message); + }); + + it.each([ + [ + { + schema_version: 1, + document_kind: "result", + role: "agent", + requested_model: null, + }, + "unsupported invocation result schema", + ], + [ + { + schema_version: 2, + document_kind: "prepared", + role: "agent", + requested_model: null, + }, + "must be 'result'", + ], + [ + { + schema_version: 2, + document_kind: "result", + role: "agent", + requested_model: null, + unexpected: true, + }, + "unknown field 'unexpected'", + ], + ])("rejects malformed invocation results %#", (value, message) => { + expect(() => parseInvocationResult(JSON.stringify(value))).toThrow(message); + }); + + it("writes JSON atomically without leaving a temporary sibling", () => { + const directory = mkdtempSync(join(tmpdir(), "copilot-protocol-")); + const path = join(directory, "result.json"); + writeJsonAtomic(path, { + schema_version: 2, + document_kind: "result", + role: "agent", + requested_model: null, + }); + expect(JSON.parse(readFileSync(path, "utf8"))).toMatchObject({ + document_kind: "result", + role: "agent", + }); + expect(readdirSync(directory)).toEqual(["result.json"]); + }); +}); + +describe("model preparation", () => { + it("keeps explicit model authoritative", () => { + expect( + resolveRequestedModel(request({ explicit_model: "frontmatter-model" }), { + ADO_AW_MODEL_AGENT_COPILOT: "role-model", + ADO_AW_DEFAULT_MODEL_COPILOT: "default-model", + }), + ).toBe("frontmatter-model"); + }); + + it("uses role-specific then shared values and ignores unresolved macros", () => { + expect( + resolveRequestedModel(request(), { + ADO_AW_MODEL_AGENT_COPILOT: "role-model", + ADO_AW_DEFAULT_MODEL_COPILOT: "default-model", + }), + ).toBe("role-model"); + expect( + resolveRequestedModel(request({ role: "detection" }), { + ADO_AW_MODEL_DETECTION_COPILOT: "", + ADO_AW_DEFAULT_MODEL_COPILOT: "default-model", + }), + ).toBe("default-model"); + expect( + resolveRequestedModel(request(), { + ADO_AW_MODEL_AGENT_COPILOT: "$(ADO_AW_MODEL_AGENT_COPILOT)", + ADO_AW_DEFAULT_MODEL_COPILOT: "$(ADO_AW_DEFAULT_MODEL_COPILOT)", + }), + ).toBeNull(); + }); + + it("rejects invalid runtime values without printing them", () => { + expect(() => + resolveRequestedModel(request(), { + ADO_AW_MODEL_AGENT_COPILOT: "secret value", + }), + ).toThrow("contains invalid characters"); + try { + resolveRequestedModel(request(), { + ADO_AW_MODEL_AGENT_COPILOT: "secret value", + }); + } catch (error) { + expect(String(error)).not.toContain("secret value"); + } + }); + + it("emits identical requested models in prepared and result documents", () => { + const { prepared: execution, result } = prepareInvocation(request(), { + ADO_AW_MODEL_AGENT_COPILOT: "runtime-model", + }); + expect(execution.requested_model).toBe("runtime-model"); + expect(result.requested_model).toBe(execution.requested_model); + expect(execution).not.toHaveProperty("explicit_model"); + expect(execution).not.toHaveProperty("result_path"); + }); +}); + +describe("argv and child environment", () => { + it("preserves prompt, MCP config, and authored values as argv elements", () => { + expect(buildCopilotArgs(prepared(), "line one\nline two")).toEqual([ + "--prompt=line one\nline two", + "--additional-mcp-config", + "@/tmp/awf-tools/mcp-config.json", + "--disable-builtin-mcps", + "--allow-tool", + "shell(cat *)", + ]); + }); + + it("rejects NUL bytes in prompt content", () => { + expect(() => buildCopilotArgs(prepared(), "before\0after")).toThrow( + "prompt contains an invalid NUL byte", + ); + }); + + it("uses only the prepared model and removes runtime selector variables", () => { + const original = { + KEEP: "yes", + COPILOT_MODEL: "old", + ADO_AW_MODEL_AGENT_COPILOT: "conflicting", + ADO_AW_DEFAULT_MODEL_COPILOT: "conflicting-default", + }; + expect(buildChildEnvironment(original, "selected")).toEqual({ + KEEP: "yes", + COPILOT_MODEL: "selected", + }); + expect(original.COPILOT_MODEL).toBe("old"); + }); +}); diff --git a/scripts/ado-script/src/copilot-shared/protocol.ts b/scripts/ado-script/src/copilot-shared/protocol.ts new file mode 100644 index 000000000..b39b41180 --- /dev/null +++ b/scripts/ado-script/src/copilot-shared/protocol.ts @@ -0,0 +1,385 @@ +import { renameSync, writeFileSync } from "node:fs"; + +export const SCHEMA_VERSION = 2; + +const MODEL_PATTERN = /^[A-Za-z0-9._:-]+$/; +const COMMAND_PATTERN = /^[A-Za-z0-9._/-]+$/; +const REQUEST_KEYS = new Set([ + "schema_version", + "document_kind", + "role", + "command", + "prompt_path", + "mcp_config_path", + "args", + "explicit_model", +]); +const PREPARED_KEYS = new Set([ + "schema_version", + "document_kind", + "role", + "command", + "prompt_path", + "mcp_config_path", + "args", + "requested_model", +]); +const RESULT_KEYS = new Set([ + "schema_version", + "document_kind", + "role", + "requested_model", +]); + +export type InvocationRole = "agent" | "detection"; + +export interface InvocationRequest { + schema_version: 2; + document_kind: "request"; + role: InvocationRole; + command: string; + prompt_path: string; + mcp_config_path: string | null; + args: string[]; + explicit_model: string | null; +} + +export interface PreparedInvocation { + schema_version: 2; + document_kind: "prepared"; + role: InvocationRole; + command: string; + prompt_path: string; + mcp_config_path: string | null; + args: string[]; + requested_model: string | null; +} + +export interface InvocationResult { + schema_version: 2; + document_kind: "result"; + role: InvocationRole; + requested_model: string | null; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseObject(raw: string, label: string): Record { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error(`${label} is not valid JSON`); + } + if (!isRecord(parsed)) { + throw new Error(`${label} must be a JSON object`); + } + return parsed; +} + +function rejectUnknown( + parsed: Record, + allowed: ReadonlySet, + label: string, +): void { + const unknown = Object.keys(parsed).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + throw new Error(`${label} contains unknown field '${unknown.sort()[0]}'`); + } +} + +function requireVersionAndKind( + parsed: Record, + kind: "request" | "prepared" | "result", + label: string, +): void { + if (parsed.schema_version !== SCHEMA_VERSION) { + throw new Error( + `unsupported ${label} schema version '${String(parsed.schema_version)}'`, + ); + } + if (parsed.document_kind !== kind) { + throw new Error(`${label} field 'document_kind' must be '${kind}'`); + } +} + +function parseRole( + parsed: Record, + label: string, +): InvocationRole { + if (parsed.role !== "agent" && parsed.role !== "detection") { + throw new Error(`${label} field 'role' must be 'agent' or 'detection'`); + } + return parsed.role; +} + +function requiredString( + value: Record, + key: string, + label: string, + validator?: (input: string) => boolean, +): string { + const candidate = value[key]; + if (typeof candidate !== "string" || candidate.length === 0) { + throw new Error(`${label} field '${key}' must be a non-empty string`); + } + if (candidate.includes("\0") || (validator && !validator(candidate))) { + throw new Error(`${label} field '${key}' is invalid`); + } + return candidate; +} + +function nullableString( + value: Record, + key: string, + label: string, + validator?: (input: string) => boolean, +): string | null { + const candidate = value[key]; + if (candidate === null) return null; + if (typeof candidate !== "string" || candidate.includes("\0")) { + throw new Error(`${label} field '${key}' must be a string or null`); + } + if (validator && !validator(candidate)) { + throw new Error(`${label} field '${key}' is invalid`); + } + return candidate; +} + +function parseArgs( + parsed: Record, + label: string, +): string[] { + if ( + !Array.isArray(parsed.args) || + !parsed.args.every((arg) => typeof arg === "string") + ) { + throw new Error(`${label} field 'args' must be an array of strings`); + } + if (parsed.args.some((arg) => arg.includes("\0"))) { + throw new Error(`${label} field 'args' contains an invalid NUL byte`); + } + return [...parsed.args]; +} + +function hasSafePathSegments(value: string, allowBareCommand: boolean): boolean { + if ( + value.includes("\n") || + value.includes("\r") || + value.includes(":") || + value.endsWith("/") + ) { + return false; + } + if (allowBareCommand && !value.includes("/")) { + return value !== "." && value !== ".."; + } + if (!value.startsWith("/")) return false; + const segments = value.slice(1).split("/"); + return ( + segments.length > 0 && + segments.every( + (segment) => segment.length > 0 && segment !== "." && segment !== "..", + ) + ); +} + +function isAbsoluteContainerPath(value: string): boolean { + return hasSafePathSegments(value, false); +} + +function isSafeCommand(value: string): boolean { + return COMMAND_PATTERN.test(value) && hasSafePathSegments(value, true); +} + +export function parseInvocationRequest(raw: string): InvocationRequest { + const label = "invocation request"; + const parsed = parseObject(raw, label); + rejectUnknown(parsed, REQUEST_KEYS, label); + requireVersionAndKind(parsed, "request", label); + return { + schema_version: SCHEMA_VERSION, + document_kind: "request", + role: parseRole(parsed, label), + command: requiredString(parsed, "command", label, isSafeCommand), + prompt_path: requiredString( + parsed, + "prompt_path", + label, + isAbsoluteContainerPath, + ), + mcp_config_path: nullableString( + parsed, + "mcp_config_path", + label, + isAbsoluteContainerPath, + ), + args: parseArgs(parsed, label), + explicit_model: nullableString( + parsed, + "explicit_model", + label, + (value) => MODEL_PATTERN.test(value), + ), + }; +} + +export function parsePreparedInvocation(raw: string): PreparedInvocation { + const label = "prepared invocation"; + const parsed = parseObject(raw, label); + rejectUnknown(parsed, PREPARED_KEYS, label); + requireVersionAndKind(parsed, "prepared", label); + return { + schema_version: SCHEMA_VERSION, + document_kind: "prepared", + role: parseRole(parsed, label), + command: requiredString(parsed, "command", label, isSafeCommand), + prompt_path: requiredString( + parsed, + "prompt_path", + label, + isAbsoluteContainerPath, + ), + mcp_config_path: nullableString( + parsed, + "mcp_config_path", + label, + isAbsoluteContainerPath, + ), + args: parseArgs(parsed, label), + requested_model: nullableString( + parsed, + "requested_model", + label, + (value) => MODEL_PATTERN.test(value), + ), + }; +} + +export function parseInvocationResult(raw: string): InvocationResult { + const label = "invocation result"; + const parsed = parseObject(raw, label); + rejectUnknown(parsed, RESULT_KEYS, label); + requireVersionAndKind(parsed, "result", label); + return { + schema_version: SCHEMA_VERSION, + document_kind: "result", + role: parseRole(parsed, label), + requested_model: nullableString( + parsed, + "requested_model", + label, + (value) => MODEL_PATTERN.test(value), + ), + }; +} + +function runtimeModelVariable(role: InvocationRole): string { + return role === "agent" + ? "ADO_AW_MODEL_AGENT_COPILOT" + : "ADO_AW_MODEL_DETECTION_COPILOT"; +} + +function isUnresolvedAdoMacro(value: string, variable: string): boolean { + return value === `$(${variable})`; +} + +export function resolveRequestedModel( + request: InvocationRequest, + env: NodeJS.ProcessEnv, +): string | null { + if (request.explicit_model !== null) { + return request.explicit_model; + } + const specific = runtimeModelVariable(request.role); + const candidates: Array<[string, string | undefined]> = [ + [specific, env[specific]], + ["ADO_AW_DEFAULT_MODEL_COPILOT", env.ADO_AW_DEFAULT_MODEL_COPILOT], + ]; + for (const [variable, candidate] of candidates) { + if ( + candidate === undefined || + candidate.length === 0 || + isUnresolvedAdoMacro(candidate, variable) + ) { + continue; + } + if (!MODEL_PATTERN.test(candidate)) { + throw new Error( + `runtime Copilot model from ${specific}/ADO_AW_DEFAULT_MODEL_COPILOT contains invalid characters`, + ); + } + return candidate; + } + return null; +} + +export function prepareInvocation( + request: InvocationRequest, + env: NodeJS.ProcessEnv, +): { prepared: PreparedInvocation; result: InvocationResult } { + const requestedModel = resolveRequestedModel(request, env); + return { + prepared: { + schema_version: SCHEMA_VERSION, + document_kind: "prepared", + role: request.role, + command: request.command, + prompt_path: request.prompt_path, + mcp_config_path: request.mcp_config_path, + args: [...request.args], + requested_model: requestedModel, + }, + result: { + schema_version: SCHEMA_VERSION, + document_kind: "result", + role: request.role, + requested_model: requestedModel, + }, + }; +} + +export function buildCopilotArgs( + document: PreparedInvocation, + prompt: string, +): string[] { + if (prompt.includes("\0")) { + throw new Error("prompt contains an invalid NUL byte"); + } + const args = [`--prompt=${prompt}`]; + if (document.mcp_config_path !== null) { + args.push("--additional-mcp-config", `@${document.mcp_config_path}`); + } + args.push(...document.args); + return args; +} + +export function buildChildEnvironment( + env: NodeJS.ProcessEnv, + requestedModel: string | null, +): NodeJS.ProcessEnv { + const childEnv = { ...env }; + delete childEnv.ADO_AW_MODEL_AGENT_COPILOT; + delete childEnv.ADO_AW_MODEL_DETECTION_COPILOT; + delete childEnv.ADO_AW_DEFAULT_MODEL_COPILOT; + if (requestedModel === null) { + delete childEnv.COPILOT_MODEL; + } else { + childEnv.COPILOT_MODEL = requestedModel; + } + return childEnv; +} + +export function writeJsonAtomic( + path: string, + value: PreparedInvocation | InvocationResult, +): void { + const temporary = `${path}.tmp-${process.pid}`; + writeFileSync(temporary, `${JSON.stringify(value)}\n`, { + encoding: "utf8", + mode: 0o600, + }); + renameSync(temporary, path); +} diff --git a/scripts/ado-script/test/azure-wif-isolation.test.ts b/scripts/ado-script/test/azure-wif-isolation.test.ts index d8ae17c2b..8bf3b2194 100644 --- a/scripts/ado-script/test/azure-wif-isolation.test.ts +++ b/scripts/ado-script/test/azure-wif-isolation.test.ts @@ -218,8 +218,12 @@ describe.skipIf(!awfEnabled)("Azure WIF real AWF boundary", () => { mkdirSync(awfTools, { recursive: true }); mkdirSync(join(adoScripts, "ado-script"), { recursive: true }); copyFileSync( - resolve(testDir, "../copilot-invoker.js"), - join(adoScripts, "ado-script/copilot-invoker.js"), + resolve(testDir, "../copilot-controller.js"), + join(adoScripts, "ado-script/copilot-controller.js"), + ); + copyFileSync( + resolve(testDir, "../copilot-runner.js"), + join(adoScripts, "ado-script/copilot-runner.js"), ); mkdirSync(join(auth, "token.d"), { recursive: true }); chmodSync(join(temp, "ado-aw-azure-auth"), 0o700); @@ -233,14 +237,16 @@ describe.skipIf(!awfEnabled)("Azure WIF real AWF boundary", () => { .find((step) => step.bash?.includes("AWF_ARGS+=(--skip-pull --env-all)")); if (!runStep?.bash) throw new Error("compiled AWF invocation is missing"); expect(runStep.bash).toContain( - "copilot-invoker.js run /tmp/awf-tools/copilot-invocation.json", + "copilot-runner.js run /tmp/awf-tools/copilot-invocation.json", ); const capture = join(directory, "awf-args"); - const invocationResult = join(awfTools, "copilot-invocation-result.json"); + const controllerSource = join(adoScripts, "ado-script/copilot-controller.js"); + const forgedResult = join(awfTools, "copilot-invocation-result.json"); writeFileSync(join(tools, "awf/awf"), `#!/bin/sh if [ "$1" = logs ]; then exit 0; fi printf '%s\\0' "$@" > '${capture}' -printf '%s\\n' '{"schema_version":1,"role":"agent","requested_model":null}' > '${invocationResult}' +printf '%s\\n' 'throw new Error("sandbox controller executed on host")' > '${controllerSource}' +printf '%s\\n' '{"schema_version":2,"document_kind":"result","role":"agent","requested_model":"forged"}' > '${forgedResult}' `, { mode: 0o755 }); const script = runStep.bash .replaceAll("$(Agent.TempDirectory)", temp) @@ -257,6 +263,22 @@ printf '%s\\n' '{"schema_version":1,"role":"agent","requested_model":null}' > '$ }; for (const name of identities) env[name] = "synthetic-identity"; run("bash", ["-c", script], env); + expect(readFileSync(controllerSource, "utf8")).toContain( + "sandbox controller executed on host", + ); + expect(JSON.parse(readFileSync(forgedResult, "utf8"))).toMatchObject({ + document_kind: "result", + requested_model: "forged", + }); + expect(JSON.parse(readFileSync( + join(temp, "ado-aw-copilot-controller/invocation-result.json"), + "utf8", + ))).toEqual({ + schema_version: 2, + document_kind: "result", + role: "agent", + requested_model: null, + }); const captured = readFileSync(capture, "utf8").split("\0").filter(Boolean); const version = captured[captured.indexOf("--image-tag") + 1]; expect(version).toMatch(/^\d+\.\d+\.\d+$/); @@ -276,7 +298,7 @@ printf '%s\\n' '{"schema_version":1,"role":"agent","requested_model":null}' > '$ const commandIndex = captured.indexOf("--"); expect(commandIndex).toBeGreaterThan(0); expect(captured[commandIndex + 1]).toContain( - `copilot-invoker.js run ${join(awfTools, "copilot-invocation.json")}`, + `copilot-runner.js run ${join(awfTools, "copilot-invocation.json")}`, ); const args: string[] = []; for (let i = 0; i < commandIndex; i++) { diff --git a/scripts/ado-script/test/smoke.test.ts b/scripts/ado-script/test/smoke.test.ts index f03b5c188..0b539d21d 100644 --- a/scripts/ado-script/test/smoke.test.ts +++ b/scripts/ado-script/test/smoke.test.ts @@ -28,7 +28,8 @@ const gateBundlePath = resolve(__dirname, "../gate.js"); const importBundlePath = resolve(__dirname, "../import.js"); const execContextPrBundlePath = resolve(__dirname, "../exec-context-pr.js"); const preparePrBaseBundlePath = resolve(__dirname, "../prepare-pr-base.js"); -const copilotInvokerBundlePath = resolve(__dirname, "../copilot-invoker.js"); +const copilotControllerBundlePath = resolve(__dirname, "../copilot-controller.js"); +const copilotRunnerBundlePath = resolve(__dirname, "../copilot-runner.js"); const gateFixturePath = resolve( __dirname, "fixtures/gate-spec-pr-title-match.json", @@ -133,14 +134,17 @@ describe("import.js smoke", () => { }, 20000); }); -describe.skipIf(process.platform === "win32")("copilot-invoker.js smoke", () => { - it("runs a fake Copilot child and publishes the requested model", () => { - withSmokeScratchDir("copilot-invoker", (dir) => { +describe.skipIf(process.platform === "win32")("Copilot controller/runner smoke", () => { + it("prepares, runs, self-removes, and preserves trusted model metadata", () => { + withSmokeScratchDir("copilot-runner", (dir) => { const command = resolve(dir, "fake-copilot"); const promptPath = resolve(dir, "prompt.md"); const resultPath = resolve(dir, "result.json"); const capturePath = resolve(dir, "capture.json"); - const documentPath = resolve(dir, "invocation.json"); + const requestPath = resolve(dir, "request.json"); + const preparedPath = resolve(dir, "prepared.json"); + const runnerPath = resolve(dir, "copilot-runner.js"); + copyFileSync(copilotRunnerBundlePath, runnerPath); writeFileSync( command, `#!/bin/sh @@ -151,22 +155,34 @@ exit 7 chmodSync(command, 0o755); writeFileSync(promptPath, "smoke prompt\n"); writeFileSync( - documentPath, + requestPath, JSON.stringify({ - schema_version: 1, + schema_version: 2, + document_kind: "request", role: "agent", command, prompt_path: promptPath, mcp_config_path: null, args: ["--no-ask-user"], explicit_model: "gpt-smoke", - result_path: resultPath, }), ); + const prepare = spawnSync( + process.execPath, + [ + copilotControllerBundlePath, + "prepare", + requestPath, + preparedPath, + resultPath, + ], + { env: { ...process.env }, encoding: "utf8" }, + ); + expect(prepare.status).toBe(0); const run = spawnSync( process.execPath, - [copilotInvokerBundlePath, "run", documentPath], + [runnerPath, "run", preparedPath], { env: { ...process.env, CAPTURE_PATH: capturePath }, encoding: "utf8", @@ -175,12 +191,15 @@ exit 7 expect(run.status).toBe(7); expect(run.stdout).toBe(""); expect(run.stderr).toBe(""); + expect(existsSync(runnerPath)).toBe(false); + expect(existsSync(preparedPath)).toBe(false); expect(JSON.parse(readFileSync(capturePath, "utf8"))).toEqual({ argv: ["--prompt=smoke prompt\n", "--no-ask-user"], model: "gpt-smoke", }); expect(JSON.parse(readFileSync(resultPath, "utf8"))).toEqual({ - schema_version: 1, + schema_version: 2, + document_kind: "result", role: "agent", requested_model: "gpt-smoke", }); diff --git a/src/compile/ado_bundle.rs b/src/compile/ado_bundle.rs index f795ea469..69684e984 100644 --- a/src/compile/ado_bundle.rs +++ b/src/compile/ado_bundle.rs @@ -53,10 +53,11 @@ pub enum Bundle { ExecContextRepo, ApprovalSummary, Conclusion, - /// Copilot process harness executed inside AWF. It receives only a - /// compiler-generated invocation document and the already-filtered Agent - /// environment, so it requires no ADO bearer. - CopilotInvoker, + /// Trusted Copilot invocation controller. Runs only on the pipeline host + /// for preflight preparation and postflight result validation. + CopilotController, + /// Minimal Copilot process runner executed only inside AWF. + CopilotRunner, /// GitHub App installation-token minter/revoker (issue #1316). Runs before /// the Copilot invocation in the Agent and Detection jobs. It authenticates /// to the **GitHub** API (not ADO REST), so it needs no ADO bearer. @@ -159,7 +160,8 @@ impl Bundle { Bundle::ExecContextRepo, Bundle::ApprovalSummary, Bundle::Conclusion, - Bundle::CopilotInvoker, + Bundle::CopilotController, + Bundle::CopilotRunner, Bundle::GithubAppToken, Bundle::PreparePrBase, Bundle::AzureWifRefresh, @@ -188,7 +190,8 @@ impl Bundle { Bundle::ExecContextRepo => paths::EXEC_CONTEXT_REPO_PATH, Bundle::ApprovalSummary => paths::APPROVAL_SUMMARY_PATH, Bundle::Conclusion => paths::CONCLUSION_PATH, - Bundle::CopilotInvoker => paths::COPILOT_INVOKER_PATH, + Bundle::CopilotController => paths::COPILOT_CONTROLLER_PATH, + Bundle::CopilotRunner => paths::COPILOT_RUNNER_PATH, Bundle::GithubAppToken => paths::GITHUB_APP_TOKEN_PATH, Bundle::PreparePrBase => paths::PREPARE_PR_BASE_PATH, Bundle::AzureWifRefresh => paths::AZURE_WIF_REFRESH_PATH, @@ -218,7 +221,8 @@ impl Bundle { | Bundle::ExecContextManual | Bundle::ExecContextRepo | Bundle::ApprovalSummary - | Bundle::CopilotInvoker + | Bundle::CopilotController + | Bundle::CopilotRunner // Authenticates to the GitHub API with its own App JWT / minted // token, not the ADO bearer. | Bundle::GithubAppToken diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index e5645a7a5..5b71934a8 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -293,17 +293,16 @@ fn build_engine_setup( let compiler_version = env!("CARGO_PKG_VERSION").to_string(); let detection_engine = crate::engine::get_engine(detection_engine_config.engine_id())?; - let agent_invocation = ctx.engine.invocation_document( + let agent_invocation = ctx.engine.invocation_request( ctx.front_matter, extension_declarations, crate::engine::CopilotInvocationContext::new( crate::engine::RuntimeModelRole::Agent, "/tmp/awf-tools/agent-prompt.md", Some("/tmp/awf-tools/mcp-config.json"), - "/tmp/awf-tools/copilot-invocation-result.json", ), )?; - let detection_invocation = detection_engine.invocation_document_with_config( + let detection_invocation = detection_engine.invocation_request_with_config( detection_engine_config, ctx.front_matter, extension_declarations, @@ -311,13 +310,12 @@ fn build_engine_setup( crate::engine::RuntimeModelRole::Detection, "/tmp/awf-tools/threat-analysis-prompt.md", None, - "/tmp/awf-tools/copilot-invocation-result.json", ), )?; let agent_invocation_json = serde_json::to_string(&agent_invocation) - .context("failed to serialize Agent Copilot invocation document")?; + .context("failed to serialize Agent Copilot invocation request")?; let detection_invocation_json = serde_json::to_string(&detection_invocation) - .context("failed to serialize Detection Copilot invocation document")?; + .context("failed to serialize Detection Copilot invocation request")?; let engine_install_steps_yaml = ctx.engine .install_steps(&front_matter.engine, &front_matter.target, ctx.ado_org())?; @@ -1606,9 +1604,10 @@ fn build_detection_job( )?)); steps.push(Step::Bash(setup_compiler_step())); - // Detection always executes the Copilot invoker bundle. Stage it before - // custom pre-steps; mint optional credentials only after those steps so - // trusted setup code receives the least privilege needed. + // Detection always executes the Copilot controller/runner bundles. + // Stage them before custom pre-steps; mint optional credentials only + // after those steps so trusted setup code receives the least privilege + // needed. steps.extend( super::extensions::ado_script::install_and_download_steps_typed( front_matter.supply_chain(), @@ -4440,6 +4439,21 @@ fn awf_exclude_env_flags(exclude_keys: &[String]) -> String { block } +const COPILOT_TRUSTED_PREFLIGHT: &str = r#"TRUSTED_CONTROLLER_DIR="$AGENT_TEMP/ado-aw-copilot-controller" +TRUSTED_CONTROLLER_PATH="$TRUSTED_CONTROLLER_DIR/copilot-controller.js" +TRUSTED_REQUEST_PATH="$TRUSTED_CONTROLLER_DIR/invocation-request.json" +TRUSTED_RESULT_PATH="$TRUSTED_CONTROLLER_DIR/invocation-result.json" +install -d -m 0700 "$TRUSTED_CONTROLLER_DIR" +install -m 0500 "$COPILOT_CONTROLLER_SOURCE_PATH" "$TRUSTED_CONTROLLER_PATH" +printf '%s\n' "$INVOCATION_REQUEST" > "$TRUSTED_REQUEST_PATH" +chmod 0600 "$TRUSTED_REQUEST_PATH" +rm -f "$SANDBOX_INVOCATION_PATH" "$TRUSTED_RESULT_PATH" +node "$TRUSTED_CONTROLLER_PATH" prepare \ + "$TRUSTED_REQUEST_PATH" \ + "$SANDBOX_INVOCATION_PATH" \ + "$TRUSTED_RESULT_PATH" +rm -f "$COPILOT_CONTROLLER_SOURCE_PATH""#; + shell_script! { /// Invoke the AI agent inside AWF's network-isolated Docker topology. /// @@ -4452,37 +4466,47 @@ shell_script! { /// - `image_flags` — `--image-tag` plus optional `--image-registry` /// - `exclude_env` — provider credentials and internal MCP identity keys /// - `awf_mounts` — the compiler-supplied chain of `--mount "…"` args - /// - `routed_invoker` — the fixed single-quoted `NO_PROXY` prefix + - /// compiler-owned invoker command that AWF runs inside the sandbox + /// - `routed_runner` — the fixed single-quoted `NO_PROXY` prefix + + /// compiler-owned runner command that AWF runs inside the sandbox RUN_AGENT { interpreter: Bash, bindings: [ AGENT_TEMP, PIPELINE_WORKSPACE, ALLOWED_DOMAINS, - INVOCATION_DOCUMENT, - INVOCATION_DOCUMENT_PATH, - INVOCATION_RESULT_PATH, - COPILOT_INVOKER_PATH + INVOCATION_REQUEST, + SANDBOX_INVOCATION_PATH, + COPILOT_CONTROLLER_SOURCE_PATH + ], + externals: [ + WORKING_DIRECTORY, + TRUSTED_CONTROLLER_PATH, + TRUSTED_RESULT_PATH ], - externals: [WORKING_DIRECTORY], fragments: [ + trusted_preflight, append_aw_info_field, topology_attach, image_flags, exclude_env, awf_mounts, - routed_invoker + routed_runner ], phases: [append_aw_info_field = super::extensions::APPEND_AW_INFO_FIELD], + fragment_uses: [ + trusted_preflight => [ + INVOCATION_REQUEST, + SANDBOX_INVOCATION_PATH, + COPILOT_CONTROLLER_SOURCE_PATH + ], + ], body: r###" set -eo pipefail AGENT_OUTPUT_FILE="$AGENT_TEMP/staging/logs/agent-output.txt" mkdir -p "$AGENT_TEMP/staging/logs" AGENT_EXIT_CODE=0 -printf '%s\n' "$INVOCATION_DOCUMENT" > "$INVOCATION_DOCUMENT_PATH" -rm -f "$INVOCATION_RESULT_PATH" +# ado-aw:fragment trusted_preflight echo "=== Running AI agent with AWF network isolation ===" echo "Allowed domains: $ALLOWED_DOMAINS" @@ -4510,7 +4534,7 @@ AWF_ARGS+=( --log-level info --proxy-logs-dir "$AGENT_TEMP/staging/logs/firewall" ) -# ado-aw:fragment routed_invoker +# ado-aw:fragment routed_runner # Stream agent output in real-time while filtering VSO commands. # sed -u = unbuffered (line-by-line) so output appears immediately. @@ -4523,7 +4547,7 @@ AWF_ARGS+=( || AGENT_EXIT_CODE=$? MODEL_RESULT_STATUS=0 -REQUESTED_MODEL=$(node "$COPILOT_INVOKER_PATH" read-result "$INVOCATION_RESULT_PATH" agent) \ +REQUESTED_MODEL=$(node "$TRUSTED_CONTROLLER_PATH" read-result "$TRUSTED_RESULT_PATH" agent) \ || MODEL_RESULT_STATUS=$? if [ "$MODEL_RESULT_STATUS" -ne 0 ]; then echo "ERROR: Agent Copilot invocation result is missing or malformed" >&2 @@ -4559,7 +4583,7 @@ fn run_agent_step( allowed_domains: &str, awf_mounts: &str, working_directory: &str, - invocation_document: &str, + invocation_request: &str, engine_env: &str, byom_exclude_keys: &[String], supply_chain: Option<&SupplyChainConfig>, @@ -4655,10 +4679,10 @@ fn run_agent_step( } else { MCPG_CONTAINER_NAME.to_string() }; - let routed_invoker = format!( + let routed_runner = format!( "AWF_ARGS+=(-- 'export NO_PROXY=\"${{NO_PROXY:+$NO_PROXY,}}{no_proxy_peers}\"; \ export no_proxy=\"$NO_PROXY\"; exec node {} run /tmp/awf-tools/copilot-invocation.json')", - super::extensions::ado_script::COPILOT_INVOKER_PATH + super::extensions::ado_script::COPILOT_RUNNER_PATH ); let mut step = ShellScript::new(&RUN_AGENT) @@ -4669,21 +4693,18 @@ fn run_agent_step( ) .bind_text("ALLOWED_DOMAINS", allowed_domains) .bind( - "INVOCATION_DOCUMENT", - Binding::document(invocation_document), + "INVOCATION_REQUEST", + Binding::document(invocation_request), ) .bind_text( - "INVOCATION_DOCUMENT_PATH", + "SANDBOX_INVOCATION_PATH", "/tmp/awf-tools/copilot-invocation.json", ) .bind_text( - "INVOCATION_RESULT_PATH", - "/tmp/awf-tools/copilot-invocation-result.json", - ) - .bind_text( - "COPILOT_INVOKER_PATH", - super::extensions::ado_script::COPILOT_INVOKER_PATH, + "COPILOT_CONTROLLER_SOURCE_PATH", + super::extensions::ado_script::COPILOT_CONTROLLER_PATH, ) + .fragment("trusted_preflight", COPILOT_TRUSTED_PREFLIGHT) .fragment( "append_aw_info_field", phase_body(&super::extensions::APPEND_AW_INFO_FIELD), @@ -4692,7 +4713,7 @@ fn run_agent_step( .fragment("image_flags", image_flags_line) .fragment("exclude_env", exclude_env_line) .fragment("awf_mounts", awf_mounts_block) - .fragment("routed_invoker", routed_invoker) + .fragment("routed_runner", routed_runner) .into_step("Run copilot (AWF network isolated)"); step.working_directory = Some(working_directory.to_string()); // Engine env comes as a multi-line YAML env block — `KEY: VALUE` lines @@ -6264,21 +6285,30 @@ shell_script! { AGENT_TEMP, PIPELINE_WORKSPACE, ALLOWED_DOMAINS, - INVOCATION_DOCUMENT, - INVOCATION_DOCUMENT_PATH, - INVOCATION_RESULT_PATH, - COPILOT_INVOKER_PATH + INVOCATION_REQUEST, + SANDBOX_INVOCATION_PATH, + COPILOT_CONTROLLER_SOURCE_PATH + ], + externals: [ + WORKING_DIRECTORY, + TRUSTED_CONTROLLER_PATH, + TRUSTED_RESULT_PATH + ], + fragments: [trusted_preflight, image_flags, exclude_env, run_runner], + fragment_uses: [ + trusted_preflight => [ + INVOCATION_REQUEST, + SANDBOX_INVOCATION_PATH, + COPILOT_CONTROLLER_SOURCE_PATH + ], ], - externals: [WORKING_DIRECTORY], - fragments: [image_flags, exclude_env, run_invoker], body: r###" -set -o pipefail +set -eo pipefail # Run threat analysis with AWF network isolation THREAT_OUTPUT_FILE="$AGENT_TEMP/threat-analysis-output.txt" AGENT_EXIT_CODE=0 -printf '%s\n' "$INVOCATION_DOCUMENT" > "$INVOCATION_DOCUMENT_PATH" -rm -f "$INVOCATION_RESULT_PATH" +# ado-aw:fragment trusted_preflight # The argument list is assembled into an array so runtime-supplied # fragments splice in as ordinary shell statements (`AWF_ARGS+=(...)`) @@ -6295,17 +6325,18 @@ AWF_ARGS+=( --log-level info --proxy-logs-dir "$AGENT_TEMP/threat-analysis-logs/firewall" ) -# ado-aw:fragment run_invoker +# ado-aw:fragment run_runner # Stream threat analysis output in real-time with VSO command filtering # shellcheck disable=SC2016 # The single-quoted engine command inside AWF_ARGS is intentionally expanded by AWF inside the sandbox +set +e "$PIPELINE_WORKSPACE/awf/awf" "${AWF_ARGS[@]}" 2>&1 \ | sed -u 's/##vso\[/[VSO-FILTERED] vso[/g; s/##\[/[VSO-FILTERED] [/g' \ | tee "$THREAT_OUTPUT_FILE" \ || AGENT_EXIT_CODE=$? MODEL_RESULT_STATUS=0 -REQUESTED_MODEL=$(node "$COPILOT_INVOKER_PATH" read-result "$INVOCATION_RESULT_PATH" detection) \ +REQUESTED_MODEL=$(node "$TRUSTED_CONTROLLER_PATH" read-result "$TRUSTED_RESULT_PATH" detection) \ || MODEL_RESULT_STATUS=$? if [ "$MODEL_RESULT_STATUS" -ne 0 ]; then echo "ERROR: Detection Copilot invocation result is missing or malformed" >&2 @@ -6324,7 +6355,7 @@ exit "$AGENT_EXIT_CODE" fn run_threat_analysis_step( allowed_domains: &str, working_directory: &str, - invocation_document: &str, + invocation_request: &str, byom_exclude_keys: &[String], detection_engine_env: &[(String, String)], github_token_var: &str, @@ -6360,9 +6391,9 @@ fn run_threat_analysis_step( format!("AWF_ARGS+=({})", parts.join(" ")) } }; - let run_invoker = format!( + let run_runner = format!( "AWF_ARGS+=(-- 'exec node {} run /tmp/awf-tools/copilot-invocation.json')", - super::extensions::ado_script::COPILOT_INVOKER_PATH + super::extensions::ado_script::COPILOT_RUNNER_PATH ); let mut step = ShellScript::new(&RUN_THREAT_ANALYSIS) @@ -6373,24 +6404,21 @@ fn run_threat_analysis_step( ) .bind_text("ALLOWED_DOMAINS", allowed_domains) .bind( - "INVOCATION_DOCUMENT", - Binding::document(invocation_document), + "INVOCATION_REQUEST", + Binding::document(invocation_request), ) .bind_text( - "INVOCATION_DOCUMENT_PATH", + "SANDBOX_INVOCATION_PATH", "/tmp/awf-tools/copilot-invocation.json", ) .bind_text( - "INVOCATION_RESULT_PATH", - "/tmp/awf-tools/copilot-invocation-result.json", - ) - .bind_text( - "COPILOT_INVOKER_PATH", - super::extensions::ado_script::COPILOT_INVOKER_PATH, + "COPILOT_CONTROLLER_SOURCE_PATH", + super::extensions::ado_script::COPILOT_CONTROLLER_PATH, ) + .fragment("trusted_preflight", COPILOT_TRUSTED_PREFLIGHT) .fragment("image_flags", image_flags_line) .fragment("exclude_env", exclude_env_line) - .fragment("run_invoker", run_invoker) + .fragment("run_runner", run_runner) .into_step("Run threat analysis (AWF network isolated)"); step.working_directory = Some(working_directory.to_string()); // env block: GITHUB_TOKEN + GITHUB_READ_ONLY — emit the latter as @@ -7801,7 +7829,7 @@ safe-outputs: "example.com", "\\", "/work", - r#"{"schema_version":1,"role":"agent"}"#, + r#"{"schema_version":2,"document_kind":"request","role":"agent"}"#, "FOO: bar", &[], None, @@ -7816,7 +7844,7 @@ safe-outputs: "example.com", "\\", "/work", - r#"{"schema_version":1,"role":"agent"}"#, + r#"{"schema_version":2,"document_kind":"request","role":"agent"}"#, "ADO_AW_MODEL_AGENT_COPILOT: $(ADO_AW_MODEL_AGENT_COPILOT)\nADO_AW_DEFAULT_MODEL_COPILOT: $(ADO_AW_DEFAULT_MODEL_COPILOT)", &[], None, @@ -7861,7 +7889,7 @@ safe-outputs: } #[test] - fn agent_runtime_model_is_delegated_to_invoker_and_recorded_after_awf() { + fn agent_runtime_model_is_prepared_by_controller_and_recorded_after_awf() { let step = runtime_agent_step_for_test(); assert!(matches!( step.env @@ -7875,10 +7903,16 @@ safe-outputs: Some(EnvValue::PipelineVar(name)) if name == crate::engine::ADO_AW_DEFAULT_MODEL_COPILOT )); - assert!(step.script.contains("copilot-invoker.js run")); + assert!(step.script.contains("copilot-runner.js run")); + assert!(step + .script + .contains("node \"$TRUSTED_CONTROLLER_PATH\" prepare")); assert!(step .script - .contains("node \"$COPILOT_INVOKER_PATH\" read-result")); + .contains("node \"$TRUSTED_CONTROLLER_PATH\" read-result")); + assert!(!step + .script + .contains("node \"$COPILOT_CONTROLLER_SOURCE_PATH\" read-result")); assert!(step.script.contains("\"model\"")); let metadata_index = step .script @@ -7917,12 +7951,12 @@ safe-outputs: } #[test] - fn prefixed_user_env_key_does_not_change_fixed_invoker_command() { + fn prefixed_user_env_key_does_not_change_fixed_runner_command() { let step = run_agent_step( "example.com", "\\", "/work", - r#"{"schema_version":1,"role":"agent","explicit_model":"static-model"}"#, + r#"{"schema_version":2,"document_kind":"request","role":"agent","explicit_model":"static-model"}"#, "ADO_AW_MODEL_AGENT_COPILOT_X: harmless", &[], None, @@ -7932,7 +7966,7 @@ safe-outputs: assert!(!step.script.contains("ADO_AW_EFFECTIVE_MODEL")); assert!(!step.script.contains("unset COPILOT_MODEL")); - assert!(step.script.contains("copilot-invoker.js run")); + assert!(step.script.contains("copilot-runner.js run")); } #[test] @@ -8761,10 +8795,27 @@ safe-outputs: .script .contains("$AGENT_TEMP/detection-runtime-model") ); - assert!(run_step.script.contains("copilot-invoker.js run")); + assert!(run_step.script.contains("copilot-runner.js run")); + assert!(run_step + .script + .contains("node \"$TRUSTED_CONTROLLER_PATH\" prepare")); assert!(run_step .script - .contains("node \"$COPILOT_INVOKER_PATH\" read-result")); + .contains("node \"$TRUSTED_CONTROLLER_PATH\" read-result")); + let prepare = run_step + .script + .find("node \"$TRUSTED_CONTROLLER_PATH\" prepare") + .unwrap(); + let disable_errexit = run_step.script.find("set +e").unwrap(); + let awf = run_step + .script + .find("\"$PIPELINE_WORKSPACE/awf/awf\"") + .unwrap(); + assert!(run_step.script.contains("set -eo pipefail")); + assert!( + prepare < disable_errexit && disable_errexit < awf, + "Detection preflight must fail closed before AWF exit capture begins" + ); assert!(!run_step.script.contains("ADO_AW_EFFECTIVE_MODEL")); assert!(!run_step.script.contains("--model")); assert!( @@ -8776,7 +8827,7 @@ safe-outputs: .script .find("$AGENT_TEMP/detection-runtime-model") .unwrap(), - "the trusted host must consume the invoker result after Detection runs" + "the trusted host must consume the controller result after Detection runs" ); assert!( !run_step diff --git a/src/compile/extensions/ado_script.rs b/src/compile/extensions/ado_script.rs index 9cc39371a..cf3f2ebf6 100644 --- a/src/compile/extensions/ado_script.rs +++ b/src/compile/extensions/ado_script.rs @@ -220,8 +220,13 @@ node "$BUNDLE" pub(crate) const GATE_EVAL_PATH: &str = "/tmp/ado-aw-scripts/ado-script/gate.js"; pub(crate) const IMPORT_EVAL_PATH: &str = "/tmp/ado-aw-scripts/ado-script/import.js"; -pub(crate) const COPILOT_INVOKER_PATH: &str = - "/tmp/ado-aw-scripts/ado-script/copilot-invoker.js"; +/// Sandbox-visible controller source. The Agent/Detection step copies this to +/// `$(Agent.TempDirectory)` and removes this copy before AWF starts. +pub(crate) const COPILOT_CONTROLLER_PATH: &str = + "/tmp/ado-aw-scripts/ado-script/copilot-controller.js"; +/// Untrusted run-only bundle executed as AWF's initial Copilot command. +pub(crate) const COPILOT_RUNNER_PATH: &str = + "/tmp/ado-aw-scripts/ado-script/copilot-runner.js"; /// Path to the ado-proxy bundle inside the unpacked `ado-script.zip`. /// /// Unlike every other bundle this one is not executed by a pipeline step. It @@ -1968,8 +1973,9 @@ mod tests { let fm: FrontMatter = serde_yaml::from_str("name: t\ndescription: t").unwrap(); let ctx = CompileContext::for_test(&fm); let steps = ext.declarations(&ctx).unwrap().agent_prepare_steps; - // The invoker is required by every Agent job, so install + download - // fire even when no other ado-script consumer is active. + // The controller and runner are required by every Agent job, so + // install + download fire even when no other ado-script consumer is + // active. assert_eq!(steps.len(), 2, "install + download only"); assert!(matches!(&steps[0], Step::Task(t) if t.task == "UseNode@1")); assert!( @@ -2601,7 +2607,7 @@ mod tests { // ── Typed-IR declarations (port-ado-script) ───────────────────── /// Setup remains empty when no gate / synth path is active, while Agent - /// preparation always stages the Copilot invoker bundle. + /// preparation always stages the Copilot controller/runner bundles. #[test] fn declarations_stages_agent_bundle_when_nothing_else_active() { let ext = ext_with(None, None, true); diff --git a/src/engine.rs b/src/engine.rs index 92a9155d2..b12fa9c31 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -106,15 +106,15 @@ impl RuntimeModelRole { } #[derive(Debug, Clone, Serialize)] -pub(crate) struct CopilotInvocationDocument { +pub(crate) struct CopilotInvocationRequest { pub(crate) schema_version: u32, + pub(crate) document_kind: &'static str, pub(crate) role: RuntimeModelRole, pub(crate) command: String, pub(crate) prompt_path: String, pub(crate) mcp_config_path: Option, pub(crate) args: Vec, pub(crate) explicit_model: Option, - pub(crate) result_path: String, } #[derive(Debug, Clone, Copy)] @@ -122,7 +122,6 @@ pub(crate) struct CopilotInvocationContext<'a> { role: RuntimeModelRole, prompt_path: &'a str, mcp_config_path: Option<&'a str>, - result_path: &'a str, } impl<'a> CopilotInvocationContext<'a> { @@ -130,13 +129,11 @@ impl<'a> CopilotInvocationContext<'a> { role: RuntimeModelRole, prompt_path: &'a str, mcp_config_path: Option<&'a str>, - result_path: &'a str, ) -> Self { Self { role, prompt_path, mcp_config_path, - result_path, } } } @@ -473,13 +470,13 @@ impl Engine { } } - pub(crate) fn invocation_document( + pub(crate) fn invocation_request( &self, front_matter: &FrontMatter, extension_declarations: &[Declarations], invocation: CopilotInvocationContext<'_>, - ) -> Result { - self.invocation_document_with_config( + ) -> Result { + self.invocation_request_with_config( &front_matter.engine, front_matter, extension_declarations, @@ -487,13 +484,13 @@ impl Engine { ) } - pub(crate) fn invocation_document_with_config( + pub(crate) fn invocation_request_with_config( &self, engine_config: &EngineConfig, front_matter: &FrontMatter, extension_declarations: &[Declarations], invocation: CopilotInvocationContext<'_>, - ) -> Result { + ) -> Result { let args = self.args_with_config(engine_config, front_matter, extension_declarations)?; match self { Engine::Copilot => { @@ -514,15 +511,15 @@ impl Engine { if let Some(model) = engine_config.model() { validate_model_name(model)?; } - Ok(CopilotInvocationDocument { - schema_version: 1, + Ok(CopilotInvocationRequest { + schema_version: 2, + document_kind: "request", role: invocation.role, command: command_path, prompt_path: invocation.prompt_path.to_string(), mcp_config_path: invocation.mcp_config_path.map(str::to_string), args, explicit_model: engine_config.model().map(str::to_string), - result_path: invocation.result_path.to_string(), }) } } @@ -782,7 +779,7 @@ fn copilot_args( fn validate_model_name(model: &str) -> Result<()> { // Validate model names before they become invocation-document values or // runtime-selected COPILOT_MODEL values. Keep this character set in sync - // with the Copilot invoker. + // with the Copilot controller/runner protocol. if model.is_empty() || !model .chars() @@ -1464,14 +1461,13 @@ mod tests { let env = Engine::Copilot.env(&front_matter.engine).unwrap(); assert!(!env.contains(COPILOT_MODEL), "{env}"); let invocation = Engine::Copilot - .invocation_document( + .invocation_request( &front_matter, &declarations_for(&front_matter), CopilotInvocationContext::new( RuntimeModelRole::Agent, "/tmp/prompt.md", None, - "/tmp/result.json", ), ) .unwrap(); @@ -1479,23 +1475,24 @@ mod tests { } #[test] - fn copilot_invocation_document_defers_runtime_model_resolution() { + fn copilot_invocation_request_defers_runtime_model_resolution() { let (front_matter, _) = parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); let invocation = Engine::Copilot - .invocation_document( + .invocation_request( &front_matter, &declarations_for(&front_matter), CopilotInvocationContext::new( RuntimeModelRole::Agent, "/tmp/prompt.md", Some("/tmp/mcp.json"), - "/tmp/result.json", ), ) .unwrap(); assert_eq!(invocation.role, RuntimeModelRole::Agent); + assert_eq!(invocation.schema_version, 2); + assert_eq!(invocation.document_kind, "request"); assert_eq!(invocation.explicit_model, None); assert_eq!( invocation.mcp_config_path.as_deref(), @@ -1505,20 +1502,19 @@ mod tests { } #[test] - fn copilot_invocation_document_keeps_explicit_model_static() { + fn copilot_invocation_request_keeps_explicit_model_static() { let (front_matter, _) = parse_markdown( "---\nname: test\ndescription: test\nengine:\n id: copilot\n model: gpt-5\n---\n", ) .unwrap(); let invocation = Engine::Copilot - .invocation_document( + .invocation_request( &front_matter, &declarations_for(&front_matter), CopilotInvocationContext::new( RuntimeModelRole::Agent, "/tmp/prompt.md", None, - "/tmp/result.json", ), ) .unwrap(); @@ -1528,11 +1524,11 @@ mod tests { } #[test] - fn copilot_detection_invocation_document_uses_independent_runtime_model() { + fn copilot_detection_invocation_request_uses_independent_runtime_model() { let (front_matter, _) = parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); let invocation = Engine::Copilot - .invocation_document_with_config( + .invocation_request_with_config( &front_matter.engine, &front_matter, &declarations_for(&front_matter), @@ -1540,7 +1536,6 @@ mod tests { RuntimeModelRole::Detection, "/tmp/threat.md", None, - "/tmp/result.json", ), ) .unwrap(); @@ -1786,14 +1781,13 @@ mod tests { "---\nname: test\ndescription: test\nengine:\n id: copilot\n command: /usr/local/bin/my-copilot\n---\n", ).unwrap(); let result = Engine::Copilot - .invocation_document( + .invocation_request( &fm, &declarations_for(&fm), CopilotInvocationContext::new( RuntimeModelRole::Agent, "/tmp/prompt.md", Some("/tmp/mcp.json"), - "/tmp/result.json", ), ) .unwrap(); @@ -1804,14 +1798,13 @@ mod tests { fn engine_command_default_uses_awf_path() { let (fm, _) = parse_markdown("---\nname: test\ndescription: test\n---\n").unwrap(); let result = Engine::Copilot - .invocation_document( + .invocation_request( &fm, &declarations_for(&fm), CopilotInvocationContext::new( RuntimeModelRole::Agent, "/tmp/prompt.md", Some("/tmp/mcp.json"), - "/tmp/result.json", ), ) .unwrap(); @@ -1823,14 +1816,13 @@ mod tests { let (fm, _) = parse_markdown( "---\nname: test\ndescription: test\nengine:\n id: copilot\n command: \"/tmp/copilot; rm -rf /\"\n---\n", ).unwrap(); - let result = Engine::Copilot.invocation_document( + let result = Engine::Copilot.invocation_request( &fm, &declarations_for(&fm), CopilotInvocationContext::new( RuntimeModelRole::Agent, "/tmp/prompt.md", None, - "/tmp/result.json", ), ); assert!(result.is_err()); @@ -1847,14 +1839,13 @@ mod tests { let (fm, _) = parse_markdown( "---\nname: test\ndescription: test\nengine:\n id: copilot\n command: \"/tmp/co'pilot\"\n---\n", ).unwrap(); - let result = Engine::Copilot.invocation_document( + let result = Engine::Copilot.invocation_request( &fm, &declarations_for(&fm), CopilotInvocationContext::new( RuntimeModelRole::Agent, "/tmp/prompt.md", None, - "/tmp/result.json", ), ); assert!(result.is_err()); diff --git a/tests/awf-copilot-safeoutputs/run.sh b/tests/awf-copilot-safeoutputs/run.sh index 5710a6937..a0fb7b459 100644 --- a/tests/awf-copilot-safeoutputs/run.sh +++ b/tests/awf-copilot-safeoutputs/run.sh @@ -6,10 +6,12 @@ umask 077 : "${ADO_AW_BIN:?ADO_AW_BIN is required}" : "${AWF_BIN:?AWF_BIN is required}" : "${COPILOT_BIN:?COPILOT_BIN is required}" -: "${COPILOT_INVOKER_BUNDLE:?COPILOT_INVOKER_BUNDLE is required}" +: "${COPILOT_CONTROLLER_BUNDLE:?COPILOT_CONTROLLER_BUNDLE is required}" +: "${COPILOT_RUNNER_BUNDLE:?COPILOT_RUNNER_BUNDLE is required}" : "${AWF_VERSION:?AWF_VERSION is required}" : "${MCPG_VERSION:?MCPG_VERSION is required}" : "${ADO_AW_COPILOT_CLI_ARTIFACT_DIR:?ADO_AW_COPILOT_CLI_ARTIFACT_DIR is required}" +: "${ADO_AW_COPILOT_CLI_CONTROL_DIR:?ADO_AW_COPILOT_CLI_CONTROL_DIR is required}" : "${COPILOT_GITHUB_TOKEN:?COPILOT_GITHUB_TOKEN is required}" readonly CONTRACT_CONTEXT="awf-copilot-safeoutputs-contract" @@ -18,6 +20,7 @@ readonly MCP_GATEWAY_CONTAINER="awmg-mcpg" readonly MCPG_IMAGE="ghcr.io/github/gh-aw-mcpg:v${MCPG_VERSION}" readonly SAFEOUTPUTS_IMAGE="ghcr.io/github/gh-aw-firewall/agent:${AWF_VERSION}" readonly ARTIFACT_DIR="${ADO_AW_COPILOT_CLI_ARTIFACT_DIR}" +readonly CONTROL_DIR="${ADO_AW_COPILOT_CLI_CONTROL_DIR}" RUNTIME_DIR="$(mktemp -d /tmp/ado-aw-awf-contract.XXXXXX)" SAFE_OUTPUTS_DIR="$(mktemp -d /tmp/ado-aw-safeoutputs.XXXXXX)" @@ -25,6 +28,7 @@ TOOLS_DIR="/tmp/awf-tools" MCPG_PID="" mkdir -p "${ARTIFACT_DIR}" "${SAFE_OUTPUTS_DIR}" "${TOOLS_DIR}" +install -d -m 0700 "${CONTROL_DIR}" cleanup() { local status=$? @@ -38,7 +42,7 @@ cleanup() { if [[ -f "${SAFE_OUTPUTS_DIR}/safe_outputs.ndjson" ]]; then cp "${SAFE_OUTPUTS_DIR}/safe_outputs.ndjson" "${ARTIFACT_DIR}/safe_outputs.ndjson" fi - rm -rf "${RUNTIME_DIR}" "${SAFE_OUTPUTS_DIR}" + rm -rf "${RUNTIME_DIR}" "${SAFE_OUTPUTS_DIR}" "${CONTROL_DIR}" return "${status}" } trap cleanup EXIT @@ -72,8 +76,12 @@ for binary in "${ADO_AW_BIN}" "${AWF_BIN}" "${COPILOT_BIN}"; do exit 1 } done -[[ -f "${COPILOT_INVOKER_BUNDLE}" ]] || { - echo "Copilot invoker bundle is missing: ${COPILOT_INVOKER_BUNDLE}" >&2 +[[ -f "${COPILOT_CONTROLLER_BUNDLE}" ]] || { + echo "Copilot controller bundle is missing: ${COPILOT_CONTROLLER_BUNDLE}" >&2 + exit 1 +} +[[ -f "${COPILOT_RUNNER_BUNDLE}" ]] || { + echo "Copilot runner bundle is missing: ${COPILOT_RUNNER_BUNDLE}" >&2 exit 1 } @@ -229,8 +237,10 @@ chmod 600 "${TOOLS_DIR}/mcp-config.json" install -m 0755 "${COPILOT_BIN}" "${TOOLS_DIR}/copilot" mkdir -p /tmp/ado-aw-scripts/ado-script -install -m 0644 "${COPILOT_INVOKER_BUNDLE}" \ - /tmp/ado-aw-scripts/ado-script/copilot-invoker.js +install -m 0644 "${COPILOT_RUNNER_BUNDLE}" \ + /tmp/ado-aw-scripts/ado-script/copilot-runner.js +install -m 0500 "${COPILOT_CONTROLLER_BUNDLE}" \ + "${CONTROL_DIR}/copilot-controller.js" cat >"${TOOLS_DIR}/agent-prompt.md" <"${TOOLS_DIR}/copilot-invocation.json" -chmod 600 "${TOOLS_DIR}/copilot-invocation.json" + explicit_model: "gpt-5-mini" + }' >"${CONTROL_DIR}/invocation-request.json" +chmod 600 "${CONTROL_DIR}/invocation-request.json" +node "${CONTROL_DIR}/copilot-controller.js" prepare \ + "${CONTROL_DIR}/invocation-request.json" \ + "${TOOLS_DIR}/copilot-invocation.json" \ + "${CONTROL_DIR}/invocation-result.json" readonly ALLOWED_DOMAINS="api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,config.edge.skype.com,copilot-proxy.githubusercontent.com,github.com,telemetry.enterprise.githubcopilot.com,*.copilot.github.com,*.githubcopilot.com" # shellcheck disable=SC2016 # AWF expands the engine command inside the sandbox. -readonly ENGINE_RUN='export NO_PROXY="${NO_PROXY:+$NO_PROXY,}awmg-mcpg"; export no_proxy="$NO_PROXY"; exec node /tmp/ado-aw-scripts/ado-script/copilot-invoker.js run /tmp/awf-tools/copilot-invocation.json' +readonly ENGINE_RUN='export NO_PROXY="${NO_PROXY:+$NO_PROXY,}awmg-mcpg"; export no_proxy="$NO_PROXY"; exec node /tmp/ado-aw-scripts/ado-script/copilot-runner.js run /tmp/awf-tools/copilot-invocation.json' set +e "${AWF_BIN}" \ @@ -286,11 +299,19 @@ if [[ "${AWF_STATUS}" -ne 0 ]]; then fi REQUESTED_MODEL="$( - node /tmp/ado-aw-scripts/ado-script/copilot-invoker.js \ - read-result "${TOOLS_DIR}/copilot-invocation-result.json" agent + node "${CONTROL_DIR}/copilot-controller.js" \ + read-result "${CONTROL_DIR}/invocation-result.json" agent )" [[ "${REQUESTED_MODEL}" == "gpt-5-mini" ]] || { - echo "Unexpected requested model from invoker: ${REQUESTED_MODEL}" >&2 + echo "Unexpected requested model from controller: ${REQUESTED_MODEL}" >&2 + exit 1 +} +[[ ! -e /tmp/ado-aw-scripts/ado-script/copilot-runner.js ]] || { + echo "Copilot runner did not remove itself before child execution" >&2 + exit 1 +} +[[ ! -e "${TOOLS_DIR}/copilot-invocation.json" ]] || { + echo "Prepared invocation document was not removed before child execution" >&2 exit 1 } diff --git a/tests/compiler_tests.rs b/tests/compiler_tests.rs index b5080cf73..7df6fa7e7 100644 --- a/tests/compiler_tests.rs +++ b/tests/compiler_tests.rs @@ -2054,18 +2054,18 @@ Call the noop tool exactly once. "agent invocation document should reference the prompt and MCP config: {agent}" ); assert!( - agent.contains("copilot-invoker.js run") + agent.contains("copilot-runner.js run") && !agent.contains("/tmp/awf-tools/copilot --prompt"), - "agent job should execute only the fixed invoker command: {agent}" + "agent job should execute only the fixed sandbox runner command: {agent}" ); assert!( detection.contains(r#""prompt_path":"/tmp/awf-tools/threat-analysis-prompt.md""#), "detection invocation document should reference the threat prompt: {detection}" ); assert!( - detection.contains("copilot-invoker.js run") + detection.contains("copilot-runner.js run") && !detection.contains("/tmp/awf-tools/copilot --prompt"), - "detection job should execute only the fixed invoker command: {detection}" + "detection job should execute only the fixed sandbox runner command: {detection}" ); assert!( agent.contains("--allow-all-tools"), @@ -2125,9 +2125,9 @@ fn test_runtime_import_frontmatter_prompt_uses_attached_copilot_prompt_flag() { "detection invocation document should reference the threat prompt: {detection}" ); assert!( - compiled.contains("copilot-invoker.js run") + compiled.contains("copilot-runner.js run") && !compiled.contains("/tmp/awf-tools/copilot --prompt"), - "compiled pipeline should use the fixed invoker command: {compiled}" + "compiled pipeline should use the fixed sandbox runner command: {compiled}" ); exercise_attached_prompt_with_pinned_copilot_cli(&fixture); @@ -5306,8 +5306,8 @@ fn test_1es_compiled_output_is_valid_yaml() { "1ES output should contain SafeOutputs references" ); assert!( - compiled.contains("copilot-invoker.js run"), - "1ES output should contain the fixed Copilot invoker command" + compiled.contains("copilot-runner.js run"), + "1ES output should contain the fixed Copilot runner command" ); assert!( compiled.contains("threat-analysis"), @@ -5846,7 +5846,7 @@ fn extract_job_block<'a>(yaml: &'a str, name: &str) -> Option<&'a str> { } /// Gate-only pipelines stage the bundle in Setup for the gate and in both -/// Copilot jobs for the invoker. +/// Copilot jobs for the controller and runner. #[test] fn test_gate_only_pipeline_downloads_bundle_in_all_consuming_jobs() { let yaml = compile_fixture("dedupe_gate_only.md"); @@ -5858,7 +5858,7 @@ fn test_gate_only_pipeline_downloads_bundle_in_all_consuming_jobs() { ); assert!( agent.contains("Download ado-aw scripts"), - "Agent job must stage the Copilot invoker bundle. Agent block contents: {}", + "Agent job must stage the Copilot controller/runner bundles. Agent block contents: {}", agent ); } @@ -5906,14 +5906,15 @@ fn test_both_features_active_downloads_bundle_in_both_jobs() { ); } -/// Even with no gate or runtime imports, Agent and Detection stage the invoker. +/// Even with no gate or runtime imports, Agent and Detection stage the +/// controller and runner. #[test] -fn test_neither_feature_active_stages_invoker_in_copilot_jobs() { +fn test_neither_feature_active_stages_copilot_bundles_in_copilot_jobs() { let yaml = compile_fixture("dedupe_neither.md"); assert_eq!( yaml.matches("Download ado-aw scripts").count(), 2, - "Agent and Detection must each stage the invoker bundle" + "Agent and Detection must each stage the controller/runner bundles" ); } @@ -8231,13 +8232,34 @@ fn runtime_model_controls_compile_across_all_targets() { "{target}: missing Detection runtime model env mapping" ); assert!( - compiled.contains("copilot-invoker.js run"), - "{target}: fixed Copilot invoker command must be emitted" + compiled.contains("copilot-runner.js run"), + "{target}: fixed sandbox runner command must be emitted" ); assert!( - compiled.contains(r#""role":"agent""#) - && compiled.contains(r#""role":"detection""#), - "{target}: Agent and Detection invocation documents must be emitted" + compiled.contains(r#""schema_version":2,"document_kind":"request","role":"agent""#) + && compiled.contains( + r#""schema_version":2,"document_kind":"request","role":"detection""# + ), + "{target}: Agent and Detection schema-v2 requests must be emitted" + ); + assert!( + compiled.contains( + r#"TRUSTED_CONTROLLER_DIR="$AGENT_TEMP/ado-aw-copilot-controller""# + ) && compiled.contains(r#"node "$TRUSTED_CONTROLLER_PATH" prepare"#) + && compiled.contains(r#"node "$TRUSTED_CONTROLLER_PATH" read-result"#), + "{target}: trusted controller preparation and result validation must use the host-private directory" + ); + assert!( + compiled.contains(r#"rm -f "$COPILOT_CONTROLLER_SOURCE_PATH""#), + "{target}: sandbox-visible controller source must be removed before AWF" + ); + assert!( + !compiled.contains(r#""result_path""#) + && !compiled.contains( + "/tmp/awf-tools/copilot-invocation-result.json" + ) + && !compiled.contains("copilot-controller.js run"), + "{target}: sandbox-visible requests and commands must not carry authoritative result/controller capabilities" ); assert!( !compiled.contains("ADO_AW_EFFECTIVE_MODEL"), @@ -8272,14 +8294,14 @@ safe-outputs: .expect("model variable producer should be emitted"); let consumer = agent .find("Run copilot (AWF network isolated)") - .expect("Copilot invoker step should be emitted"); + .expect("Copilot controller/runner step should be emitted"); assert!( producer < consumer, - "trusted variable producer must run before the invoker task: {agent}" + "trusted variable producer must run before the controller/runner task: {agent}" ); assert!( agent.contains("ADO_AW_MODEL_AGENT_COPILOT: $(ADO_AW_MODEL_AGENT_COPILOT)"), - "invoker task must resolve the model through its typed env mapping: {agent}" + "controller task must resolve the model through its typed env mapping: {agent}" ); } @@ -10204,7 +10226,7 @@ fn test_github_app_token_hyphenated_private_key_variable() { /// When another ado-script bundle feature is active in the Agent job (here a /// safe-output activates the approval-summary bundle download), the mint step /// must NOT trigger another bundle download in either Copilot job because both -/// already stage the invoker bundle. +/// already stage the controller/runner bundles. #[test] fn test_github_app_token_reuses_staged_bundle_in_agent() { fn count_downloads(compiled: &str) -> usize { From 67acc66580f6472e60dabf6ede8d377239e2ece5 Mon Sep 17 00:00:00 2001 From: James Devine Date: Mon, 5 Oct 2026 02:39:00 +0100 Subject: [PATCH 16/24] fix(compile): harden Copilot invocation review gaps Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- .../src/copilot-controller/index.test.ts | 19 +++++++ .../src/copilot-controller/index.ts | 2 +- .../src/copilot-runner/index.test.ts | 2 + .../ado-script/src/copilot-runner/index.ts | 3 +- .../src/copilot-shared/protocol.test.ts | 52 ++++++++++++++++++- src/compile/agentic_pipeline.rs | 13 ++++- 6 files changed, 87 insertions(+), 4 deletions(-) diff --git a/scripts/ado-script/src/copilot-controller/index.test.ts b/scripts/ado-script/src/copilot-controller/index.test.ts index bcc8fe108..c124bbb46 100644 --- a/scripts/ado-script/src/copilot-controller/index.test.ts +++ b/scripts/ado-script/src/copilot-controller/index.test.ts @@ -54,6 +54,25 @@ describe("copilot controller", () => { }); }); + it("commits the trusted result before exposing a prepared sandbox document", async () => { + const directory = mkdtempSync(join(tmpdir(), "copilot-controller-")); + const requestPath = writeRequest(directory); + const preparedPath = join(directory, "missing", "prepared.json"); + const resultPath = join(directory, "result.json"); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect( + main(["prepare", requestPath, preparedPath, resultPath]), + ).resolves.toBe(1); + expect(JSON.parse(readFileSync(resultPath, "utf8"))).toMatchObject({ + document_kind: "result", + role: "agent", + }); + expect(error).toHaveBeenCalledWith( + expect.stringContaining("copilot-controller:"), + ); + }); + it("reads only a matching strict result", async () => { const directory = mkdtempSync(join(tmpdir(), "copilot-controller-")); const resultPath = join(directory, "result.json"); diff --git a/scripts/ado-script/src/copilot-controller/index.ts b/scripts/ado-script/src/copilot-controller/index.ts index 7f4ede8e8..12f09d1a6 100644 --- a/scripts/ado-script/src/copilot-controller/index.ts +++ b/scripts/ado-script/src/copilot-controller/index.ts @@ -23,8 +23,8 @@ export async function main(argv: string[]): Promise { try { const request = parseInvocationRequest(readFileSync(argv[1]!, "utf8")); const { prepared, result } = prepareInvocation(request, process.env); - writeJsonAtomic(argv[2]!, prepared); writeJsonAtomic(argv[3]!, result); + writeJsonAtomic(argv[2]!, prepared); return 0; } catch (error) { const message = error instanceof Error ? error.message : "unknown error"; diff --git a/scripts/ado-script/src/copilot-runner/index.test.ts b/scripts/ado-script/src/copilot-runner/index.test.ts index 836cb8329..5ca38c8fb 100644 --- a/scripts/ado-script/src/copilot-runner/index.test.ts +++ b/scripts/ado-script/src/copilot-runner/index.test.ts @@ -118,6 +118,7 @@ describe("copilot runner", () => { error.mockRestore(); const signaledChild = childProcess(); + const listenersBefore = process.listenerCount("SIGTERM"); signaledChild.kill = vi.fn((signal: NodeJS.Signals) => { queueMicrotask(() => signaledChild.emit("close", null, signal)); return true; @@ -136,6 +137,7 @@ describe("copilot runner", () => { process.emit("SIGTERM", "SIGTERM"); await expect(signaled).resolves.toBe(143); expect(signaledChild.kill).toHaveBeenCalledWith("SIGTERM"); + expect(process.listenerCount("SIGTERM")).toBe(listenersBefore); }); it("does not expose controller modes", async () => { diff --git a/scripts/ado-script/src/copilot-runner/index.ts b/scripts/ado-script/src/copilot-runner/index.ts index 7dfbce04a..29cdc3cc8 100644 --- a/scripts/ado-script/src/copilot-runner/index.ts +++ b/scripts/ado-script/src/copilot-runner/index.ts @@ -77,7 +77,8 @@ export async function runInvocation( process.on(signal, handlers[signal]); } child.once("error", (error) => { - console.error(`copilot-runner: failed to start Copilot: ${error.message}`); + const message = error instanceof Error ? error.message : "unknown error"; + console.error(`copilot-runner: failed to start Copilot: ${message}`); settle(1); }); child.once("close", (code, signal) => { diff --git a/scripts/ado-script/src/copilot-shared/protocol.test.ts b/scripts/ado-script/src/copilot-shared/protocol.test.ts index e637c2a3f..dffc14963 100644 --- a/scripts/ado-script/src/copilot-shared/protocol.test.ts +++ b/scripts/ado-script/src/copilot-shared/protocol.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, readdirSync, readFileSync } from "node:fs"; +import { mkdtempSync, readdirSync, readFileSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -80,6 +80,10 @@ describe("Copilot invocation protocol", () => { [{ ...request(), command: "/tmp//copilot" }, "field 'command' is invalid"], [{ ...request(), command: "/tmp/copilot/" }, "field 'command' is invalid"], [{ ...request(), prompt_path: "relative.md" }, "field 'prompt_path' is invalid"], + [ + { ...request(), mcp_config_path: "/tmp/../mcp.json" }, + "field 'mcp_config_path' is invalid", + ], [{ ...request(), args: [1] }, "field 'args' must be an array of strings"], [{ ...request(), explicit_model: "bad model" }, "field 'explicit_model' is invalid"], ])("rejects malformed requests %#", (value, message) => { @@ -151,6 +155,9 @@ describe("Copilot invocation protocol", () => { role: "agent", }); expect(readdirSync(directory)).toEqual(["result.json"]); + if (process.platform !== "win32") { + expect(statSync(path).mode & 0o777).toBe(0o600); + } }); }); @@ -223,6 +230,28 @@ describe("argv and child environment", () => { ]); }); + it("keeps option-looking prompt text inside the attached prompt argument", () => { + expect(buildCopilotArgs(prepared(), "--model=attacker")).toEqual([ + "--prompt=--model=attacker", + "--additional-mcp-config", + "@/tmp/awf-tools/mcp-config.json", + "--disable-builtin-mcps", + "--allow-tool", + "shell(cat *)", + ]); + }); + + it("omits MCP arguments when no MCP config is prepared", () => { + expect( + buildCopilotArgs(prepared({ mcp_config_path: null }), "prompt"), + ).toEqual([ + "--prompt=prompt", + "--disable-builtin-mcps", + "--allow-tool", + "shell(cat *)", + ]); + }); + it("rejects NUL bytes in prompt content", () => { expect(() => buildCopilotArgs(prepared(), "before\0after")).toThrow( "prompt contains an invalid NUL byte", @@ -242,4 +271,25 @@ describe("argv and child environment", () => { }); expect(original.COPILOT_MODEL).toBe("old"); }); + + it("removes a stale Copilot model when no model was prepared", () => { + expect( + buildChildEnvironment( + { + KEEP: "yes", + COPILOT_MODEL: "stale", + ADO_AW_MODEL_DETECTION_COPILOT: "conflicting", + }, + null, + ), + ).toEqual({ KEEP: "yes" }); + }); + + it("accepts a validated bare executable command", () => { + expect( + parsePreparedInvocation( + JSON.stringify(prepared({ command: "copilot" })), + ).command, + ).toBe("copilot"); + }); }); diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index 5b71934a8..632b0a205 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -3618,7 +3618,7 @@ shell_script! { externals: [], fragments: [tail], body: r###" -set -o pipefail +set -eo pipefail mkdir -p "$DEST" locate_one() { @@ -7137,6 +7137,17 @@ mod tests { serde_yaml::from_str(yaml).expect("front matter should parse") } + #[test] + fn candidate_artifact_staging_keeps_fail_fast_verification() { + assert!( + STAGE_CANDIDATE_ARTIFACT_PAYLOAD + .body + .trim_start() + .starts_with("set -eo pipefail"), + "candidate checksum and provenance verification must abort on the first failure" + ); + } + fn test_ctx() -> StandaloneCtx { let test_pool = Pool::VmImage("ubuntu-latest".to_string()); StandaloneCtx { From ce0ade781259709106bdcd6ab1d16f3fddc5e9d3 Mon Sep 17 00:00:00 2001 From: James Devine Date: Mon, 5 Oct 2026 03:53:39 +0100 Subject: [PATCH 17/24] fix(compile): preserve Copilot execution failures Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- src/compile/agentic_pipeline.rs | 82 ++++++++++++++++++++----- src/compile/extensions/ado_aw_marker.rs | 33 +++++++++- 2 files changed, 96 insertions(+), 19 deletions(-) diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index 632b0a205..568aa2974 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -4551,9 +4551,6 @@ REQUESTED_MODEL=$(node "$TRUSTED_CONTROLLER_PATH" read-result "$TRUSTED_RESULT_P || MODEL_RESULT_STATUS=$? if [ "$MODEL_RESULT_STATUS" -ne 0 ]; then echo "ERROR: Agent Copilot invocation result is missing or malformed" >&2 - if [ "$AGENT_EXIT_CODE" -eq 0 ]; then - AGENT_EXIT_CODE="$MODEL_RESULT_STATUS" - fi else ADO_AW_INFO_JSON="$AGENT_TEMP/staging/aw_info.json" if [ -f "$ADO_AW_INFO_JSON" ]; then @@ -4561,7 +4558,8 @@ else ado_aw_append_info_field \ "model" \ "$REQUESTED_MODEL" \ - "$ADO_AW_INFO_JSON" + "$ADO_AW_INFO_JSON" \ + || MODEL_RESULT_STATUS=$? else echo "Warning: Agent metadata file not found at $ADO_AW_INFO_JSON; model metadata was not recorded" >&2 fi @@ -4573,7 +4571,10 @@ if [ -x "$PIPELINE_WORKSPACE/awf/awf" ]; then "$PIPELINE_WORKSPACE/awf/awf" logs summary --source "$AGENT_TEMP/staging/logs/firewall" 2>/dev/null || true fi -exit "$AGENT_EXIT_CODE" +if [ "$AGENT_EXIT_CODE" -ne 0 ]; then + exit "$AGENT_EXIT_CODE" +fi +exit "$MODEL_RESULT_STATUS" "###, } } @@ -6340,14 +6341,14 @@ REQUESTED_MODEL=$(node "$TRUSTED_CONTROLLER_PATH" read-result "$TRUSTED_RESULT_P || MODEL_RESULT_STATUS=$? if [ "$MODEL_RESULT_STATUS" -ne 0 ]; then echo "ERROR: Detection Copilot invocation result is missing or malformed" >&2 - if [ "$AGENT_EXIT_CODE" -eq 0 ]; then - AGENT_EXIT_CODE="$MODEL_RESULT_STATUS" - fi else printf '%s' "$REQUESTED_MODEL" > "$AGENT_TEMP/detection-runtime-model" fi -exit "$AGENT_EXIT_CODE" +if [ "$AGENT_EXIT_CODE" -ne 0 ]; then + exit "$AGENT_EXIT_CODE" +fi +exit "$MODEL_RESULT_STATUS" "###, } } @@ -6513,8 +6514,8 @@ set -eo pipefail ADO_AW_INFO_JSON="$AGENT_TEMP/analyzed_outputs/aw_info.json" ADO_AW_MODEL_FILE="$AGENT_TEMP/detection-runtime-model" if [ ! -f "$ADO_AW_INFO_JSON" ]; then - echo "ERROR: Detection could not find copied aw_info.json at $ADO_AW_INFO_JSON" >&2 - exit 1 + echo "Warning: Detection metadata file not found at $ADO_AW_INFO_JSON; model metadata was not recorded" >&2 + exit 0 fi if [ ! -f "$ADO_AW_MODEL_FILE" ]; then exit 0 @@ -7921,6 +7922,12 @@ safe-outputs: assert!(step .script .contains("node \"$TRUSTED_CONTROLLER_PATH\" read-result")); + assert!(!step + .script + .contains("AGENT_EXIT_CODE=\"$MODEL_RESULT_STATUS\"")); + assert!(step.script.contains( + "if [ \"$AGENT_EXIT_CODE\" -ne 0 ]; then\n exit \"$AGENT_EXIT_CODE\"\nfi\nexit \"$MODEL_RESULT_STATUS\"" + )); assert!(!step .script .contains("node \"$COPILOT_CONTROLLER_SOURCE_PATH\" read-result")); @@ -8607,15 +8614,17 @@ safe-outputs: } #[cfg(unix)] - fn run_detection_runtime_model_script(model: Option<&str>) -> (Output, tempfile::TempDir) { + fn run_detection_runtime_model_script_with_aw_info( + model: Option<&str>, + aw_info: Option<&str>, + ) -> (Output, tempfile::TempDir) { let temp = tempfile::tempdir().expect("temp dir"); let analyzed_outputs = temp.path().join("analyzed_outputs"); std::fs::create_dir_all(&analyzed_outputs).expect("create analyzed outputs"); - std::fs::write( - analyzed_outputs.join("aw_info.json"), - r#"{"schema":"ado-aw/aw_info/1"}"#, - ) - .expect("write aw_info.json"); + if let Some(aw_info) = aw_info { + std::fs::write(analyzed_outputs.join("aw_info.json"), aw_info) + .expect("write aw_info.json"); + } if let Some(model) = model { std::fs::write(temp.path().join("detection-runtime-model"), model) .expect("write runtime model"); @@ -8632,6 +8641,14 @@ safe-outputs: (command.output().expect("bash should run"), temp) } + #[cfg(unix)] + fn run_detection_runtime_model_script(model: Option<&str>) -> (Output, tempfile::TempDir) { + run_detection_runtime_model_script_with_aw_info( + model, + Some(r#"{"schema":"ado-aw/aw_info/1"}"#), + ) + } + #[cfg(unix)] fn read_detection_aw_info(temp: &tempfile::TempDir) -> serde_json::Value { let contents = @@ -8813,6 +8830,12 @@ safe-outputs: assert!(run_step .script .contains("node \"$TRUSTED_CONTROLLER_PATH\" read-result")); + assert!(!run_step + .script + .contains("AGENT_EXIT_CODE=\"$MODEL_RESULT_STATUS\"")); + assert!(run_step.script.contains( + "if [ \"$AGENT_EXIT_CODE\" -ne 0 ]; then\n exit \"$AGENT_EXIT_CODE\"\nfi\nexit \"$MODEL_RESULT_STATUS\"" + )); let prepare = run_step .script .find("node \"$TRUSTED_CONTROLLER_PATH\" prepare") @@ -8885,6 +8908,31 @@ safe-outputs: ); } + #[test] + #[cfg(unix)] + fn missing_detection_metadata_does_not_mask_the_detection_result() { + let (output, _) = + run_detection_runtime_model_script_with_aw_info(Some("detector-model"), None); + + assert!(output.status.success(), "{output:?}"); + assert!(String::from_utf8_lossy(&output.stderr).contains( + "Warning: Detection metadata file not found" + )); + } + + #[test] + #[cfg(unix)] + fn malformed_detection_metadata_fails_enrichment() { + let (output, _) = run_detection_runtime_model_script_with_aw_info( + Some("detector-model"), + Some("not-json"), + ); + + assert!(!output.status.success(), "{output:?}"); + assert!(String::from_utf8_lossy(&output.stderr) + .contains("aw_info.json is not a single-line JSON object")); + } + #[test] fn pool_overrides_detection_only_flows_to_compiled_job() { let source = concat!( diff --git a/src/compile/extensions/ado_aw_marker.rs b/src/compile/extensions/ado_aw_marker.rs index af5da1e0f..977ceefd6 100644 --- a/src/compile/extensions/ado_aw_marker.rs +++ b/src/compile/extensions/ado_aw_marker.rs @@ -62,10 +62,11 @@ ado_aw_append_info_field() { local field="$1" local value="$2" local file="$3" - if [ -z "$value" ] || grep -Eq "\"${field}\"[[:space:]]*:" "$file"; then + if [ -z "$value" ]; then return 0 fi local json + local compact_json local tmp local separator="," json="$(cat "$file")" @@ -76,10 +77,14 @@ ado_aw_append_info_field() { \{*\}) ;; *) echo "ERROR: aw_info.json is not a single-line JSON object" >&2 - exit 1 + return 1 ;; esac fi + compact_json="$(printf '%s' "$json" | tr -d '[:space:]')" + if printf '%s' "$compact_json" | grep -Fq "\"${field}\":"; then + return 0 + fi tmp="$(mktemp)" printf '%s%s"%s":"%s"}' "${json%?}" "$separator" "$field" "$value" > "$tmp" mv "$tmp" "$file" @@ -769,6 +774,30 @@ mod tests { assert_eq!(value["model"], "original"); } + #[test] + #[cfg(unix)] + fn append_aw_info_field_returns_failure_without_exiting_its_caller() { + let temp = tempfile::tempdir().expect("temp dir"); + let path = temp.path().join("aw_info.json"); + std::fs::write(&path, "not-json").expect("write aw_info.json"); + let script = format!( + "{}\nstatus=0\nado_aw_append_info_field model gpt-5 \"$FILE\" || status=$?\nprintf 'after:%s' \"$status\"", + APPEND_AW_INFO_FIELD.body.trim() + ); + let output = Command::new("bash") + .arg("-c") + .arg(script) + .env_clear() + .env("FILE", path) + .output() + .expect("bash should run"); + + assert!(output.status.success(), "{output:?}"); + assert_eq!(String::from_utf8_lossy(&output.stdout), "after:1"); + assert!(String::from_utf8_lossy(&output.stderr) + .contains("aw_info.json is not a single-line JSON object")); + } + #[test] fn explicit_model_emits_aw_info_model_metadata() { let fm = parse_fm("name: t\ndescription: x\nengine:\n id: copilot\n model: some-model\n"); From 36ba851c137396de24ac1388b4de741cb15f8d4c Mon Sep 17 00:00:00 2001 From: James Devine Date: Mon, 5 Oct 2026 20:46:30 +0100 Subject: [PATCH 18/24] test(compile): scope MCPG command assertions Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- tests/compiler_tests.rs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/compiler_tests.rs b/tests/compiler_tests.rs index 7df6fa7e7..af8f1d225 100644 --- a/tests/compiler_tests.rs +++ b/tests/compiler_tests.rs @@ -8,6 +8,21 @@ fn compiled_has_enabled_tool(compiled: &str, tool: &str) -> bool { }) } +fn extract_mcpg_config(compiled: &str) -> &str { + let marker = "cat > \"$AGENT_TEMP/staging/mcpg-config.json\" << '"; + let tail = compiled + .split_once(marker) + .map(|(_, tail)| tail) + .expect("compiled pipeline must stage an MCPG config"); + let (sentinel, payload) = tail + .split_once("'\n") + .expect("MCPG config heredoc must have a quoted sentinel"); + payload + .split_once(sentinel) + .map(|(config, _)| config) + .expect("MCPG config heredoc must terminate") +} + // `assert_required_markers`, `assert_pool_config`, `assert_compiler_download`, // `assert_awf_download`, `assert_mcpg_integration`, and `test_compiled_yaml_structure` // validated the legacy `src/data/base.yml` template. The standalone target @@ -2419,6 +2434,7 @@ fn test_fixture_azure_devops_mcp_compiled_output() { ); let compiled = fs::read_to_string(&output_path).expect("Should read compiled output"); + let mcpg_config = extract_mcpg_config(&compiled); // The policy document is now carried by the `POLICY` binding, which // `Binding::document` renders as a quoted heredoc in the generated @@ -2486,7 +2502,7 @@ fn test_fixture_azure_devops_mcp_compiled_output() { "MCPG config should have entrypointArgs field" ); assert!( - !compiled.contains("\"command\": "), + !mcpg_config.contains("\"command\""), "MCPG config should NOT use command field" ); @@ -2592,6 +2608,7 @@ fn test_mcpg_config_container_based_mcp() { ); let compiled = fs::read_to_string(&output_path).unwrap(); + let mcpg_config = extract_mcpg_config(&compiled); assert!(compiled.contains("\"container\": \"ghcr.io/example/my-tool:latest\"")); assert!(compiled.contains("\"entrypoint\": \"my-tool\"")); @@ -2600,7 +2617,7 @@ fn test_mcpg_config_container_based_mcp() { assert!(compiled.contains("/host/data:/app/data:ro")); assert!(compiled.contains("\"API_KEY\": \"test-key\"")); assert!(compiled.contains("\"tool_a\"")); - assert!(!compiled.contains("\"command\": ")); + assert!(!mcpg_config.contains("\"command\"")); let _ = fs::remove_dir_all(&temp_dir); } @@ -2720,11 +2737,12 @@ fn test_mcpg_config_http_based_mcp() { ); let compiled = fs::read_to_string(&output_path).unwrap(); + let mcpg_config = extract_mcpg_config(&compiled); assert!(compiled.contains("\"url\": \"https://mcp.dev.azure.com/myorg\"")); assert!(compiled.contains("\"X-MCP-Toolsets\": \"repos,wit\"")); assert!(compiled.contains("\"wit_get_work_item\"")); - assert!(!compiled.contains("\"command\": ")); + assert!(!mcpg_config.contains("\"command\"")); let _ = fs::remove_dir_all(&temp_dir); } From e1ee471ba6d3eadbf7e8f953e3df2f41d7557549 Mon Sep 17 00:00:00 2001 From: James Devine Date: Tue, 6 Oct 2026 10:02:49 +0100 Subject: [PATCH 19/24] test(ado-script): cover Copilot protocol edge cases Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- .../src/copilot-controller/index.ts | 2 ++ .../src/copilot-runner/index.test.ts | 21 +++++++++++++++++++ .../src/copilot-shared/protocol.test.ts | 13 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/scripts/ado-script/src/copilot-controller/index.ts b/scripts/ado-script/src/copilot-controller/index.ts index 12f09d1a6..f731867eb 100644 --- a/scripts/ado-script/src/copilot-controller/index.ts +++ b/scripts/ado-script/src/copilot-controller/index.ts @@ -23,6 +23,8 @@ export async function main(argv: string[]): Promise { try { const request = parseInvocationRequest(readFileSync(argv[1]!, "utf8")); const { prepared, result } = prepareInvocation(request, process.env); + // The sandbox document is the commit point: trusted preflight removes + // any prior result and cannot start AWF unless both writes succeed. writeJsonAtomic(argv[3]!, result); writeJsonAtomic(argv[2]!, prepared); return 0; diff --git a/scripts/ado-script/src/copilot-runner/index.test.ts b/scripts/ado-script/src/copilot-runner/index.test.ts index 5ca38c8fb..7a34e32fc 100644 --- a/scripts/ado-script/src/copilot-runner/index.test.ts +++ b/scripts/ado-script/src/copilot-runner/index.test.ts @@ -1,4 +1,7 @@ import { EventEmitter } from "node:events"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { @@ -149,4 +152,22 @@ describe("copilot runner", () => { ); error.mockRestore(); }); + + it("reports unreadable and malformed prepared invocations", async () => { + const directory = mkdtempSync(join(tmpdir(), "copilot-runner-")); + const missingPath = join(directory, "missing.json"); + const malformedPath = join(directory, "malformed.json"); + writeFileSync(malformedPath, "not-json"); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(main(["run", missingPath], "/tmp/runner.js")).resolves.toBe(1); + await expect(main(["run", malformedPath], "/tmp/runner.js")).resolves.toBe(1); + expect(error).toHaveBeenCalledWith( + expect.stringContaining("copilot-runner:"), + ); + expect(error).toHaveBeenCalledWith( + "copilot-runner: prepared invocation is not valid JSON", + ); + error.mockRestore(); + }); }); diff --git a/scripts/ado-script/src/copilot-shared/protocol.test.ts b/scripts/ado-script/src/copilot-shared/protocol.test.ts index dffc14963..429b5b24c 100644 --- a/scripts/ado-script/src/copilot-shared/protocol.test.ts +++ b/scripts/ado-script/src/copilot-shared/protocol.test.ts @@ -76,6 +76,8 @@ describe("Copilot invocation protocol", () => { [{ ...request(), role: "other" }, "must be 'agent' or 'detection'"], [{ ...request(), command: "copilot;sh" }, "field 'command' is invalid"], [{ ...request(), command: "bin/copilot" }, "field 'command' is invalid"], + [{ ...request(), command: "." }, "field 'command' is invalid"], + [{ ...request(), command: ".." }, "field 'command' is invalid"], [{ ...request(), command: "/tmp/../copilot" }, "field 'command' is invalid"], [{ ...request(), command: "/tmp//copilot" }, "field 'command' is invalid"], [{ ...request(), command: "/tmp/copilot/" }, "field 'command' is invalid"], @@ -184,12 +186,23 @@ describe("model preparation", () => { ADO_AW_DEFAULT_MODEL_COPILOT: "default-model", }), ).toBe("default-model"); + expect( + resolveRequestedModel(request(), { + ADO_AW_MODEL_AGENT_COPILOT: "$(ADO_AW_MODEL_AGENT_COPILOT)", + ADO_AW_DEFAULT_MODEL_COPILOT: "default-model", + }), + ).toBe("default-model"); expect( resolveRequestedModel(request(), { ADO_AW_MODEL_AGENT_COPILOT: "$(ADO_AW_MODEL_AGENT_COPILOT)", ADO_AW_DEFAULT_MODEL_COPILOT: "$(ADO_AW_DEFAULT_MODEL_COPILOT)", }), ).toBeNull(); + expect( + resolveRequestedModel(request(), { + ADO_AW_DEFAULT_MODEL_COPILOT: "", + }), + ).toBeNull(); }); it("rejects invalid runtime values without printing them", () => { From fb4e64b486cca2a7468ed733a29b4511ff9c962d Mon Sep 17 00:00:00 2001 From: James Devine Date: Tue, 6 Oct 2026 12:14:50 +0100 Subject: [PATCH 20/24] test(smoke): verify live runtime model variables Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- .../__tests__/ado-rest.test.ts | 19 ++ .../__tests__/cases.test.ts | 63 ++++++- .../__tests__/index.test.ts | 33 +++- .../__tests__/runner.test.ts | 43 +++++ .../__tests__/signals.test.ts | 59 +++++- .../src/compiler-smoke-e2e/ado-rest.ts | 13 +- .../src/compiler-smoke-e2e/cases.ts | 69 ++++++- .../src/compiler-smoke-e2e/index.ts | 3 +- .../src/compiler-smoke-e2e/runner.ts | 12 +- .../src/compiler-smoke-e2e/signals.ts | 172 +++++++++++------- tests/smoke/README.md | 18 +- tests/smoke/REGISTERED.md | 6 + tests/smoke/cases.json | 27 +++ tests/smoke/runtime-model-queue.md | 19 ++ tests/smoke/runtime-model-set-variable.md | 24 +++ 15 files changed, 495 insertions(+), 85 deletions(-) create mode 100644 tests/smoke/runtime-model-queue.md create mode 100644 tests/smoke/runtime-model-set-variable.md diff --git a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/ado-rest.test.ts b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/ado-rest.test.ts index bce3a951c..f2960ec9d 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/ado-rest.test.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/ado-rest.test.ts @@ -99,6 +99,25 @@ describe("AdoRest.queueBuild", () => { }); }); + it("serializes queue variables using the ADO Build API variable shape", async () => { + let sentBody: unknown; + const fetchImpl = vi.fn( + async (_input: RequestInfo | URL, init?: RequestInit) => { + sentBody = JSON.parse(String(init?.body)); + return jsonResponse(200, { id: 556 }); + }, + ); + const rest = makeRest(fetchImpl as unknown as typeof fetch); + await rest.queueBuild(2560, { + sourceBranch: "refs/heads/x", + sourceVersion: "deadbeef", + variables: { ADO_AW_MODEL_AGENT_COPILOT: "auto" }, + }); + expect(sentBody).toMatchObject({ + variables: { ADO_AW_MODEL_AGENT_COPILOT: { value: "auto" } }, + }); + }); + it("throws with a descriptive error on a non-2xx response", async () => { const fetchImpl = vi.fn(async () => new Response("nope", { status: 500 })); const rest = makeRest(fetchImpl as unknown as typeof fetch); diff --git a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/cases.test.ts b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/cases.test.ts index b0c6e64d5..04098633d 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/cases.test.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/cases.test.ts @@ -313,7 +313,7 @@ describe("parseManifest", () => { }, ]), ), - ).toThrow(/must declare agentCommand, pipelineText and\/or requiredBuildTags/); + ).toThrow(/must declare agentCommand, pipelineText, requiredBuildTags and\/or requestedModels/); }); it("rejects an agentCommand with no snippets", () => { @@ -356,6 +356,67 @@ describe("parseManifest", () => { forbidden: ["$SC_READ_TOKEN"], }); }); + + it("parses and validates requested model assertions", () => { + const parsed = parseManifest( + cases([ + { + id: "x", + lane: "agentic", + kind: "compiled", + modes: ["candidate", "released"], + source: "a.md", + assertions: { requestedModels: { agent: "auto", detection: "gpt-5.4" } }, + }, + ]), + ); + expect(parsed.cases[0]?.assertions?.requestedModels).toEqual({ + agent: "auto", + detection: "gpt-5.4", + }); + + expect(() => + parseManifest( + cases([ + { + id: "x", + lane: "agentic", + kind: "compiled", + modes: ["candidate", "released"], + source: "a.md", + assertions: { requestedModels: {} }, + }, + ]), + ), + ).toThrow(/must declare agent and\/or detection/); + }); + }); + + describe("queue variable validation", () => { + it("parses valid variables and rejects empty or malformed values", () => { + const entry = { + id: "x", + lane: "agentic", + kind: "compiled", + modes: ["candidate", "released"], + source: "a.md", + }; + const parsed = parseManifest( + cases([{ ...entry, queueVariables: { ADO_AW_MODEL_AGENT_COPILOT: "auto" } }]), + ); + expect(parsed.cases[0]?.queueVariables).toEqual({ + ADO_AW_MODEL_AGENT_COPILOT: "auto", + }); + expect(() => parseManifest(cases([{ ...entry, queueVariables: {} }]))).toThrow( + /must not be empty/, + ); + expect(() => + parseManifest(cases([{ ...entry, queueVariables: { "not-valid": "auto" } }])), + ).toThrow(/must match/); + expect(() => + parseManifest(cases([{ ...entry, queueVariables: { ADO_AW_MODEL_AGENT_COPILOT: "" } }])), + ).toThrow(/must be a non-empty string/); + }); }); }); diff --git a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts index 2606cf65a..c63b467ce 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts @@ -8,7 +8,13 @@ const mockCalls: string[] = []; const compiledCasePaths: string[] = []; const stagedWrites: { to: string; contents: string }[] = []; let queuedCaseIds: string[] = []; -let queuedRequests: { caseId: string; lane: string; definitionId: number; sourceBranch: string }[] = []; +let queuedRequests: { + caseId: string; + lane: string; + definitionId: number; + sourceBranch: string; + variables?: Readonly>; +}[] = []; let deletedRefs: string[] = []; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -171,7 +177,7 @@ vi.mock("../signals.js", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, - verifyCandidateAudit: vi.fn(async (results: readonly FixtureBuildResult[]) => ({ + verifyCandidateAudit: vi.fn(async (_cases: unknown, results: readonly FixtureBuildResult[]) => ({ ok: true, results: results.map((result) => ({ ...result })), })), @@ -185,7 +191,13 @@ vi.mock("../runner.js", async (importOriginal) => { runFixtures: vi.fn( async ( _client: unknown, - requests: { caseId: string; lane: string; definitionId: number; sourceBranch: string }[], + requests: { + caseId: string; + lane: string; + definitionId: number; + sourceBranch: string; + variables?: Readonly>; + }[], ) => { mockCalls.push("runFixtures"); queuedCaseIds = requests.map((request) => request.caseId); @@ -271,6 +283,8 @@ describe("smoke-e2e index.main (happy path, candidate mode)", () => { "noop-target", "custom-safe-output", "multi-repo", + "runtime-model-queue", + "runtime-model-set-variable", ]); expect(queuedCaseIds).not.toContain("janitor"); expect(compiledCasePaths).toEqual([ @@ -279,7 +293,12 @@ describe("smoke-e2e index.main (happy path, candidate mode)", () => { "tests/safe-outputs/noop-target.md", "tests/smoke/custom-safe-output.md", "tests/smoke/multi-repo.md", + "tests/smoke/runtime-model-queue.md", + "tests/smoke/runtime-model-set-variable.md", ]); + expect( + queuedRequests.find((request) => request.caseId === "runtime-model-queue")?.variables, + ).toEqual({ ADO_AW_MODEL_AGENT_COPILOT: "auto" }); // Cleanup ordering: remote refs deleted BEFORE the local worktree is removed. expect(mockCalls.indexOf("deleteRemoteRefs")).toBeGreaterThanOrEqual(0); @@ -297,9 +316,11 @@ describe("smoke-e2e index.main (happy path, candidate mode)", () => { "refs/heads/ado-aw-smoke-candidate/630001/noop-target", "refs/heads/ado-aw-smoke-candidate/630001/custom-safe-output", "refs/heads/ado-aw-smoke-candidate/630001/multi-repo", + "refs/heads/ado-aw-smoke-candidate/630001/runtime-model-queue", + "refs/heads/ado-aw-smoke-candidate/630001/runtime-model-set-variable", ]); // Every case is staged to the SAME path — the ref is what distinguishes them. - expect(stagedWrites.length).toBe(5); + expect(stagedWrites.length).toBe(7); for (const write of stagedWrites) { expect(write.to).toBe(join(WORKTREE, "candidate", ".smoke", "pipeline.yml")); // The compiler emits no trigger keys once `on:` is stripped, and a @@ -329,7 +350,7 @@ describe("smoke-e2e index.main (happy path, candidate mode)", () => { const gitModule = await import("../git.js"); const resets = vi.mocked(gitModule.resetWorktree).mock.calls; - expect(resets.length).toBe(5); + expect(resets.length).toBe(7); for (const call of resets) { expect(call[0]).toMatchObject({ commitish: "basecommit" }); } @@ -456,6 +477,8 @@ describe("smoke-e2e index.main (per-case ref retention)", () => { "refs/heads/ado-aw-smoke-candidate/630001/noop-target", "refs/heads/ado-aw-smoke-candidate/630001/custom-safe-output", "refs/heads/ado-aw-smoke-candidate/630001/multi-repo", + "refs/heads/ado-aw-smoke-candidate/630001/runtime-model-queue", + "refs/heads/ado-aw-smoke-candidate/630001/runtime-model-set-variable", ]); expect(deletedRefs).not.toContain("refs/heads/ado-aw-smoke-candidate/630001/ado-proxy"); }); diff --git a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/runner.test.ts b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/runner.test.ts index 575119e05..886ce9b04 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/runner.test.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/runner.test.ts @@ -101,6 +101,49 @@ describe("runFixtures", () => { expect(outcome.results.every((r) => r.terminalProven)).toBe(true); }); + it("forwards queue variables unchanged", async () => { + let queuedOptions: Parameters[1] | undefined; + const client: FixtureBuildClient = { + async queueBuild(_definitionId, options) { + queuedOptions = options; + return { id: 103 }; + }, + async getBuild() { + return { + status: "completed", + result: "succeeded", + definition: { id: 1 }, + sourceBranch: "refs/heads/x", + sourceVersion: "sha", + }; + }, + async cancelBuild() {}, + async addBuildTags() {}, + buildUrl(buildId) { + return `https://example/${buildId}`; + }, + }; + const request = { + ...req("runtime-model-queue", 1), + variables: { ADO_AW_MODEL_AGENT_COPILOT: "auto" }, + }; + + const outcome = await runFixtures(client, [request], { + concurrency: 1, + timeoutMs: 10_000, + pollMs: 1, + log: () => {}, + sleepImpl: noopSleep, + }); + + expect(outcome.ok).toBe(true); + expect(queuedOptions).toEqual({ + sourceBranch: "refs/heads/x", + sourceVersion: "sha", + variables: { ADO_AW_MODEL_AGENT_COPILOT: "auto" }, + }); + }); + it("preserves declaration order in results regardless of completion order", async () => { const { client } = makeFakeClient({ queueResults: { 1: { ok: true, id: 201 }, 2: { ok: true, id: 202 } }, diff --git a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts index 22e3134be..c800a5de7 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts @@ -36,6 +36,15 @@ const CASES: ResolvedCase[] = [ source: "tests/safe-outputs/canary.md", definitionId: 3006, }, + { + id: "runtime-model-queue", + lane: "agentic", + kind: "compiled", + modes: ["candidate"], + source: "tests/smoke/runtime-model-queue.md", + assertions: { requestedModels: { agent: "auto" } }, + definitionId: 3006, + }, ]; function result(overrides: Partial = {}): FixtureBuildResult { @@ -177,7 +186,7 @@ describe("verifyCandidateAudit", () => { }); const canary = result({ caseId: "canary" }); - const outcome = await verifyCandidateAudit([canary], options); + const outcome = await verifyCandidateAudit(CASES, [canary], options); expect(outcome).toEqual({ ok: true, results: [canary] }); expect(safeSpawnMock).toHaveBeenCalledWith( @@ -200,7 +209,11 @@ describe("verifyCandidateAudit", () => { stderrTruncated: false, }); - const outcome = await verifyCandidateAudit([result({ caseId: "canary" })], options); + const outcome = await verifyCandidateAudit( + CASES, + [result({ caseId: "canary" })], + options, + ); expect(outcome.ok).toBe(false); expect(outcome.results[0]?.message).toContain("***"); @@ -209,6 +222,7 @@ describe("verifyCandidateAudit", () => { it("fails closed without spawning when no successful canary build exists", async () => { const outcome = await verifyCandidateAudit( + CASES, [result({ caseId: "canary", status: "failed", result: "failed" })], options, ); @@ -216,4 +230,45 @@ describe("verifyCandidateAudit", () => { expect(outcome.ok).toBe(false); expect(safeSpawnMock).not.toHaveBeenCalled(); }); + + it("verifies the requested Agent model from audit metadata", async () => { + safeSpawnMock + .mockResolvedValueOnce({ + status: 0, + stdout: JSON.stringify({ + overview: { build_id: 42 }, + downloaded_files: [ + { path: "agent_outputs_42/agent-output.json" }, + { path: "analyzed_outputs_42/verdict.json" }, + { path: "safe_outputs/executed-safe-outputs.ndjson" }, + ], + }), + stderr: "", + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }) + .mockResolvedValueOnce({ + status: 0, + stdout: JSON.stringify({ + overview: { build_id: 43, aw_info: { model: "gpt-5.4" } }, + }), + stderr: "", + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }); + + const outcome = await verifyCandidateAudit( + CASES, + [ + result({ caseId: "canary" }), + result({ caseId: "runtime-model-queue", buildId: 43 }), + ], + options, + ); + + expect(outcome.ok).toBe(false); + expect(outcome.results[1]?.message).toMatch(/expected "auto"/); + }); }); diff --git a/scripts/ado-script/src/compiler-smoke-e2e/ado-rest.ts b/scripts/ado-script/src/compiler-smoke-e2e/ado-rest.ts index 3e953b320..ef78bce27 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/ado-rest.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/ado-rest.ts @@ -249,14 +249,23 @@ export class AdoRest { */ async queueBuild( definitionId: number, - opts: { sourceBranch: string; sourceVersion: string }, + opts: { + sourceBranch: string; + sourceVersion: string; + variables?: Readonly>; + }, ): Promise<{ id: number }> { const path = this.projPath(`_apis/build/builds?api-version=7.1`); - const body = { + const body: Record = { definition: { id: definitionId }, sourceBranch: opts.sourceBranch, sourceVersion: opts.sourceVersion, }; + if (opts.variables && Object.keys(opts.variables).length > 0) { + body.variables = Object.fromEntries( + Object.entries(opts.variables).map(([name, value]) => [name, { value }]), + ); + } const res = await this.request<{ id: number }>(path, { method: "POST", body }); if (!res) throw new Error(`queueBuild(${definitionId}) returned no body`); return res; diff --git a/scripts/ado-script/src/compiler-smoke-e2e/cases.ts b/scripts/ado-script/src/compiler-smoke-e2e/cases.ts index 0dc4cfa3e..7c1761247 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/cases.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/cases.ts @@ -62,6 +62,11 @@ export interface CaseAssertions { readonly pipelineText?: AgentCommandAssertion; /** Build tags the child run must carry, with `{buildId}` expanded to the child build id. */ readonly requiredBuildTags?: readonly string[]; + /** Requested runtime models recorded in the child's audit metadata. */ + readonly requestedModels?: { + readonly agent?: string; + readonly detection?: string; + }; } export interface SmokeLane { @@ -77,6 +82,8 @@ export interface SmokeCase { readonly modes: readonly CompilerSource[]; /** Repo-relative source path (`.md` for compiled, `.yml`/`.yaml` for raw). */ readonly source: string; + /** Non-secret variables supplied through the ADO queue-build API. */ + readonly queueVariables?: Readonly>; readonly assertions?: CaseAssertions; } @@ -162,6 +169,27 @@ function validateKindMatchesExtension(kind: CaseKind, source: string, caseId: st } } +function parseQueueVariables( + raw: unknown, + caseId: string, +): Readonly> | undefined { + if (raw === undefined) return undefined; + const obj = asRecord(raw, `case '${caseId}' queueVariables`); + const entries = Object.entries(obj); + if (entries.length === 0) { + fail(`case '${caseId}' queueVariables must not be empty`); + } + + const variables: Record = {}; + for (const [name, rawValue] of entries) { + if (!ENV_NAME_RE.test(name)) { + fail(`case '${caseId}' queue variable '${name}' must match ${ENV_NAME_RE}`); + } + variables[name] = asString(rawValue, `case '${caseId}' queueVariables.${name}`); + } + return variables; +} + function parseAssertions(raw: unknown, caseId: string): CaseAssertions | undefined { if (raw === undefined) return undefined; const obj = asRecord(raw, `case '${caseId}' assertions`); @@ -217,16 +245,41 @@ function parseAssertions(raw: unknown, caseId: string): CaseAssertions | undefin } } + let requestedModels: CaseAssertions["requestedModels"]; + if (obj.requestedModels !== undefined) { + const models = asRecord( + obj.requestedModels, + `case '${caseId}' assertions.requestedModels`, + ); + requestedModels = { + agent: + models.agent === undefined + ? undefined + : asString(models.agent, `case '${caseId}' assertions.requestedModels.agent`), + detection: + models.detection === undefined + ? undefined + : asString( + models.detection, + `case '${caseId}' assertions.requestedModels.detection`, + ), + }; + if (requestedModels.agent === undefined && requestedModels.detection === undefined) { + fail(`case '${caseId}' assertions.requestedModels must declare agent and/or detection`); + } + } + if ( agentCommand === undefined && pipelineText === undefined && - requiredBuildTags === undefined + requiredBuildTags === undefined && + requestedModels === undefined ) { fail( - `case '${caseId}' assertions must declare agentCommand, pipelineText and/or requiredBuildTags`, + `case '${caseId}' assertions must declare agentCommand, pipelineText, requiredBuildTags and/or requestedModels`, ); } - return { agentCommand, pipelineText, requiredBuildTags }; + return { agentCommand, pipelineText, requiredBuildTags, requestedModels }; } /** Expand `{buildId}` in a declared build tag. */ @@ -316,7 +369,15 @@ export function parseManifest(text: string): SmokeManifest { const source = validateSourcePath(entry.source, id); validateKindMatchesExtension(kind, source, id); - cases.push({ id, lane, kind, modes, source, assertions: parseAssertions(entry.assertions, id) }); + cases.push({ + id, + lane, + kind, + modes, + source, + queueVariables: parseQueueVariables(entry.queueVariables, id), + assertions: parseAssertions(entry.assertions, id), + }); } if (cases.length === 0) fail("cases must declare at least one case"); diff --git a/scripts/ado-script/src/compiler-smoke-e2e/index.ts b/scripts/ado-script/src/compiler-smoke-e2e/index.ts index 110499ba9..5afa09ba6 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/index.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/index.ts @@ -368,6 +368,7 @@ export async function main(): Promise { definitionId: entry.definitionId, sourceBranch: staged.get(entry.id)!.ref, sourceVersion: staged.get(entry.id)!.sha, + variables: entry.queueVariables, tags: [`smoke-case:${entry.id}`, `smoke-candidate:${config.buildId}`], })); @@ -386,7 +387,7 @@ export async function main(): Promise { const signalOutcome = await verifyCaseSignals(rest, resolved.cases, outcome.results); const auditOutcome = config.compilerSource === "candidate" - ? await verifyCandidateAudit(signalOutcome.results, { + ? await verifyCandidateAudit(resolved.cases, signalOutcome.results, { adoAwBin: config.adoAwBin, cwd: config.sourcesDirectory, orgUrl: config.orgUrl, diff --git a/scripts/ado-script/src/compiler-smoke-e2e/runner.ts b/scripts/ado-script/src/compiler-smoke-e2e/runner.ts index 4e0370921..512e0f3b0 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/runner.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/runner.ts @@ -50,7 +50,14 @@ export interface PolledBuild { /** The minimal ADO Build surface this state machine needs. */ export interface FixtureBuildClient { - queueBuild(definitionId: number, opts: { sourceBranch: string; sourceVersion: string }): Promise<{ id: number }>; + queueBuild( + definitionId: number, + opts: { + sourceBranch: string; + sourceVersion: string; + variables?: Readonly>; + }, + ): Promise<{ id: number }>; getBuild(buildId: number): Promise; cancelBuild(buildId: number): Promise; buildUrl(buildId: number): string; @@ -70,6 +77,8 @@ export interface FixtureBuildRequest { definitionId: number; sourceBranch: string; sourceVersion: string; + /** Non-secret ADO variables supplied when the child build is queued. */ + variables?: Readonly>; /** Tags applied to the queued run so it is identifiable in a shared lane's history. */ tags?: readonly string[]; } @@ -312,6 +321,7 @@ export async function runFixtures( const build = await client.queueBuild(req.definitionId, { sourceBranch: req.sourceBranch, sourceVersion: req.sourceVersion, + variables: req.variables, }); queued.push({ index: i, buildId: build.id, start }); results[i] = { diff --git a/scripts/ado-script/src/compiler-smoke-e2e/signals.ts b/scripts/ado-script/src/compiler-smoke-e2e/signals.ts index 80b270094..17d25e8d5 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/signals.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/signals.ts @@ -77,8 +77,20 @@ export async function verifyCaseSignals( }; } -/** Audit one completed candidate child through the released CLI contract. */ +interface CandidateAuditReport { + overview?: { + build_id?: number; + aw_info?: { + model?: string | null; + detection_model?: string | null; + }; + }; + downloaded_files?: { path?: string }[]; +} + +/** Audit candidate children through the released CLI contract. */ export async function verifyCandidateAudit( + cases: readonly ResolvedCase[], results: readonly FixtureBuildResult[], options: { adoAwBin: string; @@ -89,78 +101,106 @@ export async function verifyCandidateAudit( timeoutMs: number; }, ): Promise { - const target = results.find( + const canary = results.find( (result) => result.caseId === "canary" && result.status === "succeeded" && result.buildId !== undefined, ); - if (!target?.buildId) return { ok: false, results: results.map((result) => ({ ...result })) }; + if (!canary?.buildId) return { ok: false, results: results.map((result) => ({ ...result })) }; + + const casesById = new Map(cases.map((entry) => [entry.id, entry])); + const targets = results.filter((result) => { + if (result.status !== "succeeded" || result.buildId === undefined) return false; + return ( + result.caseId === "canary" || + casesById.get(result.caseId)?.assertions?.requestedModels !== undefined + ); + }); + const verified = results.map((result) => ({ ...result })); - const outputDir = await mkdtemp(join(tmpdir(), "ado-aw-smoke-audit-")); - try { - const outcome = await safeSpawn({ - cmd: options.adoAwBin, - args: [ - "audit", - String(target.buildId), - "--json", - "--no-cache", - "--output", - outputDir, - "--org", - options.orgUrl, - "--project", - options.project, - ], - cwd: options.cwd, - env: { AZURE_DEVOPS_EXT_PAT: options.token }, - timeoutMs: options.timeoutMs, - }); + for (const target of targets) { + const outputDir = await mkdtemp(join(tmpdir(), "ado-aw-smoke-audit-")); let error: string | undefined; - if (outcome.timedOut || outcome.status !== 0) { - error = `exit=${outcome.status ?? "signal"} timedOut=${outcome.timedOut}; stderr=${redact(outcome.stderr, [options.token])}`; - } else { - try { - const audit = JSON.parse(outcome.stdout) as { - overview?: { build_id?: number }; - downloaded_files?: { path?: string }[]; - }; - const paths = - audit.downloaded_files - ?.flatMap((file) => file.path ?? []) - .map((path) => path.replaceAll("\\", "/")) ?? []; - const expectedRoots = [ - `agent_outputs_${target.buildId}/`, - `analyzed_outputs_${target.buildId}/`, - "safe_outputs/", - ]; - const missingRoots = expectedRoots.filter( - (root) => !paths.some((path) => path.startsWith(root)), - ); - if (audit.overview?.build_id !== target.buildId || missingRoots.length > 0) { - error = - `JSON report did not contain the child build id and every published artifact family; ` + - `missing roots: ${missingRoots.join(", ") || ""}`; + try { + const outcome = await safeSpawn({ + cmd: options.adoAwBin, + args: [ + "audit", + String(target.buildId), + "--json", + "--no-cache", + "--output", + outputDir, + "--org", + options.orgUrl, + "--project", + options.project, + ], + cwd: options.cwd, + env: { AZURE_DEVOPS_EXT_PAT: options.token }, + timeoutMs: options.timeoutMs, + }); + if (outcome.timedOut || outcome.status !== 0) { + error = `exit=${outcome.status ?? "signal"} timedOut=${outcome.timedOut}; stderr=${redact(outcome.stderr, [options.token])}`; + } else { + try { + const audit = JSON.parse(outcome.stdout) as CandidateAuditReport; + if (audit.overview?.build_id !== target.buildId) { + error = `JSON report build id was ${audit.overview?.build_id ?? ""}; expected ${target.buildId}`; + } + + if (!error && target.caseId === "canary") { + const paths = + audit.downloaded_files + ?.flatMap((file) => file.path ?? []) + .map((path) => path.replaceAll("\\", "/")) ?? []; + const expectedRoots = [ + `agent_outputs_${target.buildId}/`, + `analyzed_outputs_${target.buildId}/`, + "safe_outputs/", + ]; + const missingRoots = expectedRoots.filter( + (root) => !paths.some((path) => path.startsWith(root)), + ); + if (missingRoots.length > 0) { + error = + `JSON report did not contain every published artifact family; ` + + `missing roots: ${missingRoots.join(", ")}`; + } + } + + const requested = casesById.get(target.caseId)?.assertions?.requestedModels; + if (!error && requested?.agent !== undefined) { + const actual = audit.overview?.aw_info?.model; + if (actual !== requested.agent) { + error = `requested Agent model was ${JSON.stringify(actual ?? null)}; expected ${JSON.stringify(requested.agent)}`; + } + } + if (!error && requested?.detection !== undefined) { + const actual = audit.overview?.aw_info?.detection_model; + if (actual !== requested.detection) { + error = `requested Detection model was ${JSON.stringify(actual ?? null)}; expected ${JSON.stringify(requested.detection)}`; + } + } + } catch (parseError) { + error = `invalid JSON report: ${parseError instanceof Error ? parseError.message : String(parseError)}`; } - } catch (parseError) { - error = `invalid JSON report: ${parseError instanceof Error ? parseError.message : String(parseError)}`; } + } finally { + await rm(outputDir, { recursive: true, force: true }); } - if (!error) return { ok: true, results: results.map((result) => ({ ...result })) }; - return { - ok: false, - results: results.map((result) => - result.caseId === target.caseId - ? { - ...result, - status: "failed", - message: - `candidate audit contract failed for build #${target.buildId} (${target.url ?? "URL unavailable"}); ` + - `expected artifacts agent_outputs_${target.buildId}, analyzed_outputs_${target.buildId}, and safe_outputs: ${error}`, - } - : { ...result }, - ), - }; - } finally { - await rm(outputDir, { recursive: true, force: true }); + if (error) { + const index = verified.findIndex((result) => result.caseId === target.caseId); + verified[index] = { + ...verified[index]!, + status: "failed", + message: + `candidate audit verification failed for build #${target.buildId} (${target.url ?? "URL unavailable"}): ${error}`, + }; + } } + + return { + ok: verified.every((result) => result.status === "succeeded"), + results: verified, + }; } diff --git a/tests/smoke/README.md b/tests/smoke/README.md index d6ddf30c7..e53b03aa4 100644 --- a/tests/smoke/README.md +++ b/tests/smoke/README.md @@ -35,7 +35,7 @@ credential class: | Lane | Secrets / service connections | Cases | | --- | --- | --- | -| `agentic` | `GITHUB_TOKEN`, `agent-playground-read`/`-write` | canary, ado-proxy, noop-target, custom-safe-output, multi-repo, janitor | +| `agentic` | `GITHUB_TOKEN`, `agent-playground-read`/`-write` | canary, ado-proxy, noop-target, custom-safe-output, multi-repo, runtime-model-queue, runtime-model-set-variable, janitor | | `infra` | none | *(reserved for AWF and the ado-proxy sidecar)* | No case currently files GitHub issues, so the lane holds no GitHub PAT beyond @@ -139,7 +139,10 @@ GitHub. "lane": "agentic", // must already exist; a NEW lane costs a registration "kind": "compiled", // or "raw" for hand-written YAML "modes": ["candidate", "released"], - "source": "tests/safe-outputs/my-case.md" + "source": "tests/safe-outputs/my-case.md", + "queueVariables": { // optional, non-secret ADO queue-time variables + "ADO_AW_MODEL_AGENT_COPILOT": "auto" + } } ``` @@ -178,10 +181,18 @@ Optional per-case assertions, so novel checks stay out of the harness code: "required": ["displayName: Start ado-proxy policy engine"], "forbidden": ["--network host"] }, - "requiredBuildTags": ["ado-aw-custom-job-{buildId}"] + "requiredBuildTags": ["ado-aw-custom-job-{buildId}"], + "requestedModels": { "agent": "auto" } } ``` +`requestedModels` runs `ado-aw audit` against the completed child and compares +the requested Agent and/or Detection model recorded in `overview.aw_info`. +Queue variables use the Build API's `variables` object and therefore exercise +the same runtime source as variables supplied in the Azure DevOps Run Pipeline +UI. A lane definition may need the variable predeclared with +`allowOverride=true` when the project restricts queue-time variables. + ### `kind: raw` For pipelines that aren't compiled from front matter — for example a future @@ -246,6 +257,7 @@ malformed or mis-laned manifest fails locally rather than in ADO. | 6 | Exactly one ref per case is created, and every ref is deleted | both | | 7 | Each build ran its lane definition on that case's own ref | both | | 8 | Queued build count equals case count (no ref push CI-triggered a lane) | both | +| 9 | Queue-time and preceding same-job variables select the requested Agent model recorded by audit | candidate | ## Fork security boundary diff --git a/tests/smoke/REGISTERED.md b/tests/smoke/REGISTERED.md index 9249482cf..b7e6e7d05 100644 --- a/tests/smoke/REGISTERED.md +++ b/tests/smoke/REGISTERED.md @@ -21,6 +21,12 @@ an explicitly supplied case ref. `infra` carries no cases yet, and a lane with no case in the running mode is never resolved, so it needs no definition until the first `infra` case lands. +The `agentic` lane accepts a non-secret queue-time override for +`ADO_AW_MODEL_AGENT_COPILOT`. If the project setting limiting queue-time +variables is enabled, declare that definition variable with +`allowOverride=true`; the `runtime-model-queue` case supplies its value through +the Build REST API and verifies the resulting audit metadata. + **Only `agentic` needs registering at cutover** — one definition for the whole suite. diff --git a/tests/smoke/cases.json b/tests/smoke/cases.json index 79cffd7fa..7d20a0df2 100644 --- a/tests/smoke/cases.json +++ b/tests/smoke/cases.json @@ -73,6 +73,33 @@ "modes": ["candidate"], "source": "tests/smoke/multi-repo.md" }, + { + "id": "runtime-model-queue", + "lane": "agentic", + "kind": "compiled", + "modes": ["candidate"], + "source": "tests/smoke/runtime-model-queue.md", + "queueVariables": { + "ADO_AW_MODEL_AGENT_COPILOT": "auto" + }, + "assertions": { + "requestedModels": { + "agent": "auto" + } + } + }, + { + "id": "runtime-model-set-variable", + "lane": "agentic", + "kind": "compiled", + "modes": ["candidate"], + "source": "tests/smoke/runtime-model-set-variable.md", + "assertions": { + "requestedModels": { + "agent": "auto" + } + } + }, { "id": "janitor", "lane": "agentic", diff --git a/tests/smoke/runtime-model-queue.md b/tests/smoke/runtime-model-queue.md new file mode 100644 index 000000000..3ba191f79 --- /dev/null +++ b/tests/smoke/runtime-model-queue.md @@ -0,0 +1,19 @@ +--- +name: "Candidate compiler smoke: queue-time runtime model" +description: "Proves an ADO queue-time variable selects the Agent Copilot model" +target: standalone +pool: + name: AZS-1ES-L-Playground-ubuntu-22.04 +engine: + id: copilot + timeout-minutes: 15 +safe-outputs: + noop: {} + threat-detection: + enabled: false +--- + +## Queue-time runtime model smoke + +Call the `noop` safe-output tool exactly once with context +`runtime-model-queue-$(Build.BuildId)`, then stop. diff --git a/tests/smoke/runtime-model-set-variable.md b/tests/smoke/runtime-model-set-variable.md new file mode 100644 index 000000000..0889be6b3 --- /dev/null +++ b/tests/smoke/runtime-model-set-variable.md @@ -0,0 +1,24 @@ +--- +name: "Candidate compiler smoke: task-setvariable runtime model" +description: "Proves a preceding same-job task.setvariable selects the Agent Copilot model" +target: standalone +pool: + name: AZS-1ES-L-Playground-ubuntu-22.04 +engine: + id: copilot + timeout-minutes: 15 +steps: + - bash: | + set -euo pipefail + echo "##vso[task.setvariable variable=ADO_AW_MODEL_AGENT_COPILOT]auto" + displayName: Select Agent runtime model +safe-outputs: + noop: {} + threat-detection: + enabled: false +--- + +## Same-job runtime model smoke + +Call the `noop` safe-output tool exactly once with context +`runtime-model-set-variable-$(Build.BuildId)`, then stop. From cfde682c610325e3ef906b5fd4486b40c184f3f9 Mon Sep 17 00:00:00 2001 From: James Devine Date: Tue, 6 Oct 2026 12:59:36 +0100 Subject: [PATCH 21/24] test(smoke): retain detection in runtime model cases Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- tests/smoke/runtime-model-queue.md | 2 -- tests/smoke/runtime-model-set-variable.md | 2 -- 2 files changed, 4 deletions(-) diff --git a/tests/smoke/runtime-model-queue.md b/tests/smoke/runtime-model-queue.md index 3ba191f79..c1c42124e 100644 --- a/tests/smoke/runtime-model-queue.md +++ b/tests/smoke/runtime-model-queue.md @@ -9,8 +9,6 @@ engine: timeout-minutes: 15 safe-outputs: noop: {} - threat-detection: - enabled: false --- ## Queue-time runtime model smoke diff --git a/tests/smoke/runtime-model-set-variable.md b/tests/smoke/runtime-model-set-variable.md index 0889be6b3..643daad3b 100644 --- a/tests/smoke/runtime-model-set-variable.md +++ b/tests/smoke/runtime-model-set-variable.md @@ -14,8 +14,6 @@ steps: displayName: Select Agent runtime model safe-outputs: noop: {} - threat-detection: - enabled: false --- ## Same-job runtime model smoke From 6edf0ede597a9de0f7c649227d8256e02bfe1fde Mon Sep 17 00:00:00 2001 From: James Devine Date: Tue, 6 Oct 2026 15:06:28 +0100 Subject: [PATCH 22/24] test(smoke): assert concrete runtime model overrides Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- .../src/compiler-smoke-e2e/__tests__/index.test.ts | 2 +- .../src/compiler-smoke-e2e/__tests__/signals.test.ts | 6 +++--- tests/smoke/README.md | 4 ++-- tests/smoke/REGISTERED.md | 10 +++++----- tests/smoke/cases.json | 6 +++--- tests/smoke/runtime-model-set-variable.md | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts index c63b467ce..4b3d38b79 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts @@ -298,7 +298,7 @@ describe("smoke-e2e index.main (happy path, candidate mode)", () => { ]); expect( queuedRequests.find((request) => request.caseId === "runtime-model-queue")?.variables, - ).toEqual({ ADO_AW_MODEL_AGENT_COPILOT: "auto" }); + ).toEqual({ ADO_AW_MODEL_AGENT_COPILOT: "gpt-5.4" }); // Cleanup ordering: remote refs deleted BEFORE the local worktree is removed. expect(mockCalls.indexOf("deleteRemoteRefs")).toBeGreaterThanOrEqual(0); diff --git a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts index c800a5de7..ffa0dcbd5 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts @@ -42,7 +42,7 @@ const CASES: ResolvedCase[] = [ kind: "compiled", modes: ["candidate"], source: "tests/smoke/runtime-model-queue.md", - assertions: { requestedModels: { agent: "auto" } }, + assertions: { requestedModels: { agent: "gpt-5.4" } }, definitionId: 3006, }, ]; @@ -251,7 +251,7 @@ describe("verifyCandidateAudit", () => { .mockResolvedValueOnce({ status: 0, stdout: JSON.stringify({ - overview: { build_id: 43, aw_info: { model: "gpt-5.4" } }, + overview: { build_id: 43, aw_info: { model: "auto" } }, }), stderr: "", timedOut: false, @@ -269,6 +269,6 @@ describe("verifyCandidateAudit", () => { ); expect(outcome.ok).toBe(false); - expect(outcome.results[1]?.message).toMatch(/expected "auto"/); + expect(outcome.results[1]?.message).toMatch(/expected "gpt-5.4"/); }); }); diff --git a/tests/smoke/README.md b/tests/smoke/README.md index e53b03aa4..b3015315d 100644 --- a/tests/smoke/README.md +++ b/tests/smoke/README.md @@ -141,7 +141,7 @@ GitHub. "modes": ["candidate", "released"], "source": "tests/safe-outputs/my-case.md", "queueVariables": { // optional, non-secret ADO queue-time variables - "ADO_AW_MODEL_AGENT_COPILOT": "auto" + "ADO_AW_MODEL_AGENT_COPILOT": "gpt-5.4" } } ``` @@ -182,7 +182,7 @@ Optional per-case assertions, so novel checks stay out of the harness code: "forbidden": ["--network host"] }, "requiredBuildTags": ["ado-aw-custom-job-{buildId}"], - "requestedModels": { "agent": "auto" } + "requestedModels": { "agent": "gpt-5.4" } } ``` diff --git a/tests/smoke/REGISTERED.md b/tests/smoke/REGISTERED.md index b7e6e7d05..0fcd806fe 100644 --- a/tests/smoke/REGISTERED.md +++ b/tests/smoke/REGISTERED.md @@ -21,11 +21,11 @@ an explicitly supplied case ref. `infra` carries no cases yet, and a lane with no case in the running mode is never resolved, so it needs no definition until the first `infra` case lands. -The `agentic` lane accepts a non-secret queue-time override for -`ADO_AW_MODEL_AGENT_COPILOT`. If the project setting limiting queue-time -variables is enabled, declare that definition variable with -`allowOverride=true`; the `runtime-model-queue` case supplies its value through -the Build REST API and verifies the resulting audit metadata. +The `agentic` lane declares an empty, non-secret +`ADO_AW_MODEL_AGENT_COPILOT` variable with `allowOverride=true`. The empty +definition value preserves the normal Copilot default for other cases, while +`runtime-model-queue` supplies a concrete value through the Build REST API and +verifies the resulting audit metadata. **Only `agentic` needs registering at cutover** — one definition for the whole suite. diff --git a/tests/smoke/cases.json b/tests/smoke/cases.json index 7d20a0df2..2418cc631 100644 --- a/tests/smoke/cases.json +++ b/tests/smoke/cases.json @@ -80,11 +80,11 @@ "modes": ["candidate"], "source": "tests/smoke/runtime-model-queue.md", "queueVariables": { - "ADO_AW_MODEL_AGENT_COPILOT": "auto" + "ADO_AW_MODEL_AGENT_COPILOT": "gpt-5.4" }, "assertions": { "requestedModels": { - "agent": "auto" + "agent": "gpt-5.4" } } }, @@ -96,7 +96,7 @@ "source": "tests/smoke/runtime-model-set-variable.md", "assertions": { "requestedModels": { - "agent": "auto" + "agent": "gpt-5.4" } } }, diff --git a/tests/smoke/runtime-model-set-variable.md b/tests/smoke/runtime-model-set-variable.md index 643daad3b..41ecc2977 100644 --- a/tests/smoke/runtime-model-set-variable.md +++ b/tests/smoke/runtime-model-set-variable.md @@ -10,7 +10,7 @@ engine: steps: - bash: | set -euo pipefail - echo "##vso[task.setvariable variable=ADO_AW_MODEL_AGENT_COPILOT]auto" + echo "##vso[task.setvariable variable=ADO_AW_MODEL_AGENT_COPILOT]gpt-5.4" displayName: Select Agent runtime model safe-outputs: noop: {} From 42400dd7364fba9861ad9358fa63befc2dbc60fe Mon Sep 17 00:00:00 2001 From: James Devine Date: Tue, 6 Oct 2026 21:34:17 +0100 Subject: [PATCH 23/24] fix(smoke): encode queue variables as build parameters Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- .../src/compiler-smoke-e2e/__tests__/ado-rest.test.ts | 5 +++-- scripts/ado-script/src/compiler-smoke-e2e/ado-rest.ts | 8 +++++--- tests/smoke/README.md | 3 ++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/ado-rest.test.ts b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/ado-rest.test.ts index f2960ec9d..e6b66be7b 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/ado-rest.test.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/ado-rest.test.ts @@ -99,7 +99,7 @@ describe("AdoRest.queueBuild", () => { }); }); - it("serializes queue variables using the ADO Build API variable shape", async () => { + it("serializes queue variables through the Build Queue parameters string", async () => { let sentBody: unknown; const fetchImpl = vi.fn( async (_input: RequestInfo | URL, init?: RequestInit) => { @@ -114,8 +114,9 @@ describe("AdoRest.queueBuild", () => { variables: { ADO_AW_MODEL_AGENT_COPILOT: "auto" }, }); expect(sentBody).toMatchObject({ - variables: { ADO_AW_MODEL_AGENT_COPILOT: { value: "auto" } }, + parameters: JSON.stringify({ ADO_AW_MODEL_AGENT_COPILOT: "auto" }), }); + expect(sentBody).not.toHaveProperty("variables"); }); it("throws with a descriptive error on a non-2xx response", async () => { diff --git a/scripts/ado-script/src/compiler-smoke-e2e/ado-rest.ts b/scripts/ado-script/src/compiler-smoke-e2e/ado-rest.ts index ef78bce27..688cc54bb 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/ado-rest.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/ado-rest.ts @@ -262,9 +262,11 @@ export class AdoRest { sourceVersion: opts.sourceVersion, }; if (opts.variables && Object.keys(opts.variables).length > 0) { - body.variables = Object.fromEntries( - Object.entries(opts.variables).map(([name, value]) => [name, { value }]), - ); + // Build Queue's legacy `parameters` string is how `az pipelines run + // --variables` sends queue-time variables. A top-level `variables` + // object belongs to the Pipelines Runs API and is silently ignored by + // this endpoint. + body.parameters = JSON.stringify(opts.variables); } const res = await this.request<{ id: number }>(path, { method: "POST", body }); if (!res) throw new Error(`queueBuild(${definitionId}) returned no body`); diff --git a/tests/smoke/README.md b/tests/smoke/README.md index b3015315d..9763054aa 100644 --- a/tests/smoke/README.md +++ b/tests/smoke/README.md @@ -188,7 +188,8 @@ Optional per-case assertions, so novel checks stay out of the harness code: `requestedModels` runs `ado-aw audit` against the completed child and compares the requested Agent and/or Detection model recorded in `overview.aw_info`. -Queue variables use the Build API's `variables` object and therefore exercise +Queue variables use the Build Queue API's `parameters` JSON string, matching +the encoding used by `az pipelines run --variables`, and therefore exercise the same runtime source as variables supplied in the Azure DevOps Run Pipeline UI. A lane definition may need the variable predeclared with `allowOverride=true` when the project restricts queue-time variables. From 109a09358f16c8fbefefe4439f04d225eec72830 Mon Sep 17 00:00:00 2001 From: James Devine Date: Wed, 7 Oct 2026 10:53:55 +0100 Subject: [PATCH 24/24] test(smoke): use GPT-6 Luna for runtime overrides Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 948a095f-e847-4d07-9f93-19a4bab86459 --- .../src/compiler-smoke-e2e/__tests__/index.test.ts | 2 +- .../src/compiler-smoke-e2e/__tests__/signals.test.ts | 4 ++-- tests/smoke/README.md | 4 ++-- tests/smoke/cases.json | 6 +++--- tests/smoke/runtime-model-set-variable.md | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts index 4b3d38b79..549ac290f 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/index.test.ts @@ -298,7 +298,7 @@ describe("smoke-e2e index.main (happy path, candidate mode)", () => { ]); expect( queuedRequests.find((request) => request.caseId === "runtime-model-queue")?.variables, - ).toEqual({ ADO_AW_MODEL_AGENT_COPILOT: "gpt-5.4" }); + ).toEqual({ ADO_AW_MODEL_AGENT_COPILOT: "gpt-6-luna" }); // Cleanup ordering: remote refs deleted BEFORE the local worktree is removed. expect(mockCalls.indexOf("deleteRemoteRefs")).toBeGreaterThanOrEqual(0); diff --git a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts index ffa0dcbd5..d67b7cf3d 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/__tests__/signals.test.ts @@ -42,7 +42,7 @@ const CASES: ResolvedCase[] = [ kind: "compiled", modes: ["candidate"], source: "tests/smoke/runtime-model-queue.md", - assertions: { requestedModels: { agent: "gpt-5.4" } }, + assertions: { requestedModels: { agent: "gpt-6-luna" } }, definitionId: 3006, }, ]; @@ -269,6 +269,6 @@ describe("verifyCandidateAudit", () => { ); expect(outcome.ok).toBe(false); - expect(outcome.results[1]?.message).toMatch(/expected "gpt-5.4"/); + expect(outcome.results[1]?.message).toMatch(/expected "gpt-6-luna"/); }); }); diff --git a/tests/smoke/README.md b/tests/smoke/README.md index 9763054aa..910939ceb 100644 --- a/tests/smoke/README.md +++ b/tests/smoke/README.md @@ -141,7 +141,7 @@ GitHub. "modes": ["candidate", "released"], "source": "tests/safe-outputs/my-case.md", "queueVariables": { // optional, non-secret ADO queue-time variables - "ADO_AW_MODEL_AGENT_COPILOT": "gpt-5.4" + "ADO_AW_MODEL_AGENT_COPILOT": "gpt-6-luna" } } ``` @@ -182,7 +182,7 @@ Optional per-case assertions, so novel checks stay out of the harness code: "forbidden": ["--network host"] }, "requiredBuildTags": ["ado-aw-custom-job-{buildId}"], - "requestedModels": { "agent": "gpt-5.4" } + "requestedModels": { "agent": "gpt-6-luna" } } ``` diff --git a/tests/smoke/cases.json b/tests/smoke/cases.json index 2418cc631..3a4195844 100644 --- a/tests/smoke/cases.json +++ b/tests/smoke/cases.json @@ -80,11 +80,11 @@ "modes": ["candidate"], "source": "tests/smoke/runtime-model-queue.md", "queueVariables": { - "ADO_AW_MODEL_AGENT_COPILOT": "gpt-5.4" + "ADO_AW_MODEL_AGENT_COPILOT": "gpt-6-luna" }, "assertions": { "requestedModels": { - "agent": "gpt-5.4" + "agent": "gpt-6-luna" } } }, @@ -96,7 +96,7 @@ "source": "tests/smoke/runtime-model-set-variable.md", "assertions": { "requestedModels": { - "agent": "gpt-5.4" + "agent": "gpt-6-luna" } } }, diff --git a/tests/smoke/runtime-model-set-variable.md b/tests/smoke/runtime-model-set-variable.md index 41ecc2977..8728d0722 100644 --- a/tests/smoke/runtime-model-set-variable.md +++ b/tests/smoke/runtime-model-set-variable.md @@ -10,7 +10,7 @@ engine: steps: - bash: | set -euo pipefail - echo "##vso[task.setvariable variable=ADO_AW_MODEL_AGENT_COPILOT]gpt-5.4" + echo "##vso[task.setvariable variable=ADO_AW_MODEL_AGENT_COPILOT]gpt-6-luna" displayName: Select Agent runtime model safe-outputs: noop: {}