From ef4d1be2eacc6757121065aa86f221ab4f9546c1 Mon Sep 17 00:00:00 2001
From: Christopher Bartz <christopher.bartz@canonical.com>
Date: Wed, 8 Jul 2026 20:56:46 +0000
Subject: [PATCH 1/4] fix(garm): deliver aproxy bootstrap as pre-install script
 so it runs before GARM's install wrapper

When a scaleset uses a server-side template, GARM injects a compiled-in
wrapper as the cloud-init install script; the wrapper must reach the
GARM API to fetch the template before any template content runs. Proxy
bootstrap living inside the template therefore can never work on
proxy-only networks: the wrapper's fetch has no egress until the proxy
is up.

The aproxy setup is now rendered as a standalone script and delivered
via the pre_install_scripts extra-spec (key "00-aproxy", sorting before
operator scripts), which GARM runs as root before the wrapper. When a
proxy is configured, apt updates are also disabled on boot
(disable_updates extra-spec): cloud-init's package upgrade runs before
any pre-install script, so it can never use the proxy and only burns
boot time on timeouts.

Two related fixes: `snap set aproxy proxy=` now receives a bare
host:port (aproxy rejects URLs; the new `snap set system
proxy.http/https=` calls take the full URL), and all pre_install_scripts
values are base64-encoded on the wire as required by GARM's
map[string][]byte extra-spec type.
---
 charms/garm/src/charm_state.py                |   1 +
 charms/garm/src/runner_template.py            | 108 +++++++++-----
 charms/garm/src/scaleset_reconciler.py        |  68 +++++++--
 charms/garm/src/templates/aproxy.j2           |  14 +-
 charms/garm/src/templates/pre_install.j2      |   3 -
 .../garm/tests/unit/test_runner_template.py   | 129 +++++++++++-----
 .../tests/unit/test_scaleset_reconciler.py    | 141 +++++++++++++++++-
 charms/tests/integration/test_garm.py         |  28 +++-
 docs/changelog.md                             |   1 +
 9 files changed, 391 insertions(+), 102 deletions(-)

diff --git a/charms/garm/src/charm_state.py b/charms/garm/src/charm_state.py
index 30f44c74..8e5f7f9e 100644
--- a/charms/garm/src/charm_state.py
+++ b/charms/garm/src/charm_state.py
@@ -102,6 +102,7 @@ def from_databag(cls, data: Mapping[str, str]) -> "RunnerConfig":
         )
         values["otel_collector_endpoint"] = _strip_newlines(values["otel_collector_endpoint"])
         values["dockerhub_mirror"] = _strip_newlines(values["dockerhub_mirror"])
+        values["runner_http_proxy"] = _strip_newlines(values["runner_http_proxy"])
         return cls(**values)
 
     def has_config(self) -> bool:
diff --git a/charms/garm/src/runner_template.py b/charms/garm/src/runner_template.py
index 6da1a531..df33d125 100644
--- a/charms/garm/src/runner_template.py
+++ b/charms/garm/src/runner_template.py
@@ -4,18 +4,26 @@
 """Render runner-behaviour options into a GARM runner-install template.
 
 GARM stores the runner-install script as a server-side template that a
-scaleset references by ``template_id``. The six operator-facing runner options
-(docker mirror, proxy/aproxy, telemetry, pre-job hook) are delivered by copying
-GARM's system ``github_linux`` template and injecting two blocks right after the
+scaleset references by ``template_id``. Three of the six operator-facing runner
+options (docker mirror, telemetry, pre-job hook) are delivered by copying GARM's
+system ``github_linux`` template and injecting two blocks right after the
 shebang — before the base template's ``set -e`` — so a best-effort step can never
 abort the whole bootstrap:
 
   * a *pre-install* block that runs as root before the runner is installed
-    (aproxy, docker registry mirror, static host prep), and
+    (docker registry mirror, static host prep), and
   * a *runner job hooks* block that writes the GitHub job-start hook and the
     runner ``env`` file under ``/home/runner/actions-runner`` (GARM hardcodes the
     ``runner`` user and that path).
 
+The proxy/aproxy option is *not* injected into this template: GARM prepends a
+compiled-in wrapper script that curls this template from the GARM metadata API
+before any of its content runs, so proxy bootstrap embedded in the template
+could never help that curl succeed on a proxy-only network. It is instead
+rendered by :func:`render_aproxy_pre_install_script` and delivered as a
+``pre_install_scripts`` extra-spec entry (see ``scaleset_reconciler``), which
+GARM runs, as root, before the wrapper.
+
 The reconciler creates/updates the per-scaleset template with the bytes returned
 by :func:`build_template_data` and only does so when :meth:`RunnerConfig.has_config`
 is true.
@@ -29,6 +37,7 @@
 import json
 import pathlib
 import shlex
+import urllib.parse
 
 import jinja2
 
@@ -102,12 +111,11 @@ def render_pre_install(config: RunnerConfig) -> str:
     Returns:
         A bash snippet, terminated by a newline.
     """
-    aproxy = _render_aproxy(config) if config.runner_http_proxy else ""
     dockerhub = (
         _render_dockerhub_mirror(config.dockerhub_mirror) if config.dockerhub_mirror else ""
     )
     return _PRE_INSTALL_TEMPLATE.render(
-        aproxy=aproxy, dockerhub=dockerhub, static_prep=_render_static_host_prep()
+        dockerhub=dockerhub, static_prep=_render_static_host_prep()
     )
 
 
@@ -156,6 +164,60 @@ def render_pre_job_hooks(config: RunnerConfig) -> str:
     )
 
 
+def render_aproxy_pre_install_script(config: RunnerConfig) -> str:
+    """Render the standalone aproxy bootstrap script.
+
+    The script installs aproxy, points it and the snap store at the configured
+    proxy, and writes an nftables DNAT ruleset that transparently redirects
+    egress traffic to aproxy's listener, both for locally-initiated connections
+    (``output`` chain) and forwarded ones (``prerouting`` chain).
+
+    Args:
+        config: The runner options (uses proxy, exclude addresses, redirect ports).
+
+    Returns:
+        A complete best-effort bash script (shebang first line, deliberately no
+        ``set -e``) configuring aproxy as a transparent forward proxy.
+    """
+    # Values are already validated at parse time (RunnerConfig.from_databag); only
+    # shell-safe embedding (quoting, the empty-string split guard below) happens here.
+    ports = [t for t in config.aproxy_redirect_ports.split(",") if t] or ["80", "443"]
+    nft_ports = ", ".join(ports)
+    excludes = [t for t in config.aproxy_exclude_addresses.split(",") if t]
+    exclude_elements = ", ".join(["127.0.0.0/8", *excludes])
+    # `\$default-ipv4` stays escaped so the literal two characters `$default-ipv4`
+    # land in the file, which nft itself resolves against the `define` above when
+    # it loads the file.
+    dnat_rule = (
+        f"ip daddr != @exclude tcp dport {{ {nft_ports} }} counter dnat to \\$default-ipv4:54969"
+    )
+    return _APROXY_TEMPLATE.render(
+        proxy=shlex.quote(config.runner_http_proxy),
+        proxy_host_port=shlex.quote(_proxy_host_port(config.runner_http_proxy)),
+        exclude_elements=exclude_elements,
+        dnat_rule=dnat_rule,
+    )
+
+
+def _proxy_host_port(proxy: str) -> str:
+    """Return the bare authority (``host:port``, optionally ``user:pass@``) of a proxy.
+
+    aproxy rejects a full URL for its ``proxy=`` setting, unlike ``snap set
+    system proxy.http/https=``, so it needs the scheme stripped. The URL
+    ``netloc`` is returned verbatim, preserving userinfo and IPv6 brackets that
+    ``urlsplit().hostname`` would drop. Accepts a URL (``http://host:port``) or
+    an already-bare ``host:port``.
+
+    Args:
+        proxy: The configured proxy value, with or without a URL scheme.
+
+    Returns:
+        The proxy authority with any scheme and path removed.
+    """
+    candidate = proxy if "//" in proxy else f"//{proxy}"
+    return urllib.parse.urlsplit(candidate).netloc
+
+
 def _render_pre_job_hook_body(config: RunnerConfig, otel_endpoint: str) -> str:
     """Render the contents of the pre-job hook file (the GitHub job-start hook).
 
@@ -207,40 +269,6 @@ def _render_custom_pre_job_script(script: str) -> str:
     return _CUSTOM_PRE_JOB_SCRIPT_TEMPLATE.render(delim=delim, script=script)
 
 
-def _render_aproxy(config: RunnerConfig) -> str:
-    """Render aproxy install + nftables redirect rules.
-
-    aproxy listens on ``:54969`` and an nftables DNAT ruleset (written to
-    ``/etc/nftables.conf``) redirects egress traffic to it, both for
-    locally-initiated connections (``output`` chain) and forwarded ones
-    (``prerouting`` chain).
-
-    Args:
-        config: The runner options (uses proxy, exclude addresses, redirect ports).
-
-    Returns:
-        A bash snippet configuring aproxy as a transparent forward proxy.
-    """
-    # Values are already validated at parse time (RunnerConfig.from_databag); only
-    # shell-safe embedding (quoting, the empty-string split guard below) happens here.
-    ports = [t for t in config.aproxy_redirect_ports.split(",") if t] or ["80", "443"]
-    nft_ports = ", ".join(ports)
-    excludes = [t for t in config.aproxy_exclude_addresses.split(",") if t]
-    exclude_elements = ", ".join(["127.0.0.0/8", *excludes])
-    # `\$default-ipv4` stays escaped so the literal two characters `$default-ipv4`
-    # land in the file, which nft itself resolves against the `define` above when
-    # it loads the file.
-    dnat_rule = (
-        f"ip daddr != @exclude tcp dport {{ {nft_ports} }} counter dnat to \\$default-ipv4:54969"
-    )
-
-    return _APROXY_TEMPLATE.render(
-        proxy=shlex.quote(config.runner_http_proxy),
-        exclude_elements=exclude_elements,
-        dnat_rule=dnat_rule,
-    )
-
-
 def _render_dockerhub_mirror(mirror: str) -> str:
     """Render Docker daemon config pointing at the registry mirror.
 
diff --git a/charms/garm/src/scaleset_reconciler.py b/charms/garm/src/scaleset_reconciler.py
index 58734e7d..2f616d60 100644
--- a/charms/garm/src/scaleset_reconciler.py
+++ b/charms/garm/src/scaleset_reconciler.py
@@ -14,7 +14,7 @@
 from garm_client.models.scale_set import ScaleSet
 from garm_client.models.template import Template
 from garm_client.models.update_scale_set_params import UpdateScaleSetParams
-from runner_template import build_template_data
+from runner_template import build_template_data, render_aproxy_pre_install_script
 
 logger = logging.getLogger(__name__)
 
@@ -22,6 +22,11 @@
 # build per-scaleset runner templates carrying the operator's runner options.
 SYSTEM_TEMPLATE_NAME = "github_linux"
 
+# GARM runs pre-install scripts in lexicographic key order; "00-aproxy" sorts
+# before the configurator's "pre_install.sh", so the proxy is up before any
+# operator-supplied script runs.
+APROXY_SCRIPT_NAME = "00-aproxy"
+
 
 @dataclass
 class ScalesetSpec:
@@ -345,9 +350,6 @@ def _to_create_params(spec: ScalesetSpec) -> CreateScaleSetParams:
         Raises:
             ValidationError: If the spec data fails Pydantic model validation.
         """
-        extra_specs: dict[str, object] = {}
-        if spec.pre_install_scripts:
-            extra_specs["pre_install_scripts"] = spec.pre_install_scripts
         return CreateScaleSetParams.model_validate(
             {
                 "name": spec.name,
@@ -361,7 +363,7 @@ def _to_create_params(spec: ScalesetSpec) -> CreateScaleSetParams:
                 "enabled": True,
                 "labels": sorted(spec.labels),
                 "github_runner_group": spec.runner_group or None,
-                "extra_specs": extra_specs or None,
+                "extra_specs": _effective_extra_specs(spec) or None,
                 "template_id": spec.template_id,
             }
         )
@@ -404,10 +406,13 @@ def _maybe_update(self, observed: ScaleSet, spec: ScalesetSpec, template_id: int
             logger.debug("Scaleset %s is up to date", spec.name)
             return
 
-        extra_specs: dict[str, object] = {}
-        if spec.pre_install_scripts:
-            extra_specs["pre_install_scripts"] = spec.pre_install_scripts
-
+        # UpdateScaleSetParams omits None fields (exclude_none), so None can only
+        # leave extra_specs untouched, never clear them. Send an explicit empty
+        # dict when the desired specs are empty but the scaleset still carries
+        # some (e.g. a proxy was unset) — otherwise a stale aproxy script would
+        # persist and _needs_update would loop forever trying to converge.
+        desired_extra = _effective_extra_specs(spec)
+        extra_specs = desired_extra or ({} if observed.extra_specs else None)
         params = UpdateScaleSetParams(
             image=spec.image,
             flavor=spec.flavor,
@@ -415,7 +420,7 @@ def _maybe_update(self, observed: ScaleSet, spec: ScalesetSpec, template_id: int
             max_runners=spec.max_runners,
             enabled=True,
             runner_group=spec.runner_group or None,
-            extra_specs=extra_specs or None,
+            extra_specs=extra_specs,
             template_id=spec.template_id,
         )
         # Send template_id when the scaleset has, or had, a custom template — a 0
@@ -431,7 +436,10 @@ def _maybe_update(self, observed: ScaleSet, spec: ScalesetSpec, template_id: int
 
     @staticmethod
     def _needs_update(observed: ScaleSet, spec: ScalesetSpec, template_id: int) -> bool:
-        observed_scripts = (observed.extra_specs or {}).get("pre_install_scripts", {})
+        # GARM round-trips extra_specs exactly as sent, so the observed script
+        # values are base64 like the desired ones — compare them encoded.
+        observed_extra = observed.extra_specs or {}
+        desired_extra = _effective_extra_specs(spec)
         return (
             observed.image != spec.image
             or observed.flavor != spec.flavor
@@ -439,6 +447,42 @@ def _needs_update(observed: ScaleSet, spec: ScalesetSpec, template_id: int) -> b
             or observed.min_idle_runners != spec.min_idle_runners
             or observed.enabled is not True
             or observed.github_runner_group != (spec.runner_group or None)
-            or observed_scripts != spec.pre_install_scripts
+            or observed_extra.get("pre_install_scripts", {})
+            != desired_extra.get("pre_install_scripts", {})
+            or bool(observed_extra.get("disable_updates"))
+            != bool(desired_extra.get("disable_updates"))
             or (observed.template_id or 0) != template_id
         )
+
+
+def _effective_extra_specs(spec: ScalesetSpec) -> dict[str, object]:
+    """Build the scaleset extra_specs a spec should produce.
+
+    Single source of truth for create, update, and drift detection: combines the
+    operator-supplied pre-install scripts with the charm's aproxy bootstrap and
+    the ``disable_updates`` flag when a runner proxy is configured.
+
+    Args:
+        spec: The desired scaleset.
+
+    Returns:
+        The extra_specs dict, with all script values base64-encoded (GARM decodes
+        ``pre_install_scripts`` as ``map[string][]byte``); empty when the spec
+        yields no extra specs.
+    """
+    scripts = dict(spec.pre_install_scripts)
+    extra_specs: dict[str, object] = {}
+    if spec.runner_config.runner_http_proxy:
+        # The aproxy bootstrap must run before GARM's compiled-in install wrapper
+        # (which needs egress to fetch the runner template), hence a pre-install
+        # script rather than template content.
+        scripts[APROXY_SCRIPT_NAME] = render_aproxy_pre_install_script(spec.runner_config)
+        # cloud-init's apt upgrade runs before any pre-install script, so it can
+        # never use the proxy — skip it instead of timing out on every mirror.
+        extra_specs["disable_updates"] = True
+    if scripts:
+        extra_specs["pre_install_scripts"] = {
+            name: base64.b64encode(content.encode("utf-8")).decode("utf-8")
+            for name, content in scripts.items()
+        }
+    return extra_specs
diff --git a/charms/garm/src/templates/aproxy.j2 b/charms/garm/src/templates/aproxy.j2
index f1a79e20..6a6bc256 100644
--- a/charms/garm/src/templates/aproxy.j2
+++ b/charms/garm/src/templates/aproxy.j2
@@ -1,6 +1,14 @@
-# Transparently forward runner egress through the configured HTTP proxy.
+#!/bin/bash
+# Best-effort proxy bootstrap, deliberately without strict-mode shell options:
+# a failure here must never abort the rest of GARM's pre-install/install-runner
+# sequence.
+
+# snapd itself needs the proxy to reach the snap store before aproxy exists.
+snap set system proxy.http={{ proxy }}
+snap set system proxy.https={{ proxy }}
+
 snap install aproxy --edge
-snap set aproxy proxy={{ proxy }} listen=:54969
+snap set aproxy proxy={{ proxy_host_port }} listen=:54969
 nft delete table ip aproxy 2>/dev/null || true
 cat << EOF > /etc/nftables.conf
 define default-ipv4 = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+')
@@ -21,4 +29,4 @@ table ip aproxy {
 }
 EOF
 systemctl enable nftables.service
-nft -f /etc/nftables.conf
\ No newline at end of file
+nft -f /etc/nftables.conf
diff --git a/charms/garm/src/templates/pre_install.j2 b/charms/garm/src/templates/pre_install.j2
index c2ed5f62..3859b3f9 100644
--- a/charms/garm/src/templates/pre_install.j2
+++ b/charms/garm/src/templates/pre_install.j2
@@ -1,8 +1,5 @@
 
 # ===== charm-injected pre-install setup (runs as root) =====
-{% if aproxy %}
-{{ aproxy }}
-{% endif %}
 {% if dockerhub %}
 {{ dockerhub }}
 {% endif %}
diff --git a/charms/garm/tests/unit/test_runner_template.py b/charms/garm/tests/unit/test_runner_template.py
index feaddc8a..d05e1e77 100644
--- a/charms/garm/tests/unit/test_runner_template.py
+++ b/charms/garm/tests/unit/test_runner_template.py
@@ -3,10 +3,12 @@
 
 """Unit tests for the runner-install template rendering."""
 
+import shlex
+
 import pytest
 
 from charm_state import RunnerConfig
-from runner_template import RUNNER_ENV_PATH, build_template_data
+from runner_template import RUNNER_ENV_PATH, build_template_data, render_aproxy_pre_install_script
 
 SAMPLE_BASE = b"#!/bin/bash\nset -e\necho original-bootstrap\n"
 
@@ -38,25 +40,14 @@ def test_build_template_data_injects_all_options():
 
     assert "mirror.example.com" in result
     assert "registry-mirrors" in result
-    assert "proxy.example.com:3128" in result
-    assert "snap install aproxy" in result
-    assert "10.0.0.0/8" in result and "192.168.1.5" in result
-    assert "8000-9000" in result
     assert "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT=http://otel.example.com:4318" in result
     assert "echo hello-from-operator" in result
     assert "ACTIONS_RUNNER_HOOK_JOB_STARTED=" in result
     assert RUNNER_ENV_PATH in result
 
-    # aproxy: correct listen port, nftables file (not a piped `nft -f -`), and
-    # both the prerouting and output DNAT chains.
-    assert "listen=:54969" in result
-    assert "default-ipv4" in result
-    assert "/etc/nftables.conf" in result
-    assert "table ip aproxy\nflush table ip aproxy" not in result
-    assert "nft delete table ip aproxy 2>/dev/null || true" in result
-    assert "prerouting" in result
-    assert "127.0.0.0/8" in result
-    assert "counter dnat to \\$default-ipv4:54969" in result
+    # aproxy is delivered as a pre-install script (scaleset extra_specs), not
+    # injected into the runner template — see render_aproxy_pre_install_script.
+    assert "snap install aproxy" not in result
 
     # docker mirror: both env vars and a full daemon-reload + restart.
     assert "DOCKERHUB_MIRROR=https://mirror.example.com" in result
@@ -97,19 +88,13 @@ def test_build_template_data_empty_config_omits_optional_blocks():
         pytest.param(
             RunnerConfig(dockerhub_mirror="https://m.test"),
             "https://m.test",
-            "snap install aproxy",
+            "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT",
             id="dockerhub-mirror-only",
         ),
-        pytest.param(
-            RunnerConfig(runner_http_proxy="http://p.test:8080"),
-            "http://p.test:8080",
-            "registry-mirrors",
-            id="proxy-only",
-        ),
         pytest.param(
             RunnerConfig(otel_collector_endpoint="http://o.test:4318"),
             "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT=http://o.test:4318",
-            "snap install aproxy",
+            "registry-mirrors",
             id="otel-only",
         ),
         pytest.param(
@@ -134,7 +119,7 @@ def test_build_template_data_per_option(config, present, absent):
 def test_aproxy_render_uses_configured_ports_and_excludes():
     """
     arrange: A runner config with a proxy plus well-formed redirect ports and excludes.
-    act: Build the runner template data.
+    act: Render the standalone aproxy pre-install script.
     assert: The nft ruleset embeds exactly those ports and addresses.
     """
     config = RunnerConfig(
@@ -142,12 +127,71 @@ def test_aproxy_render_uses_configured_ports_and_excludes():
         aproxy_redirect_ports="80,8000-9000",
         aproxy_exclude_addresses="10.0.0.0/8",
     )
-    result = build_template_data(SAMPLE_BASE, config).decode()
+    result = render_aproxy_pre_install_script(config)
 
     assert "tcp dport { 80, 8000-9000 }" in result
     assert "10.0.0.0/8" in result
 
 
+def test_render_aproxy_pre_install_script_happy_path():
+    """
+    arrange: A runner config with every aproxy-related field set to a realistic value.
+    act: Render the standalone aproxy pre-install script.
+    assert: The script is a standalone bash script that points the snap store and
+        aproxy at the proxy, and installs the nftables redirect.
+    """
+    config = RunnerConfig(
+        runner_http_proxy="http://proxy.example.com:3128",
+        aproxy_exclude_addresses="10.0.0.0/8",
+        aproxy_redirect_ports="80,443",
+    )
+    result = render_aproxy_pre_install_script(config)
+    lines = result.splitlines()
+
+    assert lines[0] == "#!/bin/bash"
+    assert "set -e" not in result
+
+    assert "snap set system proxy.http=http://proxy.example.com:3128" in result
+    assert "snap set system proxy.https=http://proxy.example.com:3128" in result
+
+    assert "snap install aproxy" in result
+    assert "snap set aproxy proxy=proxy.example.com:3128 listen=:54969" in result
+
+    assert "tcp dport { 80, 443 }" in result
+    assert "10.0.0.0/8" in result
+    assert "127.0.0.0/8" in result
+    assert "counter dnat to \\$default-ipv4:54969" in result
+    assert "/etc/nftables.conf" in result
+
+
+@pytest.mark.parametrize(
+    "proxy, expected",
+    [
+        ("http://h:3128", "h:3128"),
+        ("https://h:3128", "h:3128"),
+        ("h:3128", "h:3128"),
+        # netloc, not hostname:port — userinfo and IPv6 brackets must survive so
+        # aproxy receives a usable authority for authenticated / IPv6 proxies.
+        ("http://user:pass@h:3128", "user:pass@h:3128"),
+        ("http://[2001:db8::1]:3128", "[2001:db8::1]:3128"),
+    ],
+    ids=["http-scheme", "https-scheme", "no-scheme", "userinfo", "ipv6"],
+)
+def test_render_aproxy_pre_install_script_strips_scheme_for_aproxy(proxy, expected):
+    """
+    arrange: A proxy value with a scheme, no scheme, embedded credentials, or an IPv6 host.
+    act: Render the standalone aproxy pre-install script.
+    assert: aproxy is configured with the bare authority (it rejects URLs) with userinfo
+        and IPv6 brackets preserved.
+    """
+    config = RunnerConfig(runner_http_proxy=proxy)
+    result = render_aproxy_pre_install_script(config)
+
+    # The value is shlex-quoted in the script; an IPv6 authority thus renders
+    # bracket-quoted, which is exactly why the assertion quotes the expectation.
+    assert f"snap set aproxy proxy={shlex.quote(expected)}" in result
+
+
 def test_heredoc_delimiter_avoids_collision_with_pre_job_script():
     """
     arrange: A pre-job script containing the default heredoc delimiter token.
@@ -163,29 +207,46 @@ def test_heredoc_delimiter_avoids_collision_with_pre_job_script():
 
 def test_build_template_data_from_untrusted_databag_drops_malformed_values():
     """
-    arrange: A raw relation databag with malformed ports/addresses and newline-bearing values.
+    arrange: A raw relation databag with newline-bearing otel/dockerhub values.
     act: Parse it via RunnerConfig.from_databag and build the runner template data.
-    assert: The malformed/injected content never reaches the rendered script.
+    assert: Newlines are stripped so each value stays on a single line (the injected
+        suffix is retained, but flattened onto the same env-file line, not a new one).
     """
     config = RunnerConfig.from_databag(
         {
-            "runner_http_proxy": "http://p.test:3128",
-            "aproxy_redirect_ports": "80,not-a-port,8000-9000,99999,443-80",
-            "aproxy_exclude_addresses": "10.0.0.0/8,evil;,2001:db8::1",
             "otel_collector_endpoint": "http://o.test:4318\nMALICIOUS=1",
             "dockerhub_mirror": "https://m.test\nMALICIOUS=1",
         }
     )
     result = build_template_data(SAMPLE_BASE, config).decode()
 
+    assert "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT=http://o.test:4318MALICIOUS=1" in result
+    assert "DOCKERHUB_MIRROR=https://m.testMALICIOUS=1" in result
+    assert "CONTAINER_REGISTRY_URL=https://m.testMALICIOUS=1" in result
+    assert "\nMALICIOUS=1" not in result
+
+
+def test_render_aproxy_pre_install_script_from_untrusted_databag_drops_malformed_values():
+    """
+    arrange: A raw relation databag with malformed/out-of-range ports and addresses.
+    act: Parse it via RunnerConfig.from_databag and render the aproxy pre-install script.
+    assert: The malformed values are dropped; well-formed ones still render.
+    """
+    config = RunnerConfig.from_databag(
+        {
+            "runner_http_proxy": "http://p.test:3128\nMALICIOUS=1",
+            "aproxy_redirect_ports": "80,not-a-port,8000-9000,99999,443-80",
+            "aproxy_exclude_addresses": "10.0.0.0/8,evil;,2001:db8::1",
+        }
+    )
+    result = render_aproxy_pre_install_script(config)
+
     assert "tcp dport { 80, 8000-9000 }" in result
     assert "not-a-port" not in result
     assert "99999" not in result  # out of range
     assert "443-80" not in result  # inverted range
     assert "10.0.0.0/8" in result
+    # A newline injected into the proxy value must not survive into the script.
+    assert "\nMALICIOUS=1" not in result
     assert "evil" not in result
     assert "2001:db8::1" not in result  # IPv6 dropped: the nft table is IPv4-only
-    assert "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT=http://o.test:4318MALICIOUS=1" in result
-    assert "DOCKERHUB_MIRROR=https://m.testMALICIOUS=1" in result
-    assert "CONTAINER_REGISTRY_URL=https://m.testMALICIOUS=1" in result
-    assert "\nMALICIOUS=1" not in result
diff --git a/charms/garm/tests/unit/test_scaleset_reconciler.py b/charms/garm/tests/unit/test_scaleset_reconciler.py
index 34614830..993a21ff 100644
--- a/charms/garm/tests/unit/test_scaleset_reconciler.py
+++ b/charms/garm/tests/unit/test_scaleset_reconciler.py
@@ -359,7 +359,8 @@ def test_pre_install_scripts_passed_in_create():
     """
     arrange: FakeGarmClient with provider registered and no existing scalesets.
     act: Reconcile a spec containing pre_install_scripts.
-    assert: Created scaleset params include extra_specs with the script mapping.
+    assert: Created scaleset params include extra_specs with the base64-encoded script
+        mapping (GARM's extra_specs field is map[string][]byte on the wire).
     """
     scripts = {"setup.sh": "#!/bin/bash\napt-get update"}
     client = FakeGarmClient(providers=["openstack-demo"], scalesets=[])
@@ -367,7 +368,143 @@ def test_pre_install_scripts_passed_in_create():
 
     assert len(client.created) == 1
     _, _, params = client.created[0]
-    assert params.extra_specs == {"pre_install_scripts": scripts}
+    assert params.extra_specs == {
+        "pre_install_scripts": {
+            name: base64.b64encode(content.encode()).decode() for name, content in scripts.items()
+        }
+    }
+
+
+def test_aproxy_pre_install_script_injected_when_proxy_configured():
+    """
+    arrange: A spec with runner_config.runner_http_proxy set and one operator script.
+    act: Reconcile a create.
+    assert: The created params disable cloud-init package upgrades and carry both the
+        operator script and a "00-aproxy" entry (sorts first) whose decoded content
+        configures aproxy for the given proxy.
+    """
+    from charm_state import RunnerConfig
+
+    client = FakeGarmClient(providers=["openstack-demo"], scalesets=[])
+    _reconcile(
+        client,
+        [
+            _spec(
+                pre_install_scripts={"pre_install.sh": "echo operator-script"},
+                runner_config=RunnerConfig(runner_http_proxy="http://squid.internal:3128"),
+            )
+        ],
+    )
+
+    assert len(client.created) == 1
+    _, _, params = client.created[0]
+    assert params.extra_specs["disable_updates"] is True
+    scripts = params.extra_specs["pre_install_scripts"]
+    assert set(scripts.keys()) == {"00-aproxy", "pre_install.sh"}
+    aproxy_script = base64.b64decode(scripts["00-aproxy"]).decode()
+    assert "snap set aproxy proxy=squid.internal:3128" in aproxy_script
+
+
+def test_no_update_when_extra_specs_already_match_encoded_desired_state():
+    """
+    arrange: An observed scaleset whose extra_specs already carry the base64-encoded
+        aproxy + operator scripts and disable_updates, matching the desired spec.
+    act: Reconcile that spec.
+    assert: No update is issued.
+    """
+    from charm_state import RunnerConfig
+
+    proxy = "http://squid.internal:3128"
+    spec = _spec(
+        pre_install_scripts={"pre_install.sh": "echo operator-script"},
+        runner_config=RunnerConfig(runner_http_proxy=proxy),
+    )
+    from scaleset_reconciler import _effective_extra_specs
+
+    extra_specs = _effective_extra_specs(spec)
+    client = FakeGarmClient(
+        providers=["openstack-demo"],
+        scalesets=[_existing_scaleset(extra_specs=extra_specs)],
+    )
+    _reconcile(client, [spec])
+
+    assert client.updated == []
+
+
+def test_update_when_observed_scripts_are_legacy_raw_text():
+    """
+    arrange: An observed scaleset whose extra_specs carry raw-text (unencoded) script
+        values from before base64-encoding was introduced.
+    act: Reconcile a spec whose desired scripts are the same content.
+    assert: An update is issued (the raw-text value never matches the encoded desired one).
+    """
+    from charm_state import RunnerConfig
+
+    spec = _spec(
+        pre_install_scripts={"pre_install.sh": "echo operator-script"},
+        runner_config=RunnerConfig(runner_http_proxy="http://squid.internal:3128"),
+    )
+    client = FakeGarmClient(
+        providers=["openstack-demo"],
+        scalesets=[
+            _existing_scaleset(
+                extra_specs={
+                    "disable_updates": True,
+                    "pre_install_scripts": {"pre_install.sh": "echo operator-script"},
+                }
+            )
+        ],
+    )
+    _reconcile(client, [spec])
+
+    assert len(client.updated) == 1
+
+
+def test_update_when_disable_updates_missing_but_proxy_configured():
+    """
+    arrange: An observed scaleset whose extra_specs carry the correctly-encoded scripts
+        but lack disable_updates, while the spec configures a proxy.
+    act: Reconcile that spec.
+    assert: An update is issued to set disable_updates.
+    """
+    from charm_state import RunnerConfig
+    from scaleset_reconciler import _effective_extra_specs
+
+    spec = _spec(runner_config=RunnerConfig(runner_http_proxy="http://squid.internal:3128"))
+    extra_specs = _effective_extra_specs(spec)
+    del extra_specs["disable_updates"]
+    client = FakeGarmClient(
+        providers=["openstack-demo"],
+        scalesets=[_existing_scaleset(extra_specs=extra_specs)],
+    )
+    _reconcile(client, [spec])
+
+    assert len(client.updated) == 1
+
+
+def test_update_clears_extra_specs_with_empty_dict_when_proxy_removed():
+    """
+    arrange: An observed scaleset carrying aproxy extra_specs, and a desired spec with no
+        proxy (and no operator scripts) so the effective extra_specs are empty.
+    act: Reconcile that spec.
+    assert: The update sends an explicit empty dict — not None, which exclude_none would
+        drop, leaving the stale extra_specs (and looping forever).
+    """
+    from charm_state import RunnerConfig
+    from scaleset_reconciler import _effective_extra_specs
+
+    stale = _effective_extra_specs(
+        _spec(runner_config=RunnerConfig(runner_http_proxy="http://squid.internal:3128"))
+    )
+    client = FakeGarmClient(
+        providers=["openstack-demo"],
+        scalesets=[_existing_scaleset(extra_specs=stale)],
+    )
+    _reconcile(client, [_spec(runner_config=RunnerConfig())])
+
+    assert len(client.updated) == 1
+    _, params = client.updated[0]
+    assert params.extra_specs == {}
 
 
 class _FakeTemplate:
diff --git a/charms/tests/integration/test_garm.py b/charms/tests/integration/test_garm.py
index 6a59719c..59dd80d1 100644
--- a/charms/tests/integration/test_garm.py
+++ b/charms/tests/integration/test_garm.py
@@ -870,7 +870,8 @@ def test_runner_options_render_into_scaleset_template(
         registered so a scaleset can be created.
     act: Set the runner-behaviour config options on the configurator charm.
     assert: The scaleset references a custom template whose rendered content
-        reflects each runner option — proving the options reach GARM via live
+        reflects each template-delivered runner option, and its extra_specs carry
+        the aproxy pre-install script — proving the options reach GARM via live
         reconcile (no upgrade).
     """
     address = _get_garm_address(juju, configurator_garm)
@@ -898,16 +899,12 @@ def test_runner_options_render_into_scaleset_template(
         delay=10,
     )
 
-    # One marker per config option, proving each reaches GARM via live reconcile:
-    # dockerhub-mirror (daemon.json + env var), runner-http-proxy + aproxy-*
-    # (aproxy listener and nftables ruleset), otel-collector-endpoint (env var),
-    # and pre-job-script (job-start hook).
+    # One marker per template-delivered config option, proving each reaches GARM
+    # via live reconcile: dockerhub-mirror (daemon.json + env var),
+    # otel-collector-endpoint (env var), and pre-job-script (job-start hook).
     expected_markers = (
         "registry-mirrors",
         "DOCKERHUB_MIRROR=https://mirror.example.com",
-        "proxy=http://proxy.example.com:3128 listen=:54969",
-        "192.168.0.0/16",
-        "tcp dport { 80, 443 }",
         "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT=http://otel.example.com:4318",
         "echo integration-marker",
     )
@@ -919,3 +916,18 @@ def test_runner_options_render_into_scaleset_template(
         _SCALESET_TEST_NAME,
         expected_markers,
     )
+
+    # runner-http-proxy + aproxy-* are delivered as a pre-install script via the
+    # scaleset's extra_specs (they must run before GARM's install wrapper), not
+    # via the template.
+    scaleset = _wait_for_scaleset(base_url, token, _SCALESET_TEST_NAME)
+    extra_specs = scaleset.get("extra_specs") or {}
+    assert extra_specs.get("disable_updates") is True, (
+        f"Expected disable_updates in extra_specs, got: {extra_specs}"
+    )
+    aproxy_encoded = (extra_specs.get("pre_install_scripts") or {}).get("00-aproxy")
+    assert aproxy_encoded, f"Expected a 00-aproxy pre-install script, got: {extra_specs}"
+    aproxy_script = base64.b64decode(aproxy_encoded).decode()
+    assert "proxy=proxy.example.com:3128 listen=:54969" in aproxy_script
+    assert "192.168.0.0/16" in aproxy_script
+    assert "tcp dport { 80, 443 }" in aproxy_script
diff --git a/docs/changelog.md b/docs/changelog.md
index b05d0a5f..f9aafdd0 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -10,6 +10,7 @@ Each revision is versioned by the date of the revision.
 
 ## 2026-07-08
 
+- `garm`: fix runner provisioning on proxy-only networks. GARM prepends a compiled-in wrapper as the cloud-init install script that must reach the GARM API before any runner-template content runs, so the aproxy bootstrap embedded in the template could never bring up egress in time and runners never registered. The aproxy setup is now delivered as a pre-install script (which GARM runs before the wrapper), apt updates are disabled on boot when a proxy is configured, and `snap set aproxy proxy=` now receives a bare `host:port` as aproxy requires.
 -  Fix runner provisioning in the GARM workload: the OpenStack provider binary in the bare-base rock was built dynamically linked, so GARM could not execute it (`fork/exec /usr/local/bin/garm-provider-openstack: no such file or directory`) and no runners were spawned, even though the charm reported `active`. The provider is now built as a fully static, pure-Go binary, matching the GARM binary.
 - `garm-configurator`: log the full config-validation detail when configuration is invalid. Juju truncates the blocked unit status to its first line, so a multi-line validation error (for example from an invalid runner option) was not fully visible. The charm now also logs the complete detail at warning level, so it is discoverable in `juju debug-log`.
 

From 7463814b3b87868edcc092120db10d630b056c8e Mon Sep 17 00:00:00 2001
From: Christopher Bartz <christopher.bartz@canonical.com>
Date: Thu, 9 Jul 2026 05:25:24 +0000
Subject: [PATCH 2/4] docs(garm): fix vale spellcheck on proxy changelog entry

Add 'aproxy' to the docs wordlist and reword 'prepends' (absent from the
base dictionary) to 'injects' so the docs spellcheck passes.
---
 docs/.custom_wordlist.txt | 1 +
 docs/changelog.md         | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/.custom_wordlist.txt b/docs/.custom_wordlist.txt
index b8f2301f..fd5aa154 100644
--- a/docs/.custom_wordlist.txt
+++ b/docs/.custom_wordlist.txt
@@ -85,3 +85,4 @@ configurator
 scaleset
 scalesets
 OAuth
+aproxy
diff --git a/docs/changelog.md b/docs/changelog.md
index f9aafdd0..99b446ce 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -10,7 +10,7 @@ Each revision is versioned by the date of the revision.
 
 ## 2026-07-08
 
-- `garm`: fix runner provisioning on proxy-only networks. GARM prepends a compiled-in wrapper as the cloud-init install script that must reach the GARM API before any runner-template content runs, so the aproxy bootstrap embedded in the template could never bring up egress in time and runners never registered. The aproxy setup is now delivered as a pre-install script (which GARM runs before the wrapper), apt updates are disabled on boot when a proxy is configured, and `snap set aproxy proxy=` now receives a bare `host:port` as aproxy requires.
+- `garm`: fix runner provisioning on proxy-only networks. GARM injects a compiled-in wrapper as the cloud-init install script that must reach the GARM API before any runner-template content runs, so the aproxy bootstrap embedded in the template could never bring up egress in time and runners never registered. The aproxy setup is now delivered as a pre-install script (which GARM runs before the wrapper), apt updates are disabled on boot when a proxy is configured, and `snap set aproxy proxy=` now receives a bare `host:port` as aproxy requires.
 -  Fix runner provisioning in the GARM workload: the OpenStack provider binary in the bare-base rock was built dynamically linked, so GARM could not execute it (`fork/exec /usr/local/bin/garm-provider-openstack: no such file or directory`) and no runners were spawned, even though the charm reported `active`. The provider is now built as a fully static, pure-Go binary, matching the GARM binary.
 - `garm-configurator`: log the full config-validation detail when configuration is invalid. Juju truncates the blocked unit status to its first line, so a multi-line validation error (for example from an invalid runner option) was not fully visible. The charm now also logs the complete detail at warning level, so it is discoverable in `juju debug-log`.
 

From a89be7d47c523f85839d0118037aa9cc45a9cba0 Mon Sep 17 00:00:00 2001
From: Christopher Bartz <christopher.bartz@canonical.com>
Date: Thu, 9 Jul 2026 05:36:52 +0000
Subject: [PATCH 3/4] fix(garm): gate aproxy redirect on the snap being up and
 log failures

The nftables DNAT sends :80/:443 to aproxy's :54969; applying it when the
aproxy snap failed to install or start would black-hole all egress into a
dead port rather than leaving the box on whatever path it had. Only install
the redirect once 'snap services aproxy' reports active, tolerate an
already-baked-in snap, and log every step (and both bail-out paths) to the
cloud-init console output so a broken proxy bootstrap is diagnosable.

Also correct the header comment: cloud-init runs each pre-install script as
its own runcmd entry and continues past a non-zero exit, so strict mode
would not have protected GARM's install sequence anyway.
---
 charms/garm/src/templates/aproxy.j2           | 39 ++++++++++++++++---
 .../garm/tests/unit/test_runner_template.py   |  9 +++++
 2 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/charms/garm/src/templates/aproxy.j2 b/charms/garm/src/templates/aproxy.j2
index 6a6bc256..4a8ecb92 100644
--- a/charms/garm/src/templates/aproxy.j2
+++ b/charms/garm/src/templates/aproxy.j2
@@ -1,14 +1,33 @@
 #!/bin/bash
-# Best-effort proxy bootstrap, deliberately without strict-mode shell options:
-# a failure here must never abort the rest of GARM's pre-install/install-runner
-# sequence.
+# aproxy proxy bootstrap, delivered as a GARM pre-install script so it runs
+# before the runner-install wrapper. cloud-init runs each pre-install script as
+# its own runcmd entry and continues past a non-zero exit, so strict mode here
+# would not protect GARM's sequence anyway -- a failure already surfaces
+# downstream as the wrapper's curl to the GARM API failing. Instead we log each
+# step (to /var/log/cloud-init-output.log and the serial console) so a broken
+# proxy bootstrap is diagnosable, and we install the nftables redirect only once
+# aproxy is actually listening: a DNAT to a dead :54969 would black-hole all
+# :80/:443 egress instead of leaving the box on whatever path it had.
+
+log() { echo "[aproxy-bootstrap] $*"; }
 
 # snapd itself needs the proxy to reach the snap store before aproxy exists.
 snap set system proxy.http={{ proxy }}
 snap set system proxy.https={{ proxy }}
 
-snap install aproxy --edge
-snap set aproxy proxy={{ proxy_host_port }} listen=:54969
+log "installing aproxy snap"
+# A non-zero exit is expected when aproxy is already baked into the image; only
+# bail if the snap is genuinely absent afterwards.
+if ! snap install aproxy --edge && ! snap list aproxy >/dev/null 2>&1; then
+    log "ERROR: aproxy install failed and the snap is not present; skipping redirect"
+    exit 0
+fi
+
+if ! snap set aproxy proxy={{ proxy_host_port }} listen=:54969; then
+    log "ERROR: failed to point aproxy at the upstream proxy; skipping redirect"
+    exit 0
+fi
+
 nft delete table ip aproxy 2>/dev/null || true
 cat << EOF > /etc/nftables.conf
 define default-ipv4 = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+')
@@ -29,4 +48,12 @@ table ip aproxy {
 }
 EOF
 systemctl enable nftables.service
-nft -f /etc/nftables.conf
+
+# Only redirect once aproxy is actually listening; otherwise the DNAT would
+# black-hole all :80/:443 egress by sending it to a dead port.
+if snap services aproxy 2>/dev/null | grep -q active; then
+    log "aproxy is listening; applying nftables redirect"
+    nft -f /etc/nftables.conf
+else
+    log "ERROR: aproxy service is not active; leaving egress unchanged (no redirect)"
+fi
diff --git a/charms/garm/tests/unit/test_runner_template.py b/charms/garm/tests/unit/test_runner_template.py
index d05e1e77..59e4ac1e 100644
--- a/charms/garm/tests/unit/test_runner_template.py
+++ b/charms/garm/tests/unit/test_runner_template.py
@@ -163,6 +163,15 @@ def test_render_aproxy_pre_install_script_happy_path():
     assert "counter dnat to \\$default-ipv4:54969" in result
     assert "/etc/nftables.conf" in result
 
+    # The nftables redirect must be gated on aproxy actually listening — a DNAT
+    # to a dead :54969 would black-hole all egress rather than fall back.
+    assert "snap services aproxy" in result
+    assert result.index("snap services aproxy") < result.index("nft -f /etc/nftables.conf")
+
+    # Failures are logged so a broken bootstrap is visible in the console logs.
+    assert "[aproxy-bootstrap]" in result
+    assert "ERROR" in result
+
 
 @pytest.mark.parametrize(
     "proxy, expected",

From c6a09e5fc4c19a12207373ad12ba3d69c5948d88 Mon Sep 17 00:00:00 2001
From: Christopher Bartz <christopher.bartz@canonical.com>
Date: Thu, 9 Jul 2026 06:18:48 +0000
Subject: [PATCH 4/4] refactor(garm): pipe nft ruleset directly and hoist test
 imports

Review follow-ups on the aproxy pre-install script:

- Runner VMs are single-use and never reboot, so persisting the ruleset to
  /etc/nftables.conf and enabling nftables.service is dead weight. Pipe the
  ruleset straight into the kernel with 'nft -f -' instead, keeping the
  aproxy-active guard around it.
- Hoist the repeated function-local test imports (RunnerConfig,
  _effective_extra_specs, build_template_data) to the module top.
---
 charms/garm/src/templates/aproxy.j2            | 16 +++++++---------
 charms/garm/tests/unit/test_runner_template.py |  9 +++++++--
 .../tests/unit/test_scaleset_reconciler.py     | 18 +++---------------
 3 files changed, 17 insertions(+), 26 deletions(-)

diff --git a/charms/garm/src/templates/aproxy.j2 b/charms/garm/src/templates/aproxy.j2
index 4a8ecb92..d62a9711 100644
--- a/charms/garm/src/templates/aproxy.j2
+++ b/charms/garm/src/templates/aproxy.j2
@@ -28,8 +28,13 @@ if ! snap set aproxy proxy={{ proxy_host_port }} listen=:54969; then
     exit 0
 fi
 
-nft delete table ip aproxy 2>/dev/null || true
-cat << EOF > /etc/nftables.conf
+# Only redirect once aproxy is actually listening; otherwise the DNAT would
+# black-hole all :80/:443 egress by sending it to a dead port. Runner VMs are
+# single-use and never reboot, so the ruleset is piped straight into the kernel
+# rather than persisted to a file and reloaded by nftables.service on boot.
+if snap services aproxy 2>/dev/null | grep -q active; then
+    log "aproxy is listening; applying nftables redirect"
+    nft -f - << EOF
 define default-ipv4 = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+')
 table ip aproxy {
       set exclude {
@@ -47,13 +52,6 @@ table ip aproxy {
       }
 }
 EOF
-systemctl enable nftables.service
-
-# Only redirect once aproxy is actually listening; otherwise the DNAT would
-# black-hole all :80/:443 egress by sending it to a dead port.
-if snap services aproxy 2>/dev/null | grep -q active; then
-    log "aproxy is listening; applying nftables redirect"
-    nft -f /etc/nftables.conf
 else
     log "ERROR: aproxy service is not active; leaving egress unchanged (no redirect)"
 fi
diff --git a/charms/garm/tests/unit/test_runner_template.py b/charms/garm/tests/unit/test_runner_template.py
index 59e4ac1e..d48f0451 100644
--- a/charms/garm/tests/unit/test_runner_template.py
+++ b/charms/garm/tests/unit/test_runner_template.py
@@ -161,12 +161,17 @@ def test_render_aproxy_pre_install_script_happy_path():
     assert "10.0.0.0/8" in result
     assert "127.0.0.0/8" in result
     assert "counter dnat to \\$default-ipv4:54969" in result
-    assert "/etc/nftables.conf" in result
+
+    # The ruleset is piped straight into the kernel (single-use VM, no reboot),
+    # not persisted to a file or nftables.service.
+    assert "nft -f -" in result
+    assert "/etc/nftables.conf" not in result
+    assert "systemctl enable nftables" not in result
 
     # The nftables redirect must be gated on aproxy actually listening — a DNAT
     # to a dead :54969 would black-hole all egress rather than fall back.
     assert "snap services aproxy" in result
-    assert result.index("snap services aproxy") < result.index("nft -f /etc/nftables.conf")
+    assert result.index("snap services aproxy") < result.index("nft -f -")
 
     # Failures are logged so a broken bootstrap is visible in the console logs.
     assert "[aproxy-bootstrap]" in result
diff --git a/charms/garm/tests/unit/test_scaleset_reconciler.py b/charms/garm/tests/unit/test_scaleset_reconciler.py
index 993a21ff..9476957d 100644
--- a/charms/garm/tests/unit/test_scaleset_reconciler.py
+++ b/charms/garm/tests/unit/test_scaleset_reconciler.py
@@ -7,8 +7,10 @@
 
 import pytest
 
+from charm_state import RunnerConfig
 from garm_client.models.template import Template
-from scaleset_reconciler import ScalesetReconciler, ScalesetSpec
+from runner_template import build_template_data
+from scaleset_reconciler import ScalesetReconciler, ScalesetSpec, _effective_extra_specs
 
 
 class _FakeProvider:
@@ -138,8 +140,6 @@ def _spec(
     template_id=None,
     runner_config=None,
 ):
-    from charm_state import RunnerConfig
-
     return ScalesetSpec(
         name=name,
         provider_name=provider_name,
@@ -383,7 +383,6 @@ def test_aproxy_pre_install_script_injected_when_proxy_configured():
         operator script and a "00-aproxy" entry (sorts first) whose decoded content
         configures aproxy for the given proxy.
     """
-    from charm_state import RunnerConfig
 
     client = FakeGarmClient(providers=["openstack-demo"], scalesets=[])
     _reconcile(
@@ -412,14 +411,12 @@ def test_no_update_when_extra_specs_already_match_encoded_desired_state():
     act: Reconcile that spec.
     assert: No update is issued.
     """
-    from charm_state import RunnerConfig
 
     proxy = "http://squid.internal:3128"
     spec = _spec(
         pre_install_scripts={"pre_install.sh": "echo operator-script"},
         runner_config=RunnerConfig(runner_http_proxy=proxy),
     )
-    from scaleset_reconciler import _effective_extra_specs
 
     extra_specs = _effective_extra_specs(spec)
     client = FakeGarmClient(
@@ -438,7 +435,6 @@ def test_update_when_observed_scripts_are_legacy_raw_text():
     act: Reconcile a spec whose desired scripts are the same content.
     assert: An update is issued (the raw-text value never matches the encoded desired one).
     """
-    from charm_state import RunnerConfig
 
     spec = _spec(
         pre_install_scripts={"pre_install.sh": "echo operator-script"},
@@ -467,8 +463,6 @@ def test_update_when_disable_updates_missing_but_proxy_configured():
     act: Reconcile that spec.
     assert: An update is issued to set disable_updates.
     """
-    from charm_state import RunnerConfig
-    from scaleset_reconciler import _effective_extra_specs
 
     spec = _spec(runner_config=RunnerConfig(runner_http_proxy="http://squid.internal:3128"))
     extra_specs = _effective_extra_specs(spec)
@@ -490,8 +484,6 @@ def test_update_clears_extra_specs_with_empty_dict_when_proxy_removed():
     assert: The update sends an explicit empty dict — not None, which exclude_none would
         drop, leaving the stale extra_specs (and looping forever).
     """
-    from charm_state import RunnerConfig
-    from scaleset_reconciler import _effective_extra_specs
 
     stale = _effective_extra_specs(
         _spec(runner_config=RunnerConfig(runner_http_proxy="http://squid.internal:3128"))
@@ -571,7 +563,6 @@ def delete_template(self, template_id):
 
 def _spec_with_runner_config(**rc_kwargs):
     """Build a spec with runner_config populated from the given kwargs."""
-    from charm_state import RunnerConfig
 
     return _spec(runner_config=RunnerConfig(**rc_kwargs))
 
@@ -637,7 +628,6 @@ def test_template_created_from_charmed_template_when_scaleset_references_it():
     act: Reconcile the desired spec.
     assert: The custom template is derived from the charmed template rather than the bare system one.
     """
-    from charm_state import RunnerConfig
 
     client = _TemplateTrackingClient(
         providers=["openstack-demo"],
@@ -673,8 +663,6 @@ def test_template_updated_when_runner_config_changes():
     act: Reconcile with a changed runner config.
     assert: The custom template is updated (not recreated); the scaleset template_id is unchanged.
     """
-    from charm_state import RunnerConfig
-    from runner_template import build_template_data
 
     old_config = RunnerConfig(dockerhub_mirror="https://old.example.com")
     custom_template = _FakeTemplate(