From f0f7ede6890eecd4f9037324b0f20afe6db765c3 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Fri, 7 Aug 2026 18:51:11 +0800 Subject: [PATCH] Enforce coarse application network policy Add portable public and local runtime-network controls that default to deny, compile into the locked runtime policy, and grant declared inbound ports only to the persistent workload shape. Classify translation and tunneling ranges conservatively, require both ordinary grants by default, and provide a documented temporary escape hatch for environments that explicitly need the ambiguous class. Install IPv4 and IPv6 nftables policy through the trusted startup helper, retain declared-only inbound admission even when egress is unrestricted, then irreversibly drop setup capabilities and assume the final application identity for workloads, commands, shells, lifecycle actions, and private-environment execution. Preserve declared endpoint traffic, constrain Docker DNS, admit only established responses plus related ICMP network errors, and surface setup failures with their backend diagnostics. Add unit and live Docker coverage for all policy combinations, public exceptions, ambiguous ranges, declared and undeclared inbound ports, root and non-root execution, DNS, endpoint persistence, and transient isolation. Run the live policy matrix in CI, document the security boundary and temporary escape hatch, update the deferred gateway backlog, and revise the security changelog fragment. --- .../+coarse-application-network-policy.yaml | 2 + .github/workflows/integration.yml | 8 + docs/APT_PROVIDER_DETAIL_DESIGN.md | 29 +- docs/BACKLOG.md | 23 +- docs/BLUEPRINT_ENVIRONMENT_MODEL.md | 71 ++- docs/CONTROLLED_SESSION_DESIGN.md | 130 +++-- go.mod | 5 + go.sum | 10 + internal/blueprint/model.go | 23 +- internal/blueprint/resolve.go | 48 +- internal/blueprint/resolve_test.go | 34 ++ internal/blueprint/syntax.go | 9 +- internal/deploy/runtime_policy.go | 58 ++- internal/deploy/runtime_policy_test.go | 26 +- internal/deploy/runtime_verifier.go | 2 +- ...ication_network_policy_integration_test.go | 445 ++++++++++++++++++ .../dockerdeploy/application_sandbox_plan.go | 63 +++ .../application_sandbox_plan_test.go | 145 +++++- .../dockerdeploy/build_publication_test.go | 1 + internal/dockerdeploy/command_execution.go | 11 +- .../dockerdeploy/command_execution_test.go | 12 +- .../current_workload_lifecycle.go | 3 +- .../current_workload_lifecycle_test.go | 2 +- internal/dockerdeploy/execution_plan.go | 2 +- internal/dockerdeploy/execution_render.go | 67 ++- .../dockerdeploy/execution_render_test.go | 2 +- internal/dockerdeploy/full_validation_test.go | 1 + .../installed_service_container.go | 1 + .../prepared_python_graph_reuse_test.go | 2 + .../private_workload_environment_inject.go | 13 +- ...e_workload_environment_integration_test.go | 1 + .../private_workload_environment_test.go | 30 +- .../provider_graph_validation_test.go | 1 + .../provider_install_host_execute.go | 1 + .../runtime_host_preflight_test.go | 3 +- internal/dockerdeploy/runtime_logs.go | 8 +- internal/dockerdeploy/runtime_plan.go | 39 +- internal/dockerdeploy/runtime_plan_test.go | 47 ++ .../dockerdeploy/runtime_policy_compile.go | 43 ++ .../runtime_policy_compile_test.go | 27 ++ .../dockerdeploy/runtime_readiness_test.go | 11 + internal/dockerdeploy/runtime_test.go | 17 + .../testdata/network_policy_helper/main.go | 340 +++++++++++++ .../testdata/resolved_compose.yaml | 9 +- internal/probe/main.go | 15 +- internal/probe/network_firewall_linux.go | 382 +++++++++++++++ internal/probe/network_firewall_linux_test.go | 102 ++++ internal/probe/sandbox_exec.go | 139 ++++++ internal/probe/sandbox_exec_linux.go | 92 ++++ internal/probe/sandbox_exec_other.go | 9 + internal/probe/sandbox_exec_test.go | 91 ++++ internal/probe/startup_verifier.go | 2 + internal/probe/startup_verifier_test.go | 4 + 53 files changed, 2551 insertions(+), 110 deletions(-) create mode 100644 .changes/unreleased/+coarse-application-network-policy.yaml create mode 100644 internal/dockerdeploy/application_network_policy_integration_test.go create mode 100644 internal/dockerdeploy/testdata/network_policy_helper/main.go create mode 100644 internal/probe/network_firewall_linux.go create mode 100644 internal/probe/network_firewall_linux_test.go create mode 100644 internal/probe/sandbox_exec.go create mode 100644 internal/probe/sandbox_exec_linux.go create mode 100644 internal/probe/sandbox_exec_other.go create mode 100644 internal/probe/sandbox_exec_test.go diff --git a/.changes/unreleased/+coarse-application-network-policy.yaml b/.changes/unreleased/+coarse-application-network-policy.yaml new file mode 100644 index 00000000..08b3a689 --- /dev/null +++ b/.changes/unreleased/+coarse-application-network-policy.yaml @@ -0,0 +1,2 @@ +kind: Security +body: 'Deny public and local application networking by default, conservatively require both grants for ambiguous translation and tunneling ranges, select host-derived or public DNS from those grants, and enforce the policy for workloads, commands, shells, and lifecycle actions while preserving declared inbound endpoints.' diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a0813e14..20e162df 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -101,6 +101,14 @@ jobs: go test -timeout 10m ./internal/dockerdeploy -run '^(TestPrivateWorkloadEnvironmentDockerIntegrationMasksFilesAndInjectsValues|TestPrivateRuntimeMasksDockerIntegrationProtectTransientContainer|TestPrivateWorkloadEnvironmentRealDockerIsolation)$' + - name: Run application network-policy integration tests + if: matrix.name == 'Linux amd64' + env: + REPLOY_DOCKER_INTEGRATION: "1" + run: >- + go test -timeout 10m ./internal/dockerdeploy + -run '^TestApplicationNetworkPolicyDockerIntegration$' + - name: Run CLI runtime integration if: ${{ matrix.persistent_install == false }} env: diff --git a/docs/APT_PROVIDER_DETAIL_DESIGN.md b/docs/APT_PROVIDER_DETAIL_DESIGN.md index a542b45a..5f0a8263 100644 --- a/docs/APT_PROVIDER_DETAIL_DESIGN.md +++ b/docs/APT_PROVIDER_DETAIL_DESIGN.md @@ -2127,6 +2127,8 @@ intersections with reserved system, Reploy, or provider paths are invalid. ```go type RuntimePolicyV1 struct { Schema string + StartupVerifier ApplicationStartupVerifierV1 + Network RuntimeNetwork ProtectedPaths []ProtectedPathV1 Plans []RuntimePlanV1 } @@ -2139,6 +2141,7 @@ type ProtectedPathV1 struct { type RuntimePlanV1 struct { ID string + InboundTCP []string Mounts []RuntimeMountV1 Executables []QualifiedOutput } @@ -2150,11 +2153,20 @@ type RuntimeMountV1 struct { } ``` -`Schema` is `runtime-policy-v1`. Protected paths are unique normalized absolute -paths sorted by path. Protected kind is `reploy-root`, +`Schema` is `runtime-policy-v1`. `StartupVerifier` identifies the exact trusted +application setup recipe embedded in the runtime image. `Network` records the +effective independent `public` and `local` access values, each `allow` or +`deny`, plus the `ambiguous` translation/tunneling policy, either `require-both` +or `allow`. IPv4-mapped IPv6 sockets follow their embedded IPv4 class because +Linux emits them as IPv4 packets. Protected paths are unique normalized +absolute paths sorted by path. +Protected kind is `reploy-root`, `provider-root`, `provider-leaf`, or `executable-path`; owner is the stable node or qualified-output identity. Plans use the stable command/workload/probe ID and -are sorted by ID. Mounts are sorted by destination; `SourceKind` records only +are sorted by ID. `InboundTCP` records sorted unique canonical decimal port +strings for that exact container shape: declared ports belong only to the +workload plan, while shell and transient command plans use an empty array. Mounts are sorted by +destination; `SourceKind` records only the resolved kind (`file`, `directory`, or `generated`) and never a host source path. Executables are unique and sorted by qualified identity. @@ -2169,10 +2181,13 @@ build identity input. Each one-shot command and `reploy shell` mounts a fresh 64 MiB tmpfs at `/mnt/reploy-home` for `HOME` and `TMPDIR`. The mount is mode `0700`, owned by the selected runtime UID/GID, and removed with the transient container. Docker -starts the resolved executable directly under that final numeric identity; no -root bootstrap helper is involved. Explicit interruption cleanup force-removes -the transient container. Workload containers use the same bounded tmpfs-home -policy at `/mnt/reploy-home`. +starts only the trusted Reploy helper as container root with the minimal setup +capabilities. The helper installs any required application-network rules, +assumes the selected numeric identity, irreversibly drops its setup authority, +verifies the final kernel state, and executes the resolved application argv. +Explicit interruption cleanup force-removes the transient container. Workload +containers use the same setup contract and bounded tmpfs-home policy at +`/mnt/reploy-home`. The policy digest is `canonical.Sum("runtime-policy", "runtime-policy-v1", policy)`. It is recorded diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index 2d1f6bae..35db6179 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -29,17 +29,6 @@ This file is the day-to-day queue for design and implementation gaps. stopping, and whether the next step needs user review, approval, input, or no user action. -## Now - -- [ ] `P1` Implement the initial coarse application-network policy. Preserve - independent public and local policy intent with both denied by default, - apply it consistently to workloads, commands, shells, and lifecycle - commands, and use only proven isolation and endpoint primitives from the - active runtime backend. Permit exact declared inbound endpoints without - granting general local access. Fail closed when a backend cannot realize - a requested combination, and do not represent this slice as destination-, - domain-, or packet-level filtering. - ## Pre-release - [ ] `P1` Accept APT install transaction records with the optional trailing @@ -203,7 +192,13 @@ This file is the day-to-day queue for design and implementation gaps. cases justify its scope, precedence, user/system ownership, validation, and portability. Initial potential use case: overriding the otherwise fixed host-owned limits for controlled-session endpoint streams and - connection-open rates. + connection-open rates. Also record host-owned DNS resolver configuration + used to provide DNS under the coarse application network grants. The + default local-capable path should use the host's configured resolver so + VPN and split-DNS behavior remains available; the public-only path should + use the built-in Google Public DNS profile (`8.8.8.8`, `8.8.4.4`). Allow + host configuration to override either choice. Resolver selection is + machine policy, not blueprint policy. - [ ] `P2` Design and implement a Reploy userland L3 policy gateway. Keep this separate from the initial public/local kill switches and controlled @@ -217,7 +212,9 @@ This file is the day-to-day queue for design and implementation gaps. initial controlled-session host-loopback endpoint publication so only the lease-owned Host Reploy operation can reach the recorded application; include multi-user-host tests proving unrelated local processes cannot - bypass the session endpoint grant. + bypass the session endpoint grant. Replace the temporary, discouraged + `environment.runtime.network.ambiguous: allow` escape hatch with precise + translated-destination policy and deprecate that coarse override. - [ ] `P2` Evaluate and prioritize the Dingo development-environment gaps. Use `docs/DINGO_GAPS.md` as the needs and evidence record for portable diff --git a/docs/BLUEPRINT_ENVIRONMENT_MODEL.md b/docs/BLUEPRINT_ENVIRONMENT_MODEL.md index 9c9b5bfd..266709b1 100644 --- a/docs/BLUEPRINT_ENVIRONMENT_MODEL.md +++ b/docs/BLUEPRINT_ENVIRONMENT_MODEL.md @@ -92,6 +92,10 @@ environment: allow_concurrent: auto # App-command and shell overlap policy. runtime: user: reploy # Container-local account name; defaults to reploy. + network: + public: deny # Public Internet access; defaults to deny. + local: deny # Local/private network access; defaults to deny. + ambiguous: require-both # Translation/tunnel ranges require both grants. terminal: {} # Terminal/color integration. install: {} # Installation target, identity, and success output. mounts: {} # Runtime filesystem contracts. @@ -106,7 +110,11 @@ one to 32 bytes, beginning with a lowercase ASCII letter or underscore, followed only by lowercase ASCII letters, digits, underscores, or hyphens. `root` is reserved and cannot be selected through this field. If the base image already defines the same account name with a different numeric ID, runtime-layer -construction fails rather than rewriting that unrelated account. Backend-specific +construction fails rather than rewriting that unrelated account. +`runtime.network.public` and +`runtime.network.local` independently accept `allow` or `deny`; both default to +`deny`. `runtime.network.ambiguous` accepts `require-both` or the temporary, +discouraged `allow` escape hatch; it defaults to `require-both`. Backend-specific runtime choices remain under the top-level `docker` node. ## Internal Execution Phases @@ -1382,6 +1390,67 @@ contract is implemented. Reploy rejects these combinations before container creation or output-path preparation. Docker-managed volumes and tmpfs remain available because they do not expose a host filesystem path directly. +Application networking is also a portable environment policy rather than a +Docker mode. `public` controls globally routable IP destinations. `local` +controls private, link-local, multicast, reserved, and infrastructure metadata +destinations. Translation and tunneling ranges that can represent either class +are `ambiguous`: by default, `ambiguous: require-both` permits them only when +both `public` and `local` are allowed. IPv4-mapped IPv6 socket addresses use +the class of their embedded IPv4 destination because Linux emits them as IPv4 +packets. Container-local loopback remains available and cannot address host +loopback through the container network namespace. The backend configures the +container's DNS path from the same grants. With neither network class granted, +DNS is unavailable. With only `local`, it uses the host's configured resolver +so local, VPN, and split-DNS behavior remains available. With only `public`, it +uses the built-in Google Public DNS profile (`8.8.8.8` and `8.8.4.4`). With +both, it uses the host resolver, which normally provides both local and public +resolution. + +For Docker, the local-capable path leaves DNS selection to Docker so it derives +the container's resolver path from the host; the public-only path passes the +selected Google Public DNS profile through Docker's per-container DNS +configuration. Docker writes the resulting container resolver configuration. +The default bridge normally exposes host-derived resolver addresses, while a +custom network exposes Docker's embedded resolver at `127.0.0.11` and forwards +to the selected upstreams. Before installing the packet filter, Reploy's +trusted startup helper reads those engine-authored resolver addresses and +admits TCP and UDP port 53 only to them whenever either network class is +granted. This engine-owned exception does not grant general access to the +resolver's address class. Resolver selection is host policy rather than +blueprint policy; future Reploy host configuration may override the default +local and public resolver choices. The backend does not filter DNS answers. +Connections to every resolved address still pass the ordinary destination +policy, so an answer outside the granted address class remains unreachable. + +`ambiguous: allow` is a temporary, discouraged escape hatch for environments +that intentionally need those translation or tunneling ranges. It grants every +range in that coarse class even when either ordinary network class is denied, +so it weakens the isolation expressed by `public` and `local`. Reploy expects to +deprecate this option after the planned L3 policy gateway can classify the real +destination instead of its translated address. + +Declared workload endpoints remain reachable from their explicit host +publication, which uses loopback by default: the application firewall permits +new inbound TCP connections only to declared endpoint ports and permits the +corresponding established response traffic. When DNS is enabled, resolved +connections remain subject to the destination policy. + +The Linux-container backend realizes this policy with a trusted Reploy startup +helper and IPv4/IPv6 nftables rules inside the container network namespace, +including when both egress classes are allowed so undeclared inbound ports +remain closed. Docker starts only that helper as container root with the minimal +setup capabilities. After installing the rules, the helper changes to the +planned application UID/GID, empties every capability set and the capability +bounding set, locks securebits and `no-new-privileges`, verifies seccomp and +the final kernel state, and executes the exact application argv. Reploy-issued +execs into an application container use the same authority-dropping helper; +they never invoke an application command through raw `docker exec`. + +This is coarse IP-class enforcement. It is not domain, URL, DNS-content, +general outbound port policy, or packet auditing, and it does not defend a +container from an operator who already controls the Docker daemon. A backend +that cannot install and verify the requested policy fails closed. + This is a portable blueprint contract with target-specific realization. The current backend writes Linux account databases. A future native-Windows or other target backend may realize the same local identity through different OS diff --git a/docs/CONTROLLED_SESSION_DESIGN.md b/docs/CONTROLLED_SESSION_DESIGN.md index dd8bcfca..7261e1dd 100644 --- a/docs/CONTROLLED_SESSION_DESIGN.md +++ b/docs/CONTROLLED_SESSION_DESIGN.md @@ -836,17 +836,70 @@ All Reploy application runtime containers default to: - local network disabled. These are independent policy switches. A controlled workflow applies them -separately to the controller and workload environments. Local -denial includes host gateways, Docker peers outside the granted operation, -loopback redirection, private and link-local address ranges, IPv6 local ranges, -and infrastructure metadata endpoints. - -The initial implementation preserves this coarse public/local policy intent -and exact declared endpoint grants, using only backend isolation and endpoint -primitives whose behavior Reploy can verify. A backend that cannot realize a -requested combination fails closed. This slice does not introduce a custom -packet gateway and must not claim destination-, port-, domain-, DNS-, or -packet-level enforcement beyond what the selected primitive actually proves. +separately to the controller and workload environments. Local denial includes +host-loopback redirection, private and link-local address ranges, IPv6 local +ranges, and infrastructure metadata endpoints. Translation and tunneling +ranges that can represent either public or local destinations require both +grants by default. The initial coarse classifier follows the destination +address; topology-resistant peer and gateway confinement belongs to the +deferred L3 gateway design. + +The initial Linux/Docker implementation enforces this coarse public/local +policy with IPv4 and IPv6 nftables rules inside each application container's +network namespace. A trusted Reploy helper begins with only the setup +capabilities needed to install those rules and assume the planned application +identity. It then empties every capability set and the capability bounding +set, locks securebits and `no-new-privileges`, verifies seccomp and the final +kernel state, and executes the exact application argv. Reploy-issued execs use +the same guarded authority-drop path. A raw Docker daemon client remains a +trusted host operator outside this sandbox boundary. + +`public` classifies globally routable IP destinations. `local` classifies +private, link-local, multicast, reserved, and infrastructure metadata +destinations. `ambiguous` covers predefined translation and tunneling ranges; +its default `require-both` admits them only when both ordinary classes are +allowed. IPv4-mapped IPv6 socket addresses use the class of their embedded IPv4 +destination because Linux emits them as IPv4 packets. The temporary +`ambiguous: allow` escape hatch admits the remaining ambiguous class +independently. It is intentionally discouraged because an apparently public +translated address may reach a local destination, and it should be deprecated +once the deferred L3 gateway can enforce the underlying destination policy. +Container-local loopback remains available and cannot address host loopback +through the container network namespace. The backend configures the +container's DNS path from the same two grants: + +| Public | Local | DNS path | +| --- | --- | --- | +| deny | deny | no DNS | +| deny | allow | the host's configured resolver, preserving local, VPN, and split-DNS behavior | +| allow | deny | the built-in Google Public DNS profile (`8.8.8.8`, `8.8.4.4`) | +| allow | allow | the host's configured resolver, which normally provides both local and public resolution | + +For Docker, the local-capable path leaves DNS selection to Docker so it derives +the container's resolver path from the host. The public-only path passes the +selected Google Public DNS profile through Docker's per-container DNS +configuration. Docker writes the resulting container resolver configuration: +the default bridge normally receives host-derived resolver addresses, while a +custom network exposes Docker's embedded resolver at `127.0.0.11` and forwards +to the selected upstreams. Before installing the packet filter, the trusted +startup helper reads those engine-authored resolver addresses and admits TCP +and UDP port 53 only to them whenever either network class is granted. This is +an engine-owned DNS exception, not general access to the resolver's address +class. + +Resolver selection is host policy, not blueprint policy. Future Reploy host +configuration may override the default local and public resolver choices. The +backend does not classify or filter DNS answers. A local resolver may return a +public address and a public resolver may return a private address, but every +subsequent connection still passes the ordinary destination policy. When both +ordinary classes are enabled, egress is unrestricted, but the packet filter +remains in place to admit new inbound connections only on declared endpoint +ports. The helper always applies the same identity and authority-drop invariant. +A backend that cannot install and verify the policy fails closed. + +This slice does not introduce the deferred userland L3 gateway and must not +claim domain-, URL-, DNS-content-, general outbound destination-port-, or +audit-level policy. A controller may receive an explicit session-local grant to a declared workload endpoint. That grant is not treated as general local-network @@ -878,12 +931,12 @@ remains a separate prerequisite; this endpoint forwarding path is not a general router, HTTP policy engine, or domain-aware firewall. General network isolation and auditability are a separate design surface. -Future work may include an HTTP/HTTPS proxy, destination policy, controlled -DNS, and agent-sandbox audit records. HTTPS `CONNECT` can filter and audit a +Future work may include an HTTP/HTTPS proxy, destination and DNS-content +policy, and agent-sandbox audit records. HTTPS `CONNECT` can filter and audit a destination hostname without TLS interception, but cannot inspect encrypted URLs or content. Direct egress must be blocked to prevent proxy bypass. -Redirects, DNS rebinding, CDNs, WebSockets, QUIC, and workload-to-network -policy require explicit treatment. +Redirects, DNS rebinding, CDNs, WebSockets, QUIC, and workload-to-network policy +require explicit treatment. Until that design is implemented, rough public and local kill switches must fail closed and must not be described as domain-level isolation. @@ -1120,27 +1173,34 @@ Implementation status: the canonical application sandbox plan and its identity and kernel baseline are implemented for persistent Compose workloads and transient application commands. Reploy now imports canonical supplementary groups, rejects root-group membership for non-root identities, starts transient -commands directly as the final identity, drops all capabilities, enables +commands through the trusted setup helper, drops all capabilities before the +application starts, enables `no-new-privileges`, explicitly selects Docker's built-in seccomp profile, and prohibits privileged mode, host namespaces, and host devices in the common plan. Live Docker tests inspect both runtime paths. Trusted production startup verification is also implemented: Reploy packages the platform-specific probe in a final runtime layer, creates the locked container-local account there, records that layer outside the provider graph, and uses its fixed -verify-and-exec contract as the outermost process for persistent +sandbox-and-exec contract as the outermost process for persistent workloads, transient commands, shells, and lifecycle commands. The verifier fails closed unless `/proc/self/status` reports seccomp filtering, -`no-new-privileges`, and empty effective, permitted, and bounding capability +`no-new-privileges`, and empty inheritable, effective, permitted, bounding, and +ambient capability sets, then directly executes the exact application argv. Private-environment workloads use one additional fixed Reploy step: after verification, the probe executes the environment injector, which imports the private variables and then -executes the unchanged application argv. Network denial and resource limits -remain separate prerequisite slices. Root host authority is now enforced at -runtime: host sources are classified as input, shared state, or explicit -output; UID 0 is rejected for all three before container creation; and root -output options are rejected before host-path preparation. Docker-managed -volumes and tmpfs remain available to root. Ordinary binds also reject -canonical host root, `/proc`, +executes the unchanged application argv. The same helper now installs the +default-deny application-network policy for persistent and transient containers +while preserving exact declared inbound endpoints. Live Docker coverage +exercises all four public/local combinations over IPv4 and IPv6, strict and +escaped ambiguous-range handling, a globally reachable exception nested inside +a reserved parent range, a root default-denial case, non-root authority removal, +guarded exec, and host-loopback endpoint publication. Resource limits remain a +separate prerequisite slice. Root host authority is now enforced at runtime: +host sources are classified as input, shared state, or explicit output; UID 0 is +rejected for all three before container creation; and root output options are +rejected before host-path preparation. Docker-managed volumes and tmpfs remain +available to root. Ordinary binds also reject canonical host root, `/proc`, `/dev`, and `/sys` sources plus equivalent protected filesystem mounts detected through native filesystem identity. Explicit non-root directory binds intentionally grant access to their remaining unmasked contents, including @@ -1217,16 +1277,16 @@ lease protocol. ### Network Isolation and Audit -After the coarse public/local kill switches, define a separate Reploy userland -L3 policy gateway for finer network control. Its design should cover a -capability-free application network namespace, one-shot route initialization, -an isolated data path whose only peer is the gateway, private gateway control, -root-resistant route invariants, direct-egress prevention, destination and -port grants, DNS and IPv6 policy, metadata protection, auditing, resource -limits, failure behavior, reconciliation, and portable Docker/Podman -integration. The one-way, exact endpoint forwarding used by the initial -controlled session remains intentionally narrower and does not depend on this -later gateway. +After the implemented coarse public/local kill switches, define a separate +Reploy userland L3 policy gateway for finer network control. Its design should +cover a capability-free application network namespace, one-shot route +initialization, an isolated data path whose only peer is the gateway, private +gateway control, root-resistant route invariants, direct-egress prevention, +destination and port grants, DNS and IPv6 policy, metadata protection, +auditing, resource limits, failure behavior, reconciliation, and portable +Docker/Podman integration. The one-way, exact endpoint forwarding used by the +initial controlled session remains intentionally narrower and does not depend +on this later gateway. ### Disposable Writable Workspaces diff --git a/go.mod b/go.mod index ce7e23f5..3c647c8a 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/charmbracelet/x/term v0.2.2 github.com/distribution/reference v0.6.0 github.com/go-git/go-git/v5 v5.19.1 + github.com/google/nftables v0.3.0 github.com/pelletier/go-toml/v2 v2.4.3 github.com/yusufpapurcu/wmi v1.2.4 golang.org/x/sys v0.43.0 @@ -37,6 +38,7 @@ require ( github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect @@ -44,6 +46,8 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect + github.com/mdlayher/socket v0.5.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect @@ -56,6 +60,7 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/crypto v0.50.0 // indirect golang.org/x/net v0.53.0 // indirect + golang.org/x/sync v0.20.0 // indirect golang.org/x/text v0.36.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index f1cb8ae9..50e4105a 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,8 @@ github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8J github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg= +github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -89,6 +91,10 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg= +github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o= +github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI= +github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= @@ -121,6 +127,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= @@ -135,6 +143,8 @@ golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aI golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/blueprint/model.go b/internal/blueprint/model.go index b9a9248d..08f2d3c6 100644 --- a/internal/blueprint/model.go +++ b/internal/blueprint/model.go @@ -41,7 +41,28 @@ type Environment struct { } type EnvironmentRuntime struct { - User string + User string + Network RuntimeNetwork +} + +type NetworkAccess string + +const ( + NetworkAccessDeny NetworkAccess = "deny" + NetworkAccessAllow NetworkAccess = "allow" +) + +type AmbiguousNetworkAccess string + +const ( + AmbiguousNetworkAccessRequireBoth AmbiguousNetworkAccess = "require-both" + AmbiguousNetworkAccessAllow AmbiguousNetworkAccess = "allow" +) + +type RuntimeNetwork struct { + Public NetworkAccess + Local NetworkAccess + Ambiguous AmbiguousNetworkAccess } type EnvironmentPackages struct { diff --git a/internal/blueprint/resolve.go b/internal/blueprint/resolve.go index 2df9526b..8f385d1f 100644 --- a/internal/blueprint/resolve.go +++ b/internal/blueprint/resolve.go @@ -40,6 +40,10 @@ func Resolve(source Syntax) (Document, error) { if err != nil { return Document{}, err } + runtimeNetwork, err := resolveRuntimeNetwork(source.Environment.Runtime.Network) + if err != nil { + return Document{}, err + } extended, err := resolveExtends(source) if err != nil { return Document{}, err @@ -59,7 +63,7 @@ func Resolve(source Syntax) (Document, error) { Applications: map[string]Application{}, Components: map[string]Component{}, AllowConcurrent: allowConcurrent, - Runtime: EnvironmentRuntime{User: runtimeUser}, + Runtime: EnvironmentRuntime{User: runtimeUser, Network: runtimeNetwork}, Terminal: Terminal{ColorEnv: strings.TrimSpace(source.Environment.Terminal.ColorEnv)}, Install: resolveInstallSyntax(source.Environment.Install, variables), Mounts: map[string]EnvironmentMount{}, @@ -87,6 +91,48 @@ func Resolve(source Syntax) (Document, error) { return document, nil } +func resolveRuntimeNetwork(source RuntimeNetworkSyntax) (RuntimeNetwork, error) { + public, err := resolveNetworkAccess("environment.runtime.network.public", source.Public) + if err != nil { + return RuntimeNetwork{}, err + } + local, err := resolveNetworkAccess("environment.runtime.network.local", source.Local) + if err != nil { + return RuntimeNetwork{}, err + } + ambiguous, err := resolveAmbiguousNetworkAccess(source.Ambiguous) + if err != nil { + return RuntimeNetwork{}, err + } + return RuntimeNetwork{Public: public, Local: local, Ambiguous: ambiguous}, nil +} + +func resolveNetworkAccess(field string, value string) (NetworkAccess, error) { + access := NetworkAccess(strings.TrimSpace(value)) + if access == "" { + return NetworkAccessDeny, nil + } + switch access { + case NetworkAccessDeny, NetworkAccessAllow: + return access, nil + default: + return "", fmt.Errorf("%s must be allow or deny", field) + } +} + +func resolveAmbiguousNetworkAccess(value string) (AmbiguousNetworkAccess, error) { + access := AmbiguousNetworkAccess(strings.TrimSpace(value)) + if access == "" { + return AmbiguousNetworkAccessRequireBoth, nil + } + switch access { + case AmbiguousNetworkAccessRequireBoth, AmbiguousNetworkAccessAllow: + return access, nil + default: + return "", fmt.Errorf("environment.runtime.network.ambiguous must be require-both or allow") + } +} + func resolveRuntimeUser(value string) (string, error) { value = strings.TrimSpace(value) if value == "" { diff --git a/internal/blueprint/resolve_test.go b/internal/blueprint/resolve_test.go index 5deffb58..475bd9c5 100644 --- a/internal/blueprint/resolve_test.go +++ b/internal/blueprint/resolve_test.go @@ -26,6 +26,9 @@ func TestResolveProducesTypedEnvironment(t *testing.T) { if document.Environment.Runtime.User != DefaultRuntimeUser { t.Fatalf("runtime user = %q", document.Environment.Runtime.User) } + if document.Environment.Runtime.Network != (RuntimeNetwork{Public: NetworkAccessDeny, Local: NetworkAccessDeny, Ambiguous: AmbiguousNetworkAccessRequireBoth}) { + t.Fatalf("runtime network = %#v", document.Environment.Runtime.Network) + } if got := document.Blueprint.Compatibility.Platforms; !reflect.DeepEqual(got, []Platform{ {OS: "linux", Architecture: "amd64", Canonical: "linux/amd64"}, {OS: "linux", Architecture: "arm64", Canonical: "linux/arm64"}, @@ -51,6 +54,37 @@ func TestResolveProducesTypedEnvironment(t *testing.T) { } } +func TestResolveRuntimeNetwork(t *testing.T) { + for _, test := range []struct { + name string + source RuntimeNetworkSyntax + want RuntimeNetwork + field string + }{ + {name: "default", want: RuntimeNetwork{Public: NetworkAccessDeny, Local: NetworkAccessDeny, Ambiguous: AmbiguousNetworkAccessRequireBoth}}, + {name: "public only", source: RuntimeNetworkSyntax{Public: "allow"}, want: RuntimeNetwork{Public: NetworkAccessAllow, Local: NetworkAccessDeny, Ambiguous: AmbiguousNetworkAccessRequireBoth}}, + {name: "local only", source: RuntimeNetworkSyntax{Local: "allow"}, want: RuntimeNetwork{Public: NetworkAccessDeny, Local: NetworkAccessAllow, Ambiguous: AmbiguousNetworkAccessRequireBoth}}, + {name: "both", source: RuntimeNetworkSyntax{Public: "allow", Local: "allow"}, want: RuntimeNetwork{Public: NetworkAccessAllow, Local: NetworkAccessAllow, Ambiguous: AmbiguousNetworkAccessRequireBoth}}, + {name: "ambiguous escape hatch", source: RuntimeNetworkSyntax{Ambiguous: "allow"}, want: RuntimeNetwork{Public: NetworkAccessDeny, Local: NetworkAccessDeny, Ambiguous: AmbiguousNetworkAccessAllow}}, + {name: "invalid public", source: RuntimeNetworkSyntax{Public: "yes"}, field: "environment.runtime.network.public"}, + {name: "invalid local", source: RuntimeNetworkSyntax{Local: "none"}, field: "environment.runtime.network.local"}, + {name: "invalid ambiguous", source: RuntimeNetworkSyntax{Ambiguous: "deny"}, field: "environment.runtime.network.ambiguous"}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := resolveRuntimeNetwork(test.source) + if test.field == "" { + if err != nil || got != test.want { + t.Fatalf("runtime network = %#v, %v; want %#v", got, err, test.want) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.field) { + t.Fatalf("runtime network error = %v", err) + } + }) + } +} + func TestResolveRuntimeUser(t *testing.T) { for _, test := range []struct { name string diff --git a/internal/blueprint/syntax.go b/internal/blueprint/syntax.go index 0aa1f943..42770f6d 100644 --- a/internal/blueprint/syntax.go +++ b/internal/blueprint/syntax.go @@ -44,7 +44,14 @@ type EnvironmentSyntax struct { } type EnvironmentRuntimeSyntax struct { - User string `yaml:"user"` + User string `yaml:"user"` + Network RuntimeNetworkSyntax `yaml:"network"` +} + +type RuntimeNetworkSyntax struct { + Public string `yaml:"public"` + Local string `yaml:"local"` + Ambiguous string `yaml:"ambiguous"` } type TerminalSyntax struct { diff --git a/internal/deploy/runtime_policy.go b/internal/deploy/runtime_policy.go index f4459562..3480d9d4 100644 --- a/internal/deploy/runtime_policy.go +++ b/internal/deploy/runtime_policy.go @@ -3,6 +3,8 @@ package deploy import ( "fmt" "path" + "slices" + "strconv" "strings" "github.com/omry/reploy/internal/blueprint" @@ -28,6 +30,7 @@ const ( type RuntimePolicyV1 struct { Schema string `json:"schema"` StartupVerifier ApplicationStartupVerifierV1 `json:"startup_verifier"` + Network blueprint.RuntimeNetwork `json:"network"` ProtectedPaths []ProtectedPathV1 `json:"protected_paths"` Plans []RuntimePlanV1 `json:"plans"` } @@ -40,6 +43,7 @@ type ProtectedPathV1 struct { type RuntimePlanV1 struct { ID string `json:"id"` + InboundTCP []string `json:"inbound_tcp"` Mounts []RuntimeMountV1 `json:"mounts"` Executables []providers.QualifiedOutput `json:"executables"` } @@ -67,6 +71,15 @@ func ValidateRuntimePolicyV1(policy RuntimePolicyV1) error { if err := ValidateApplicationStartupVerifierV1(policy.StartupVerifier, false); err != nil { return fmt.Errorf("runtime policy startup verifier: %w", err) } + if err := validateRuntimeNetworkAccessV1("public", policy.Network.Public); err != nil { + return err + } + if err := validateRuntimeNetworkAccessV1("local", policy.Network.Local); err != nil { + return err + } + if err := validateRuntimeAmbiguousNetworkAccessV1(policy.Network.Ambiguous); err != nil { + return err + } for index, protected := range policy.ProtectedPaths { if err := validateRuntimeAbsolutePath("protected path", protected.Path); err != nil { return err @@ -90,9 +103,23 @@ func ValidateRuntimePolicyV1(policy RuntimePolicyV1) error { if index > 0 && policy.Plans[index-1].ID >= plan.ID { return fmt.Errorf("runtime plans must be unique and sorted by ID") } - if plan.Mounts == nil || plan.Executables == nil { + if plan.InboundTCP == nil || plan.Mounts == nil || plan.Executables == nil { return fmt.Errorf("runtime plan %q collections must use arrays", plan.ID) } + previousPort := 0 + for portIndex, rawPort := range plan.InboundTCP { + port, err := strconv.Atoi(rawPort) + if err != nil || strconv.Itoa(port) != rawPort { + return fmt.Errorf("runtime plan %q inbound TCP ports must use canonical decimal strings", plan.ID) + } + if port < 1 || port > 65535 { + return fmt.Errorf("runtime plan %q inbound TCP port must be between 1 and 65535", plan.ID) + } + if portIndex > 0 && previousPort >= port { + return fmt.Errorf("runtime plan %q inbound TCP ports must be unique and sorted", plan.ID) + } + previousPort = port + } for mountIndex, mount := range plan.Mounts { if err := validateRuntimeAbsolutePath("mount destination", mount.Destination); err != nil { return fmt.Errorf("runtime plan %q: %w", plan.ID, err) @@ -124,6 +151,35 @@ func ValidateRuntimePolicyV1(policy RuntimePolicyV1) error { return nil } +func CanonicalRuntimeInboundTCPV1(values []int) []string { + ports := append([]int{}, values...) + slices.Sort(ports) + ports = slices.Compact(ports) + result := make([]string, len(ports)) + for index, port := range ports { + result[index] = strconv.Itoa(port) + } + return result +} + +func validateRuntimeNetworkAccessV1(name string, access blueprint.NetworkAccess) error { + switch access { + case blueprint.NetworkAccessDeny, blueprint.NetworkAccessAllow: + return nil + default: + return fmt.Errorf("runtime policy %s network access must be allow or deny", name) + } +} + +func validateRuntimeAmbiguousNetworkAccessV1(access blueprint.AmbiguousNetworkAccess) error { + switch access { + case blueprint.AmbiguousNetworkAccessRequireBoth, blueprint.AmbiguousNetworkAccessAllow: + return nil + default: + return fmt.Errorf("runtime policy ambiguous network access must be require-both or allow") + } +} + func validateRuntimeReservedDestination(destination string) error { if destination == "/" { return fmt.Errorf("mount destination must not be the container filesystem root") diff --git a/internal/deploy/runtime_policy_test.go b/internal/deploy/runtime_policy_test.go index 88ea5222..7325fce5 100644 --- a/internal/deploy/runtime_policy_test.go +++ b/internal/deploy/runtime_policy_test.go @@ -4,18 +4,20 @@ import ( "strings" "testing" + "github.com/omry/reploy/internal/blueprint" "github.com/omry/reploy/internal/providers" ) func validRuntimePolicy() RuntimePolicyV1 { return RuntimePolicyV1{ Schema: RuntimePolicySchemaV1, StartupVerifier: ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []ProtectedPathV1{ {Path: "/.reploy", Kind: ProtectedPathReployRoot, Owner: "reploy"}, {Path: "/opt/app/bin/tool", Kind: ProtectedPathExecutablePath, Owner: "app.tool"}, }, Plans: []RuntimePlanV1{{ - ID: "serve", + ID: "serve", InboundTCP: []string{"8080"}, Mounts: []RuntimeMountV1{ {Destination: "/mnt/config", SourceKind: RuntimeMountSourceFile, ReadOnly: true}, {Destination: "/workspace/output", SourceKind: RuntimeMountSourceDirectory}, @@ -38,6 +40,23 @@ func TestRuntimePolicyDigestV1IsStable(t *testing.T) { if first != second || first == "" { t.Fatalf("runtime policy digests = %q, %q", first, second) } + policy.Network.Public = blueprint.NetworkAccessAllow + changed, err := RuntimePolicyDigestV1(policy) + if err != nil { + t.Fatal(err) + } + if changed == first { + t.Fatal("runtime policy digest ignored network policy") + } + policy.Network.Public = blueprint.NetworkAccessDeny + policy.Plans[0].InboundTCP = []string{"8081"} + changed, err = RuntimePolicyDigestV1(policy) + if err != nil { + t.Fatal(err) + } + if changed == first { + t.Fatal("runtime policy digest ignored inbound TCP grants") + } } func TestValidateRuntimePolicyV1RejectsNoncanonicalStructure(t *testing.T) { @@ -51,6 +70,11 @@ func TestValidateRuntimePolicyV1RejectsNoncanonicalStructure(t *testing.T) { value.ProtectedPaths[0], value.ProtectedPaths[1] = value.ProtectedPaths[1], value.ProtectedPaths[0] }, want: "protected paths"}, {name: "unsafe owner", mutate: func(value *RuntimePolicyV1) { value.ProtectedPaths[0].Owner = "\n" }, want: "owner"}, + {name: "invalid public network", mutate: func(value *RuntimePolicyV1) { value.Network.Public = "sometimes" }, want: "public network"}, + {name: "invalid local network", mutate: func(value *RuntimePolicyV1) { value.Network.Local = "sometimes" }, want: "local network"}, + {name: "nil inbound TCP", mutate: func(value *RuntimePolicyV1) { value.Plans[0].InboundTCP = nil }, want: "collections"}, + {name: "unsorted inbound TCP", mutate: func(value *RuntimePolicyV1) { value.Plans[0].InboundTCP = []string{"8081", "8080"} }, want: "inbound TCP ports"}, + {name: "invalid inbound TCP", mutate: func(value *RuntimePolicyV1) { value.Plans[0].InboundTCP = []string{"65536"} }, want: "between 1 and 65535"}, {name: "nil mounts", mutate: func(value *RuntimePolicyV1) { value.Plans[0].Mounts = nil }, want: "collections"}, {name: "unsorted mounts", mutate: func(value *RuntimePolicyV1) { value.Plans[0].Mounts[0], value.Plans[0].Mounts[1] = value.Plans[0].Mounts[1], value.Plans[0].Mounts[0] diff --git a/internal/deploy/runtime_verifier.go b/internal/deploy/runtime_verifier.go index 3e25b532..55576d73 100644 --- a/internal/deploy/runtime_verifier.go +++ b/internal/deploy/runtime_verifier.go @@ -11,7 +11,7 @@ import ( const ( ApplicationStartupVerifierSchemaV1 = "application-startup-verifier-v1" - ApplicationStartupVerifierRecipeV1 = "linux-proc-status-verify-exec-v1" + ApplicationStartupVerifierRecipeV1 = "linux-network-policy-sandbox-exec-v1" ApplicationStartupVerifierPathV1 = "/reploy-probe" ApplicationRuntimeLayerSchemaV1 = "application-runtime-layer-v1" ApplicationLocalAccountSchemaV1 = "application-local-account-v1" diff --git a/internal/dockerdeploy/application_network_policy_integration_test.go b/internal/dockerdeploy/application_network_policy_integration_test.go new file mode 100644 index 00000000..03acbb5c --- /dev/null +++ b/internal/dockerdeploy/application_network_policy_integration_test.go @@ -0,0 +1,445 @@ +package dockerdeploy + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/omry/reploy/internal/blueprint" +) + +func TestApplicationNetworkPolicyDockerIntegration(t *testing.T) { + if os.Getenv("REPLOY_DOCKER_INTEGRATION") != "1" { + t.Skip("set REPLOY_DOCKER_INTEGRATION=1 to run Docker integration evidence") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + image, _ := buildApplicationStartupVerifierIntegrationImage(t, ctx) + helper := buildNetworkPolicyIntegrationHelper(t, ctx) + localNetwork, publicNetwork, publicExceptionNetwork, ambiguousNetwork, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, dnsName := createNetworkPolicyIntegrationPeers(t, ctx, image, helper) + directResolver := createNetworkPolicyDirectResolver(t, ctx, image, helper) + + for _, test := range []struct { + name string + public blueprint.NetworkAccess + local blueprint.NetworkAccess + want bool + }{ + {name: "deny both", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny}, + {name: "public only", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessDeny, want: true}, + {name: "local only", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessAllow, want: true}, + {name: "allow both", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessAllow, want: true}, + } { + for _, transport := range []string{"udp", "tcp"} { + t.Run("engine-selected DNS "+transport+" "+test.name, func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, test.public, test.local, blueprint.AmbiguousNetworkAccessRequireBoth) + command := ResolvedEnvironmentCommand{Argv: []string{"/network-test", "dns", "reploy.test", strconv.FormatBool(test.want), transport}} + execution, err := PlanTransientContainerExecutionV1(plan, command, nil, "run-0000000000000001", false, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = exec.CommandContext(context.Background(), "docker", execution.Cleanup.Args...).Run() }) + runDockerIntegration(t, ctx, dockerCreateWithDNSV1(t, execution.Create.Args, net.JoinHostPort(directResolver, "53"))...) + output := runDockerIntegration(t, ctx, execution.Start.Args...) + if !strings.Contains(output, "DNS_PASS") { + t.Fatalf("direct local DNS output = %q", output) + } + }) + } + } + + t.Run("public-only default DNS profile", func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, blueprint.NetworkAccessAllow, blueprint.NetworkAccessDeny, blueprint.AmbiguousNetworkAccessRequireBoth) + command := ResolvedEnvironmentCommand{Argv: []string{"/network-test", "dns", "example.com", "true", "udp"}} + execution, err := PlanTransientContainerExecutionV1(plan, command, nil, "run-0000000000000001", false, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = exec.CommandContext(context.Background(), "docker", execution.Cleanup.Args...).Run() }) + runDockerIntegration(t, ctx, execution.Create.Args...) + output := runDockerIntegration(t, ctx, execution.Start.Args...) + if !strings.Contains(output, "DNS_PASS") { + t.Fatalf("public-only default DNS output = %q", output) + } + }) + + for _, test := range []struct { + name string + public blueprint.NetworkAccess + local blueprint.NetworkAccess + ambiguous blueprint.AmbiguousNetworkAccess + wantPublic bool + wantLocal bool + wantAmbiguous bool + }{ + {name: "deny-deny", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, + {name: "allow-deny", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, wantPublic: true}, + {name: "deny-allow", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessAllow, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, wantLocal: true}, + {name: "ambiguous-escape-hatch", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessAllow, wantAmbiguous: true}, + {name: "allow-allow", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessAllow, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, wantPublic: true, wantLocal: true, wantAmbiguous: true}, + } { + t.Run("transient "+test.name, func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, test.public, test.local, test.ambiguous) + command := ResolvedEnvironmentCommand{Argv: networkPolicyIntegrationWorkloadArgv( + plan.Sandbox.RuntimeUser.UID, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, + test.wantLocal, test.wantPublic, test.wantAmbiguous, test.wantPublic, false, dnsName, + )} + execution, err := PlanTransientContainerExecutionV1(plan, command, nil, "run-0000000000000001", false, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = exec.CommandContext(context.Background(), "docker", execution.Cleanup.Args...).Run() }) + runDockerIntegration(t, ctx, dockerCreateWithDNSV1(t, execution.Create.Args, localAddresses[0])...) + runDockerIntegration(t, ctx, "network", "connect", localNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", publicNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", publicExceptionNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", ambiguousNetwork, execution.Container) + output := runDockerIntegration(t, ctx, execution.Start.Args...) + if !strings.Contains(output, "NETWORK_POLICY_PASS") { + t.Fatalf("transient network-policy output = %q", output) + } + }) + } + + t.Run("transient root deny-deny", func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, blueprint.NetworkAccessDeny, blueprint.NetworkAccessDeny, blueprint.AmbiguousNetworkAccessRequireBoth) + plan.Sandbox = newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 0, GID: 0, DockerUser: "0:0"}, + blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, + ) + command := ResolvedEnvironmentCommand{Argv: networkPolicyIntegrationWorkloadArgv( + 0, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, false, false, false, false, false, dnsName, + )} + execution, err := PlanTransientContainerExecutionV1(plan, command, nil, "run-0000000000000001", false, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = exec.CommandContext(context.Background(), "docker", execution.Cleanup.Args...).Run() }) + runDockerIntegration(t, ctx, dockerCreateWithDNSV1(t, execution.Create.Args, localAddresses[0])...) + runDockerIntegration(t, ctx, "network", "connect", localNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", publicNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", publicExceptionNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", ambiguousNetwork, execution.Container) + output := runDockerIntegration(t, ctx, execution.Start.Args...) + if !strings.Contains(output, "NETWORK_POLICY_PASS") { + t.Fatalf("root network-policy output = %q", output) + } + }) + + for _, test := range []struct { + name string + public blueprint.NetworkAccess + local blueprint.NetworkAccess + wantPublic bool + wantLocal bool + }{ + {name: "deny-deny", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny}, + {name: "allow-deny", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessDeny, wantPublic: true}, + {name: "deny-allow", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessAllow, wantLocal: true}, + {name: "allow-allow", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessAllow, wantPublic: true, wantLocal: true}, + } { + t.Run("persistent "+test.name, func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, test.public, test.local, blueprint.AmbiguousNetworkAccessRequireBoth) + plan.Workload = &WorkloadExecutionPlan{ + Argv: networkPolicyIntegrationWorkloadArgv( + plan.Sandbox.RuntimeUser.UID, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, + test.wantLocal, test.wantPublic, test.wantLocal && test.wantPublic, test.wantPublic, false, dnsName, + ), + Endpoints: map[string]EndpointExecutionPlan{}, + } + rendered, err := RenderDockerInputs(plan, "network-policy") + if err != nil { + t.Fatal(err) + } + composePath := filepath.Join(t.TempDir(), "compose.yaml") + if err := os.WriteFile(composePath, rendered.Compose, 0o600); err != nil { + t.Fatal(err) + } + composeArgs := []string{"compose", "--project-name", plan.NetworkName, "-f", composePath} + t.Cleanup(func() { + args := append(append([]string(nil), composeArgs...), "down", "--remove-orphans") + _ = exec.CommandContext(context.Background(), "docker", args...).Run() + }) + runDockerIntegration(t, ctx, append(composeArgs, "create", "--pull", "never")...) + runDockerIntegration(t, ctx, "network", "connect", localNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicExceptionNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", ambiguousNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, append(composeArgs, "start")...) + runDockerIntegration(t, ctx, "wait", plan.ContainerName) + logs := runDockerIntegration(t, ctx, append(composeArgs, "logs")...) + if !strings.Contains(logs, "NETWORK_POLICY_PASS") { + t.Fatalf("persistent network-policy logs = %q", logs) + } + }) + } + + t.Run("persistent endpoint survives default denial", func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, blueprint.NetworkAccessDeny, blueprint.NetworkAccessDeny, blueprint.AmbiguousNetworkAccessRequireBoth) + plan.Workload = &WorkloadExecutionPlan{ + Argv: networkPolicyIntegrationWorkloadArgv(plan.Sandbox.RuntimeUser.UID, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, false, false, false, false, true, dnsName), + Endpoints: map[string]EndpointExecutionPlan{ + "http": {Scheme: "http", PublishAddress: "127.0.0.1", PublishedPort: reserveNetworkPolicyIntegrationPort(t), ContainerPort: 8080}, + }, + } + rendered, err := RenderDockerInputs(plan, "network-policy") + if err != nil { + t.Fatal(err) + } + composePath := filepath.Join(t.TempDir(), "compose.yaml") + if err := os.WriteFile(composePath, rendered.Compose, 0o600); err != nil { + t.Fatal(err) + } + composeArgs := []string{"compose", "--project-name", plan.NetworkName, "-f", composePath} + t.Cleanup(func() { + args := append(append([]string(nil), composeArgs...), "down", "--remove-orphans") + _ = exec.CommandContext(context.Background(), "docker", args...).Run() + }) + runDockerIntegration(t, ctx, append(composeArgs, "create", "--pull", "never")...) + runDockerIntegration(t, ctx, "network", "connect", localNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicExceptionNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", ambiguousNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, append(composeArgs, "start")...) + url := fmt.Sprintf("http://127.0.0.1:%d/", plan.Workload.Endpoints["http"].PublishedPort) + waitNetworkPolicyIntegrationEndpoint(t, url, func() string { return runDockerIntegration(t, ctx, append(composeArgs, "logs")...) }) + runDockerIntegration(t, ctx, "restart", plan.ContainerName) + waitNetworkPolicyIntegrationEndpoint(t, url, func() string { return runDockerIntegration(t, ctx, append(composeArgs, "logs")...) }) + logs := runDockerIntegration(t, ctx, append(composeArgs, "logs")...) + if !strings.Contains(logs, "NETWORK_POLICY_PASS") { + t.Fatalf("persistent network-policy logs = %q", logs) + } + }) + + t.Run("persistent unrestricted egress still limits inbound", func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, blueprint.NetworkAccessAllow, blueprint.NetworkAccessAllow, blueprint.AmbiguousNetworkAccessRequireBoth) + plan.Workload = &WorkloadExecutionPlan{ + Argv: networkPolicyIntegrationWorkloadArgv(plan.Sandbox.RuntimeUser.UID, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, true, true, true, true, true, dnsName), + Endpoints: map[string]EndpointExecutionPlan{ + "http": {Scheme: "http", PublishAddress: "127.0.0.1", PublishedPort: reserveNetworkPolicyIntegrationPort(t), ContainerPort: 8080}, + }, + } + rendered, err := RenderDockerInputs(plan, "network-policy") + if err != nil { + t.Fatal(err) + } + composePath := filepath.Join(t.TempDir(), "compose.yaml") + if err := os.WriteFile(composePath, rendered.Compose, 0o600); err != nil { + t.Fatal(err) + } + composeArgs := []string{"compose", "--project-name", plan.NetworkName, "-f", composePath} + t.Cleanup(func() { + args := append(append([]string(nil), composeArgs...), "down", "--remove-orphans") + _ = exec.CommandContext(context.Background(), "docker", args...).Run() + }) + runDockerIntegration(t, ctx, append(composeArgs, "create", "--pull", "never")...) + runDockerIntegration(t, ctx, "network", "connect", localNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicExceptionNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", ambiguousNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, append(composeArgs, "start")...) + url := fmt.Sprintf("http://127.0.0.1:%d/", plan.Workload.Endpoints["http"].PublishedPort) + waitNetworkPolicyIntegrationEndpoint(t, url, func() string { return runDockerIntegration(t, ctx, append(composeArgs, "logs")...) }) + runDockerIntegration(t, ctx, "network", "connect", plan.NetworkName, dnsName) + t.Cleanup(func() { + _ = exec.CommandContext(context.Background(), "docker", "network", "disconnect", "--force", plan.NetworkName, dnsName).Run() + }) + runDockerIntegration(t, ctx, "exec", dnsName, "/network-test", "dial", plan.ContainerName+":8080", "true") + runDockerIntegration(t, ctx, "exec", dnsName, "/network-test", "dial", plan.ContainerName+":8081", "false") + }) +} + +func dockerCreateWithDNSV1(t *testing.T, args []string, resolver string) []string { + t.Helper() + host, _, err := net.SplitHostPort(resolver) + if err != nil { + t.Fatalf("split integration resolver %q: %v", resolver, err) + } + entrypoint := -1 + for index, argument := range args { + if argument == "--entrypoint" { + entrypoint = index + break + } + } + if entrypoint < 0 { + t.Fatal("transient Docker create command has no entrypoint boundary") + } + result := make([]string, 0, len(args)+2) + for index := 0; index < entrypoint; index++ { + if args[index] == "--dns" { + if index+1 >= entrypoint { + t.Fatal("transient Docker create command has an incomplete DNS option") + } + index++ + continue + } + result = append(result, args[index]) + } + result = append(result, "--dns", host) + return append(result, args[entrypoint:]...) +} + +func waitNetworkPolicyIntegrationEndpoint(t *testing.T, url string, logs func() string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + var requestErr error + for { + response, err := http.Get(url) + requestErr = err + if err == nil { + _ = response.Body.Close() + if response.StatusCode == http.StatusOK { + return + } + } + if time.Now().After(deadline) { + t.Fatalf("persistent endpoint did not become ready: %v\n%s", requestErr, logs()) + } + time.Sleep(100 * time.Millisecond) + } +} + +func networkPolicyIntegrationPlan(t *testing.T, image string, helper string, public blueprint.NetworkAccess, local blueprint.NetworkAccess, ambiguous blueprint.AmbiguousNetworkAccess) DockerExecutionPlan { + t.Helper() + return DockerExecutionPlan{ + EnvironmentID: "network-policy", DeploymentDir: t.TempDir(), Phase: blueprint.PhaseStaged, + Image: image, ContainerName: uniqueDockerIntegrationName("reploy-network-policy"), + NetworkName: uniqueDockerIntegrationName("reploy-network-policy-network"), + Sandbox: newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 12345, GID: 23456, DockerUser: "12345:23456"}, + blueprint.RuntimeNetwork{Public: public, Local: local, Ambiguous: ambiguous}, + ), + Mounts: []MountExecutionPlan{{ + Name: "network-test", Mode: blueprint.MountBind, Source: helper, + SourceKind: "file", Target: "/network-test", ReadOnly: true, + }}, + } +} + +func networkPolicyIntegrationWorkloadArgv(uid int, local []string, public []string, ambiguous string, publicException string, wantLocal bool, wantPublic bool, wantAmbiguous bool, wantPublicException bool, serve bool, dnsName string) []string { + mode := "exit" + if serve { + mode = "serve" + } + return []string{ + "/network-test", "workload", strconv.Itoa(uid), + local[0], local[1], public[0], public[1], ambiguous, publicException, + strconv.FormatBool(wantLocal), strconv.FormatBool(wantPublic), strconv.FormatBool(wantAmbiguous), strconv.FormatBool(wantPublicException), mode, + dnsName, strconv.FormatBool(wantPublic || wantLocal), + } +} + +func buildNetworkPolicyIntegrationHelper(t *testing.T, ctx context.Context) string { + t.Helper() + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skipf("network-policy Docker integration does not build a helper for %s", runtime.GOARCH) + } + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + output := filepath.Join(dockerIntegrationSharedTempDir(t), "network-policy-helper") + command := exec.CommandContext(ctx, "go", "build", "-buildvcs=false", "-o", output, "./internal/dockerdeploy/testdata/network_policy_helper") + command.Dir = root + command.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux", "GOARCH="+runtime.GOARCH, "GOCACHE="+filepath.Join(t.TempDir(), "go-cache")) + if content, err := command.CombinedOutput(); err != nil { + t.Fatalf("build network-policy integration helper: %v\n%s", err, content) + } + return output +} + +func createNetworkPolicyIntegrationPeers(t *testing.T, ctx context.Context, image string, helper string) (string, string, string, string, []string, []string, string, string, string) { + t.Helper() + seed := int(time.Now().UnixNano()%180) + 40 + localNetwork := uniqueDockerIntegrationName("reploy-network-local") + publicNetwork := uniqueDockerIntegrationName("reploy-network-public") + publicExceptionNetwork := uniqueDockerIntegrationName("reploy-network-public-exception") + ambiguousNetwork := uniqueDockerIntegrationName("reploy-network-ambiguous") + localV4 := fmt.Sprintf("172.30.%d.3", seed) + localV6 := fmt.Sprintf("fd30:%x::3", seed) + publicV4 := fmt.Sprintf("11.%d.0.3", seed) + publicV6 := fmt.Sprintf("2001:4860:%x::3", seed) + publicExceptionV6 := fmt.Sprintf("2001:20:%x::3", seed) + ambiguousV4 := fmt.Sprintf("172.29.%d.3", seed) + ambiguousV6 := fmt.Sprintf("64:ff9b:1:%x::3", seed) + runDockerIntegration(t, ctx, "network", "create", "--ipv6", "--subnet", fmt.Sprintf("172.30.%d.0/24", seed), "--subnet", fmt.Sprintf("fd30:%x::/64", seed), localNetwork) + runDockerIntegration(t, ctx, "network", "create", "--ipv6", "--subnet", fmt.Sprintf("11.%d.0.0/24", seed), "--subnet", fmt.Sprintf("2001:4860:%x::/64", seed), publicNetwork) + runDockerIntegration(t, ctx, "network", "create", "--ipv6", "--subnet", fmt.Sprintf("172.28.%d.0/24", seed), "--subnet", fmt.Sprintf("2001:20:%x::/64", seed), publicExceptionNetwork) + runDockerIntegration(t, ctx, "network", "create", "--ipv6", "--subnet", fmt.Sprintf("172.29.%d.0/24", seed), "--subnet", fmt.Sprintf("64:ff9b:1:%x::/64", seed), ambiguousNetwork) + t.Cleanup(func() { + _ = exec.CommandContext(context.Background(), "docker", "network", "rm", localNetwork, publicNetwork, publicExceptionNetwork, ambiguousNetwork).Run() + }) + publicPeerName := "" + for _, peer := range []struct { + name string + network string + ipv4 string + ipv6 string + }{ + {name: uniqueDockerIntegrationName("reploy-network-local-peer"), network: localNetwork, ipv4: localV4, ipv6: localV6}, + {name: uniqueDockerIntegrationName("reploy-network-public-peer"), network: publicNetwork, ipv4: publicV4, ipv6: publicV6}, + {name: uniqueDockerIntegrationName("reploy-network-public-exception-peer"), network: publicExceptionNetwork, ipv6: publicExceptionV6}, + {name: uniqueDockerIntegrationName("reploy-network-ambiguous-peer"), network: ambiguousNetwork, ipv4: ambiguousV4, ipv6: ambiguousV6}, + } { + if peer.network == publicNetwork && publicPeerName == "" { + publicPeerName = peer.name + } + args := []string{ + "run", "--detach", "--pull", "never", "--name", peer.name, + "--network", peer.network, + } + if peer.ipv4 != "" { + args = append(args, "--ip", peer.ipv4) + } + args = append(args, + "--ip6", peer.ipv6, + "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges=true", + "--mount", "type=bind,source="+helper+",target=/network-test,readonly", + "--entrypoint", "/network-test", image, "peer", + ) + runDockerIntegration(t, ctx, args...) + t.Cleanup(func() { + _ = exec.CommandContext(context.Background(), "docker", "container", "rm", "--force", peer.name).Run() + }) + } + time.Sleep(250 * time.Millisecond) + return localNetwork, publicNetwork, publicExceptionNetwork, ambiguousNetwork, + []string{localV4 + ":9090", "[" + localV6 + "]:9090"}, + []string{publicV4 + ":9090", "[" + publicV6 + "]:9090"}, "[" + ambiguousV6 + "]:9090", "[" + publicExceptionV6 + "]:9090", publicPeerName +} + +func createNetworkPolicyDirectResolver(t *testing.T, ctx context.Context, image string, helper string) string { + t.Helper() + name := uniqueDockerIntegrationName("reploy-network-direct-dns") + runDockerIntegration(t, ctx, + "run", "--detach", "--name", name, "--network", "bridge", + "--mount", "type=bind,source="+helper+",target=/network-test,readonly", + "--entrypoint", "/network-test", image, "peer", + ) + t.Cleanup(func() { _ = exec.CommandContext(context.Background(), "docker", "rm", "--force", name).Run() }) + address := strings.TrimSpace(runDockerIntegration(t, ctx, "inspect", "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", name)) + if net.ParseIP(address) == nil { + t.Fatalf("direct DNS peer address = %q", address) + } + return address +} + +func reserveNetworkPolicyIntegrationPort(t *testing.T) int { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + return listener.Addr().(*net.TCPAddr).Port +} diff --git a/internal/dockerdeploy/application_sandbox_plan.go b/internal/dockerdeploy/application_sandbox_plan.go index c9ec2f15..fe29f013 100644 --- a/internal/dockerdeploy/application_sandbox_plan.go +++ b/internal/dockerdeploy/application_sandbox_plan.go @@ -6,10 +6,13 @@ import ( "slices" "strconv" + "github.com/omry/reploy/internal/blueprint" "github.com/omry/reploy/internal/deploy" ) const applicationSeccompProfileBuiltinV1 = "builtin" +const applicationGooglePublicDNSPrimaryV1 = "8.8.8.8" +const applicationGooglePublicDNSSecondaryV1 = "8.8.4.4" type ApplicationKernelPolicyV1 struct { DropAllCapabilities bool @@ -26,18 +29,31 @@ type ApplicationKernelPolicyV1 struct { // renderer-specific flags. type ApplicationSandboxPlanV1 struct { RuntimeUser RuntimeUserPlan + Network ApplicationNetworkPolicyV1 ReadOnlyRoot bool TemporaryHome string StartupVerifier deploy.ApplicationStartupVerifierV1 Kernel ApplicationKernelPolicyV1 } +type ApplicationNetworkPolicyV1 struct { + Public blueprint.NetworkAccess + Local blueprint.NetworkAccess + Ambiguous blueprint.AmbiguousNetworkAccess +} + func newApplicationSandboxPlanV1(runtimeUser RuntimeUserPlan) ApplicationSandboxPlanV1 { + return newApplicationSandboxPlanWithNetworkV1(runtimeUser, blueprint.RuntimeNetwork{}) +} + +func newApplicationSandboxPlanWithNetworkV1(runtimeUser RuntimeUserPlan, network blueprint.RuntimeNetwork) ApplicationSandboxPlanV1 { if runtimeUser.LocalUser == "" { runtimeUser.LocalUser = runtimeLocalUserNameV1("", runtimeUser.UID) } + network = normalizeRuntimeNetworkV1(network) return ApplicationSandboxPlanV1{ RuntimeUser: runtimeUser, + Network: ApplicationNetworkPolicyV1{Public: network.Public, Local: network.Local, Ambiguous: network.Ambiguous}, ReadOnlyRoot: true, TemporaryHome: environmentTemporaryHome, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), @@ -51,10 +67,39 @@ func newApplicationSandboxPlanV1(runtimeUser RuntimeUserPlan) ApplicationSandbox } } +func normalizeRuntimeNetworkV1(network blueprint.RuntimeNetwork) blueprint.RuntimeNetwork { + if network.Public == "" { + network.Public = blueprint.NetworkAccessDeny + } + if network.Local == "" { + network.Local = blueprint.NetworkAccessDeny + } + if network.Ambiguous == "" { + network.Ambiguous = blueprint.AmbiguousNetworkAccessRequireBoth + } + return network +} + +func applicationDockerDNSResolversV1(network ApplicationNetworkPolicyV1) []string { + if network.Public == blueprint.NetworkAccessAllow && network.Local == blueprint.NetworkAccessDeny { + return []string{applicationGooglePublicDNSPrimaryV1, applicationGooglePublicDNSSecondaryV1} + } + return nil +} + func ValidateApplicationSandboxPlanV1(plan ApplicationSandboxPlanV1) error { if plan.RuntimeUser.UID < 0 || plan.RuntimeUser.GID < 0 { return fmt.Errorf("application sandbox requires a non-negative numeric UID and GID") } + if err := validateApplicationNetworkAccessV1("public", plan.Network.Public); err != nil { + return err + } + if err := validateApplicationNetworkAccessV1("local", plan.Network.Local); err != nil { + return err + } + if err := validateApplicationAmbiguousNetworkAccessV1(plan.Network.Ambiguous); err != nil { + return err + } wantUser := strconv.Itoa(plan.RuntimeUser.UID) + ":" + strconv.Itoa(plan.RuntimeUser.GID) if plan.RuntimeUser.DockerUser != wantUser { return fmt.Errorf("application sandbox Docker user must match its numeric UID and GID") @@ -110,6 +155,24 @@ func ValidateApplicationSandboxPlanV1(plan ApplicationSandboxPlanV1) error { return nil } +func validateApplicationNetworkAccessV1(name string, access blueprint.NetworkAccess) error { + switch access { + case blueprint.NetworkAccessDeny, blueprint.NetworkAccessAllow: + return nil + default: + return fmt.Errorf("application sandbox %s network access must be allow or deny", name) + } +} + +func validateApplicationAmbiguousNetworkAccessV1(access blueprint.AmbiguousNetworkAccess) error { + switch access { + case blueprint.AmbiguousNetworkAccessRequireBoth, blueprint.AmbiguousNetworkAccessAllow: + return nil + default: + return fmt.Errorf("application sandbox ambiguous network access must be require-both or allow") + } +} + func applicationLocalAccountV1(plan ApplicationSandboxPlanV1) (deploy.ApplicationLocalAccountV1, error) { if err := ValidateApplicationSandboxPlanV1(plan); err != nil { return deploy.ApplicationLocalAccountV1{}, err diff --git a/internal/dockerdeploy/application_sandbox_plan_test.go b/internal/dockerdeploy/application_sandbox_plan_test.go index 88a448d1..b811e99f 100644 --- a/internal/dockerdeploy/application_sandbox_plan_test.go +++ b/internal/dockerdeploy/application_sandbox_plan_test.go @@ -20,6 +20,7 @@ func TestApplicationRenderersConsumeCanonicalSandboxPlan(t *testing.T) { Sandbox: newApplicationSandboxPlanV1(RuntimeUserPlan{ UID: 501, GID: 20, SupplementaryGIDs: []int{33, 44}, DockerUser: "501:20", }), + Workload: &WorkloadExecutionPlan{Argv: []string{"/bin/true"}, Endpoints: map[string]EndpointExecutionPlan{}}, } persistent, err := RenderDockerInputs(plan, "demo") @@ -31,11 +32,13 @@ func TestApplicationRenderersConsumeCanonicalSandboxPlan(t *testing.T) { t.Fatal(err) } service := compose.Services["environment"] - if service.User != plan.Sandbox.RuntimeUser.DockerUser || !service.ReadOnly { + if service.User != "0:0" || !service.ReadOnly { t.Fatalf("persistent sandbox identity/read-only = user %q, read-only %t", service.User, service.ReadOnly) } - if !slices.Equal(service.GroupAdd, []string{"33", "44"}) || !slices.Equal(service.CapDrop, []string{"ALL"}) || !slices.Equal(service.SecurityOpt, []string{"no-new-privileges:true", "seccomp=builtin"}) { - t.Fatalf("persistent kernel sandbox = groups %#v, caps %#v, security %#v", service.GroupAdd, service.CapDrop, service.SecurityOpt) + if len(service.GroupAdd) != 0 || !slices.Equal(service.CapDrop, []string{"ALL"}) || + !slices.Equal(service.CapAdd, []string{"NET_ADMIN", "SETGID", "SETPCAP", "SETUID"}) || + !slices.Equal(service.SecurityOpt, []string{"no-new-privileges:true", "seccomp=builtin"}) { + t.Fatalf("persistent kernel sandbox = groups %#v, drop %#v, add %#v, security %#v", service.GroupAdd, service.CapDrop, service.CapAdd, service.SecurityOpt) } if service.Environment["HOME"] != plan.Sandbox.TemporaryHome || service.Environment["TMPDIR"] != plan.Sandbox.TemporaryHome { t.Fatalf("persistent sandbox environment = %#v", service.Environment) @@ -43,6 +46,20 @@ func TestApplicationRenderersConsumeCanonicalSandboxPlan(t *testing.T) { if !containsString(service.Tmpfs, temporaryHomeMountForPlan(plan)) { t.Fatalf("persistent sandbox temporary home = %#v", service.Tmpfs) } + baseOnly := plan + baseOnly.Workload = nil + baseRendered, err := RenderDockerInputs(baseOnly, "demo") + if err != nil { + t.Fatal(err) + } + var baseCompose composePlanDocument + if err := yaml.Unmarshal(baseRendered.Compose, &baseCompose); err != nil { + t.Fatal(err) + } + baseService := baseCompose.Services["environment"] + if baseService.User != plan.Sandbox.RuntimeUser.DockerUser || len(baseService.CapAdd) != 0 || !slices.Equal(baseService.GroupAdd, []string{"33", "44"}) { + t.Fatalf("base-only dormant service authority = user %q, groups %#v, caps %#v", baseService.User, baseService.GroupAdd, baseService.CapAdd) + } transient, err := TransientCommandSpec( plan, @@ -63,10 +80,10 @@ func TestApplicationRenderersConsumeCanonicalSandboxPlan(t *testing.T) { }) { t.Fatalf("transient sandbox environment = %#v", transient.Args) } - if !containsInOrder(transient.Args, []string{"--user", "501:20", "--cap-drop", "ALL"}) || - !containsInOrder(transient.Args, []string{"--group-add", "33", "--group-add", "44"}) || + if !containsInOrder(transient.Args, []string{"--user", "0:0", "--cap-drop", "ALL"}) || + !containsInOrder(transient.Args, []string{"--cap-add", "NET_ADMIN", "--cap-add", "SETGID", "--cap-add", "SETPCAP", "--cap-add", "SETUID"}) || !containsInOrder(transient.Args, []string{"--security-opt", "no-new-privileges=true", "--security-opt", "seccomp=builtin"}) || - !containsInOrder(transient.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "verify-exec", "--", "/bin/true"}) { + !containsInOrder(transient.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "sandbox-exec", "--uid", "501", "--gid", "20", "--groups", "33,44", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/bin/true"}) { t.Fatalf("transient sandbox runtime identity = %#v", transient.Args) } @@ -118,6 +135,122 @@ func TestApplicationSandboxPlanRejectsIdentityAndKernelEscapes(t *testing.T) { } } +func TestApplicationNetworkPolicyControlsOnlySetupCapability(t *testing.T) { + for _, test := range []struct { + name string + public blueprint.NetworkAccess + local blueprint.NetworkAccess + ambiguous blueprint.AmbiguousNetworkAccess + netAdmin bool + }{ + {name: "deny both", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, netAdmin: true}, + {name: "public only", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, netAdmin: true}, + {name: "local only", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessAllow, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, netAdmin: true}, + {name: "ambiguous escape hatch", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessAllow, netAdmin: true}, + {name: "allow both", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessAllow, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, netAdmin: true}, + } { + t.Run(test.name, func(t *testing.T) { + plan := newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 501, GID: 20, DockerUser: "501:20"}, + blueprint.RuntimeNetwork{Public: test.public, Local: test.local, Ambiguous: test.ambiguous}, + ) + capabilities := applicationSetupCapabilitiesV1(plan) + if containsString(capabilities, "NET_ADMIN") != test.netAdmin { + t.Fatalf("setup capabilities = %#v", capabilities) + } + argv := sandboxApplicationArgvV1(DockerExecutionPlan{ + Sandbox: plan, + Workload: &WorkloadExecutionPlan{Endpoints: map[string]EndpointExecutionPlan{ + "http": {ContainerPort: 8080}, + }}, + }, []string{"/bin/true"}, true, []int{8080}) + if !containsInOrder(argv, []string{"--public", string(test.public), "--local", string(test.local), "--ambiguous", string(test.ambiguous), "--inbound-tcp", "8080", "--", "/bin/true"}) { + t.Fatalf("sandbox argv = %#v", argv) + } + }) + } +} + +func TestApplicationDockerDNSResolversFollowNetworkPolicy(t *testing.T) { + for _, test := range []struct { + name string + public blueprint.NetworkAccess + local blueprint.NetworkAccess + want []string + }{ + {name: "deny both", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny}, + {name: "public only", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessDeny, want: []string{"8.8.8.8", "8.8.4.4"}}, + {name: "local only", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessAllow}, + {name: "allow both", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessAllow}, + } { + t.Run(test.name, func(t *testing.T) { + plan := newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 501, GID: 20, DockerUser: "501:20"}, + blueprint.RuntimeNetwork{Public: test.public, Local: test.local}, + ) + if got := applicationDockerDNSResolversV1(plan.Network); !slices.Equal(got, test.want) { + t.Fatalf("Docker DNS resolvers = %#v, want %#v", got, test.want) + } + }) + } +} + +func TestApplicationRenderersConfigurePublicOnlyDNS(t *testing.T) { + plan := DockerExecutionPlan{ + EnvironmentID: "demo", DeploymentDir: t.TempDir(), Phase: blueprint.PhaseStaged, + Image: "reploy/demo:staging", ContainerName: "demo", NetworkName: "demo", + Sandbox: newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 501, GID: 20, DockerUser: "501:20"}, + blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessAllow, Local: blueprint.NetworkAccessDeny}, + ), + Workload: &WorkloadExecutionPlan{Argv: []string{"/bin/true"}, Endpoints: map[string]EndpointExecutionPlan{}}, + } + persistent, err := RenderDockerInputs(plan, "demo") + if err != nil { + t.Fatal(err) + } + var compose composePlanDocument + if err := yaml.Unmarshal(persistent.Compose, &compose); err != nil { + t.Fatal(err) + } + want := []string{"8.8.8.8", "8.8.4.4"} + if got := compose.Services["environment"].DNS; !slices.Equal(got, want) { + t.Fatalf("persistent DNS = %#v, want %#v", got, want) + } + transient, err := TransientCommandSpec(plan, ResolvedEnvironmentCommand{Argv: []string{"/bin/true"}}, nil, false, false) + if err != nil { + t.Fatal(err) + } + if !containsInOrder(transient.Args, []string{"--dns", "8.8.8.8", "--dns", "8.8.4.4"}) { + t.Fatalf("transient DNS arguments = %#v", transient.Args) + } +} + +func TestRootApplicationSetupCanClearSupplementaryGroups(t *testing.T) { + plan := newApplicationSandboxPlanV1(RuntimeUserPlan{UID: 0, GID: 0, DockerUser: "0:0"}) + capabilities := applicationSetupCapabilitiesV1(plan) + if !containsString(capabilities, "SETGID") || !containsString(capabilities, "SETPCAP") || !containsString(capabilities, "NET_ADMIN") || containsString(capabilities, "SETUID") { + t.Fatalf("root setup capabilities = %#v", capabilities) + } +} + +func TestTransientApplicationDoesNotInheritWorkloadEndpointGrants(t *testing.T) { + plan := DockerExecutionPlan{ + DeploymentDir: t.TempDir(), Image: "reploy/demo:staging", ContainerName: "demo", + Sandbox: newApplicationSandboxPlanV1(RuntimeUserPlan{UID: 501, GID: 20, DockerUser: "501:20"}), + Workload: &WorkloadExecutionPlan{Endpoints: map[string]EndpointExecutionPlan{ + "http": {ContainerPort: 8080}, + }}, + } + spec, err := TransientCommandSpec(plan, ResolvedEnvironmentCommand{Argv: []string{"/bin/true"}}, nil, false, false) + if err != nil { + t.Fatal(err) + } + if containsString(spec.Args, "--inbound-tcp") || containsString(spec.Args, "8080") { + t.Fatalf("transient command inherited workload endpoint grant: %#v", spec.Args) + } +} + func containsString(values []string, want string) bool { for _, value := range values { if value == want { diff --git a/internal/dockerdeploy/build_publication_test.go b/internal/dockerdeploy/build_publication_test.go index 00fcacb7..cce209ba 100644 --- a/internal/dockerdeploy/build_publication_test.go +++ b/internal/dockerdeploy/build_publication_test.go @@ -407,6 +407,7 @@ func publicationLockFixture(t *testing.T, dir string, imageChar string, configCh image := providers.RealizedImageV1{Digest: rendererDigest(imageChar), ConfigDigest: rendererDigest(configChar), RootFSSubject: runtimeRootFS} policy := deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{}, } policyDigest, err := deploy.RuntimePolicyDigestV1(policy) diff --git a/internal/dockerdeploy/command_execution.go b/internal/dockerdeploy/command_execution.go index 239fd200..2c9c3976 100644 --- a/internal/dockerdeploy/command_execution.go +++ b/internal/dockerdeploy/command_execution.go @@ -216,13 +216,16 @@ func transientContainerCommandSpecV1(operation string, container string, plan Do home := temporaryHomeForPlan(plan) args := []string{ operation, "--pull", "never", "--rm", "--name", container, - "--user", plan.Sandbox.RuntimeUser.DockerUser, + "--user", "0:0", "--cap-drop", "ALL", "--security-opt", "no-new-privileges=true", "--security-opt", "seccomp=" + plan.Sandbox.Kernel.SeccompProfile, } - for _, group := range dockerSupplementaryGroupsV1(plan.Sandbox.RuntimeUser.SupplementaryGIDs) { - args = append(args, "--group-add", group) + for _, capability := range applicationSetupCapabilitiesV1(plan.Sandbox) { + args = append(args, "--cap-add", capability) + } + for _, resolver := range applicationDockerDNSResolversV1(plan.Sandbox.Network) { + args = append(args, "--dns", resolver) } if plan.Sandbox.ReadOnlyRoot { args = append(args, "--read-only") @@ -286,7 +289,7 @@ func transientContainerCommandSpecV1(operation string, container string, plan Do "--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, ) - args = append(args, verifiedApplicationArgvV1(command.Argv)...) + args = append(args, sandboxApplicationArgvV1(plan, command.Argv, true, []int{})...) return CommandSpec{Name: "docker", Args: args}, nil } diff --git a/internal/dockerdeploy/command_execution_test.go b/internal/dockerdeploy/command_execution_test.go index 6e1e44ca..97ef6681 100644 --- a/internal/dockerdeploy/command_execution_test.go +++ b/internal/dockerdeploy/command_execution_test.go @@ -85,7 +85,7 @@ func TestTransientAndShellCommandsUseDockerExecArgv(t *testing.T) { t.Fatal(err) } joined := strings.Join(spec.Args, "|") - if strings.Contains(joined, "sh|-c") || !containsInOrder(spec.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "verify-exec", "--", "/opt/demo", ";rm", "$(touch pwned)"}) { + if strings.Contains(joined, "sh|-c") || !containsInOrder(spec.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "sandbox-exec", "--uid", "501", "--gid", "20", "--groups", "33,44", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/demo", ";rm", "$(touch pwned)"}) { t.Fatalf("spec = %#v", spec) } if !containsInOrder(spec.Args, []string{"--mount", "type=bind,source=" + outputDir + ",target=" + runtimeOutputRoot, "--env", runtimeOutputFileVariable + "=" + runtimeOutputRoot + "/output"}) { @@ -94,13 +94,13 @@ func TestTransientAndShellCommandsUseDockerExecArgv(t *testing.T) { if !containsAdjacent(spec.Args, "--pull", "never") { t.Fatalf("transient command permits image pulls: %#v", spec.Args) } - if !containsInOrder(spec.Args, []string{"--user", "501:20", "--cap-drop", "ALL"}) || - !containsInOrder(spec.Args, []string{"--group-add", "33", "--group-add", "44"}) || - !containsInOrder(spec.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "verify-exec", "--", "/opt/demo", ";rm", "$(touch pwned)"}) { + if !containsInOrder(spec.Args, []string{"--user", "0:0", "--cap-drop", "ALL"}) || + !containsInOrder(spec.Args, []string{"--cap-add", "NET_ADMIN", "--cap-add", "SETGID", "--cap-add", "SETPCAP", "--cap-add", "SETUID"}) || + !containsInOrder(spec.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "sandbox-exec", "--uid", "501", "--gid", "20", "--groups", "33,44", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/demo", ";rm", "$(touch pwned)"}) { t.Fatalf("transient command does not start directly with its final identity and command: %#v", spec.Args) } shell := ShellCommandSpec(plan, true, true) - if !strings.Contains(strings.Join(shell.Args, " "), "--interactive --tty") || !containsInOrder(shell.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "verify-exec", "--", "/bin/sh"}) { + if !strings.Contains(strings.Join(shell.Args, " "), "--interactive --tty") || !containsInOrder(shell.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "sandbox-exec", "--uid", "501", "--gid", "20", "--groups", "33,44", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/bin/sh"}) { t.Fatalf("shell = %#v", shell) } if !containsInOrder(shell.Args, []string{"--read-only", "--tmpfs", transientHomeMountForPlan(plan)}) || @@ -204,7 +204,7 @@ func TestPlanTransientContainerExecutionV1SeparatesCreateStartAndCleanup(t *test t.Fatalf("create prefix = %#v", execution.Create.Args) } if !containsInOrder(execution.Create.Args, []string{"--interactive", "--tty"}) || - !containsInOrder(execution.Create.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "verify-exec", "--", "/opt/demo", "export"}) { + !containsInOrder(execution.Create.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "sandbox-exec", "--uid", "501", "--gid", "20", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/demo", "export"}) { t.Fatalf("create args = %#v", execution.Create.Args) } if !reflect.DeepEqual(execution.Start.Args, []string{"start", "--attach", "--interactive", wantContainer}) { diff --git a/internal/dockerdeploy/current_workload_lifecycle.go b/internal/dockerdeploy/current_workload_lifecycle.go index 34c2c80e..15ddbd53 100644 --- a/internal/dockerdeploy/current_workload_lifecycle.go +++ b/internal/dockerdeploy/current_workload_lifecycle.go @@ -44,7 +44,7 @@ type currentWorkloadLifecycleBackendV1 struct { cleanup func(string) CommandSpec runTemporary func(temporaryCommandRunner, CommandSpec, CommandSpec, RunOptions) error runCommand func(CommandSpec, RunOptions) error - inject func(context.Context, string, string, privateWorkloadEnvironmentV1, RunOptions, commandRunner) error + inject func(context.Context, string, string, ApplicationSandboxPlanV1, privateWorkloadEnvironmentV1, RunOptions, commandRunner) error readiness func(context.Context, EndpointExecutionPlan, func(context.Context) error) error serviceCheck func(string, string, time.Duration) error } @@ -222,6 +222,7 @@ func runCurrentWorkloadLifecycleV1(ctx context.Context, input CurrentWorkloadLif startCtx, spec.Name, input.Plan.Docker.ContainerName, + input.Plan.Docker.Sandbox, input.PrivateEnvironment, runOptions, backend.runCommand, diff --git a/internal/dockerdeploy/current_workload_lifecycle_test.go b/internal/dockerdeploy/current_workload_lifecycle_test.go index bfac86ed..571e72e9 100644 --- a/internal/dockerdeploy/current_workload_lifecycle_test.go +++ b/internal/dockerdeploy/current_workload_lifecycle_test.go @@ -469,7 +469,7 @@ func currentWorkloadLifecycleTestBackend(t *testing.T, lifecycle LifecyclePlan, *order = append(*order, "run "+spec.Name) return nil }, - inject: func(context.Context, string, string, privateWorkloadEnvironmentV1, RunOptions, commandRunner) error { + inject: func(context.Context, string, string, ApplicationSandboxPlanV1, privateWorkloadEnvironmentV1, RunOptions, commandRunner) error { *order = append(*order, "inject private environment") return nil }, diff --git a/internal/dockerdeploy/execution_plan.go b/internal/dockerdeploy/execution_plan.go index 12eb8b23..bb62a369 100644 --- a/internal/dockerdeploy/execution_plan.go +++ b/internal/dockerdeploy/execution_plan.go @@ -143,7 +143,7 @@ func PlanDockerExecution(document blueprint.Document, context DockerPlanContext) if err != nil { return DockerExecutionPlan{}, err } - plan.Sandbox = newApplicationSandboxPlanV1(runtimeUser) + plan.Sandbox = newApplicationSandboxPlanWithNetworkV1(runtimeUser, document.Environment.Runtime.Network) if err := ValidateApplicationSandboxPlanV1(plan.Sandbox); err != nil { return DockerExecutionPlan{}, err } diff --git a/internal/dockerdeploy/execution_render.go b/internal/dockerdeploy/execution_render.go index 16c71277..e7a55c6a 100644 --- a/internal/dockerdeploy/execution_render.go +++ b/internal/dockerdeploy/execution_render.go @@ -2,6 +2,7 @@ package dockerdeploy import ( "fmt" + "slices" "sort" "strconv" "strings" @@ -52,6 +53,7 @@ type composePlanService struct { User string `yaml:"user"` GroupAdd []string `yaml:"group_add"` CapDrop []string `yaml:"cap_drop"` + CapAdd []string `yaml:"cap_add"` SecurityOpt []string `yaml:"security_opt"` Restart string `yaml:"restart,omitempty"` Entrypoint []string `yaml:"entrypoint,omitempty,flow"` @@ -62,6 +64,7 @@ type composePlanService struct { ReadOnly bool `yaml:"read_only"` Environment map[string]string `yaml:"environment"` Tmpfs []string `yaml:"tmpfs"` + DNS []string `yaml:"dns,omitempty"` } type composePlanMount struct { @@ -108,10 +111,14 @@ func RenderDockerInputs(plan DockerExecutionPlan, controlScript string) (DockerR CapDrop: []string{"ALL"}, SecurityOpt: []string{"no-new-privileges:true", "seccomp=" + plan.Sandbox.Kernel.SeccompProfile}, ReadOnly: plan.Sandbox.ReadOnlyRoot, Environment: temporaryEnvironmentForPlan(plan), Tmpfs: []string{temporaryHomeMountForPlan(plan)}, + DNS: applicationDockerDNSResolversV1(plan.Sandbox.Network), } if plan.Workload != nil { + service.User = "0:0" + service.GroupAdd = []string{} + service.CapAdd = applicationSetupCapabilitiesV1(plan.Sandbox) service.Entrypoint = []string{plan.Sandbox.StartupVerifier.Path} - service.Command = verifiedApplicationArgvV1(plan.Workload.Argv) + service.Command = sandboxApplicationArgvV1(plan, plan.Workload.Argv, true, applicationInboundTCPPortsV1(plan)) } if plan.PrivateEnvironment { if plan.Workload == nil { @@ -123,7 +130,7 @@ func RenderDockerInputs(plan DockerExecutionPlan, controlScript string) (DockerR composeLauncher := strings.ReplaceAll(privateWorkloadEnvironmentLauncherV1, "$", "$$") launcher := []string{"/bin/sh", "-c", composeLauncher, "reploy-private-environment"} launcher = append(launcher, plan.Workload.Argv...) - service.Command = verifiedApplicationArgvV1(launcher) + service.Command = sandboxApplicationArgvV1(plan, launcher, true, applicationInboundTCPPortsV1(plan)) service.StdinOpen = true } volumes := map[string]any{} @@ -213,11 +220,63 @@ func RenderDockerInputs(plan DockerExecutionPlan, controlScript string) (DockerR }, nil } -func verifiedApplicationArgvV1(argv []string) []string { - result := []string{"verify-exec", "--"} +func sandboxApplicationArgvV1(plan DockerExecutionPlan, argv []string, installNetwork bool, inboundTCP []int) []string { + mode := "restricted-exec" + if installNetwork { + mode = "sandbox-exec" + } + user := plan.Sandbox.RuntimeUser + result := []string{mode, "--uid", strconv.Itoa(user.UID), "--gid", strconv.Itoa(user.GID)} + if len(user.SupplementaryGIDs) != 0 { + result = append(result, "--groups", joinDecimalValuesV1(user.SupplementaryGIDs)) + } + if installNetwork { + result = append(result, + "--public", string(plan.Sandbox.Network.Public), + "--local", string(plan.Sandbox.Network.Local), + "--ambiguous", string(plan.Sandbox.Network.Ambiguous), + ) + ports := append([]int{}, inboundTCP...) + sort.Ints(ports) + ports = slices.Compact(ports) + if len(ports) != 0 { + result = append(result, "--inbound-tcp", joinDecimalValuesV1(ports)) + } + } + result = append(result, "--") return append(result, argv...) } +func applicationSetupCapabilitiesV1(plan ApplicationSandboxPlanV1) []string { + result := []string{"NET_ADMIN", "SETGID", "SETPCAP"} + if plan.RuntimeUser.UID != 0 { + result = append(result, "SETUID") + } + sort.Strings(result) + return result +} + +func applicationInboundTCPPortsV1(plan DockerExecutionPlan) []int { + if plan.Workload == nil { + return []int{} + } + ports := make([]int, 0, len(plan.Workload.Endpoints)) + for _, endpoint := range plan.Workload.Endpoints { + ports = append(ports, endpoint.ContainerPort) + } + sort.Ints(ports) + ports = slices.Compact(ports) + return ports +} + +func joinDecimalValuesV1(values []int) string { + items := make([]string, len(values)) + for index, value := range values { + items[index] = strconv.Itoa(value) + } + return strings.Join(items, ",") +} + func temporaryHomeForPlan(plan DockerExecutionPlan) string { return plan.Sandbox.TemporaryHome } diff --git a/internal/dockerdeploy/execution_render_test.go b/internal/dockerdeploy/execution_render_test.go index d4da5db2..c7c77476 100644 --- a/internal/dockerdeploy/execution_render_test.go +++ b/internal/dockerdeploy/execution_render_test.go @@ -30,7 +30,7 @@ func TestRenderDockerInputsFromResolvedPlan(t *testing.T) { if compose != normalizedGolden { t.Fatalf("compose golden mismatch\nactual:\n%s\nwant:\n%s", compose, wantGolden) } - for _, want := range []string{"image: reploy/demo:staging", "pull_policy: never", `user: "501:20"`, "cap_drop:", "- ALL", "no-new-privileges:true", "seccomp=builtin", "read_only: true", "HOME: /mnt/reploy-home", "TMPDIR: /mnt/reploy-home", "- /mnt/reploy-home:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=501,gid=20", "type: bind", "127.0.0.1:18080:8080", "/opt/reploy/python/bin/demo", "name: demo-staging-abcd"} { + for _, want := range []string{"image: reploy/demo:staging", "pull_policy: never", `user: "0:0"`, "cap_drop:", "- ALL", "cap_add:", "- NET_ADMIN", "- SETGID", "- SETPCAP", "- SETUID", "sandbox-exec", "--public", "deny", "--local", "deny", "--inbound-tcp", "no-new-privileges:true", "seccomp=builtin", "read_only: true", "HOME: /mnt/reploy-home", "TMPDIR: /mnt/reploy-home", "- /mnt/reploy-home:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=501,gid=20", "type: bind", "127.0.0.1:18080:8080", "/opt/reploy/python/bin/demo", "name: demo-staging-abcd"} { if !strings.Contains(compose, want) { t.Fatalf("compose missing %q:\n%s", want, compose) } diff --git a/internal/dockerdeploy/full_validation_test.go b/internal/dockerdeploy/full_validation_test.go index faf1f333..4687fd62 100644 --- a/internal/dockerdeploy/full_validation_test.go +++ b/internal/dockerdeploy/full_validation_test.go @@ -27,6 +27,7 @@ func fullValidationInput(t *testing.T, digestChar string) FullImageValidationInp Image: request.Source, Profiles: []providers.RequirementProfile{}, Outputs: []providers.RealizedOutput{}, RuntimePolicy: deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{}, }, } diff --git a/internal/dockerdeploy/installed_service_container.go b/internal/dockerdeploy/installed_service_container.go index f4653d8b..9c3ddc57 100644 --- a/internal/dockerdeploy/installed_service_container.go +++ b/internal/dockerdeploy/installed_service_container.go @@ -103,6 +103,7 @@ func RunInstalledServiceContainerV1(ctx context.Context, deploymentDir string, a start, cleanup, plan.Docker.ContainerName, + plan.Docker.Sandbox, environment, options, runDockerCommand, diff --git a/internal/dockerdeploy/prepared_python_graph_reuse_test.go b/internal/dockerdeploy/prepared_python_graph_reuse_test.go index 24425241..6ed01e16 100644 --- a/internal/dockerdeploy/prepared_python_graph_reuse_test.go +++ b/internal/dockerdeploy/prepared_python_graph_reuse_test.go @@ -463,6 +463,7 @@ func newPreparedPythonGraphReuseFixtureWithManifest(t *testing.T, sourceManifest Catalog: append([]providers.RealizedOutput{}, request.EarlierCatalog...), RuntimePolicy: deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{}, }, RuntimeLayer: testApplicationRuntimeLayerV1(t, request.Platform, resultImage, providers.RealizedImageV1{ @@ -581,6 +582,7 @@ func newPreparedAPTGraphReuseFixture(t *testing.T) ( Catalog: []providers.RealizedOutput{}, RuntimePolicy: deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{}, }, RuntimeLayer: testApplicationRuntimeLayerV1(t, descriptor.Platform, resultImage, providers.RealizedImageV1{ diff --git a/internal/dockerdeploy/private_workload_environment_inject.go b/internal/dockerdeploy/private_workload_environment_inject.go index 978cf400..ffe50007 100644 --- a/internal/dockerdeploy/private_workload_environment_inject.go +++ b/internal/dockerdeploy/private_workload_environment_inject.go @@ -20,6 +20,7 @@ func injectPrivateWorkloadEnvironmentV1( ctx context.Context, dockerPath string, containerName string, + sandbox ApplicationSandboxPlanV1, environment privateWorkloadEnvironmentV1, options RunOptions, run commandRunner, @@ -47,13 +48,12 @@ func injectPrivateWorkloadEnvironmentV1( "exec", "-i", containerName, - "/bin/sh", - "-c", - privateWorkloadEnvironmentRelayV1, - "reploy-private-environment", - privateWorkloadEnvironmentFIFOPathV1, }, } + relay := []string{"/bin/sh", "-c", privateWorkloadEnvironmentRelayV1, "reploy-private-environment", privateWorkloadEnvironmentFIFOPathV1} + wrapperPlan := DockerExecutionPlan{Sandbox: sandbox} + spec.Args = append(spec.Args, sandbox.StartupVerifier.Path) + spec.Args = append(spec.Args, sandboxApplicationArgvV1(wrapperPlan, relay, false, []int{})...) if err := run(spec, runOptions); err != nil { return fmt.Errorf("inject private workload environment through one-shot FIFO relay: %w", err) } @@ -65,6 +65,7 @@ func startAndInjectPrivateWorkloadEnvironmentV1( start CommandSpec, cleanup CommandSpec, containerName string, + sandbox ApplicationSandboxPlanV1, environment privateWorkloadEnvironmentV1, options RunOptions, run commandRunner, @@ -78,7 +79,7 @@ func startAndInjectPrivateWorkloadEnvironmentV1( if !environment.Present { return nil } - if err := injectPrivateWorkloadEnvironmentV1(ctx, start.Name, containerName, environment, options, run); err != nil { + if err := injectPrivateWorkloadEnvironmentV1(ctx, start.Name, containerName, sandbox, environment, options, run); err != nil { cleanupOptions := options cleanupOptions.Context = context.WithoutCancel(ctx) cleanupOptions.Stdin = nil diff --git a/internal/dockerdeploy/private_workload_environment_integration_test.go b/internal/dockerdeploy/private_workload_environment_integration_test.go index d567808f..76ea4738 100644 --- a/internal/dockerdeploy/private_workload_environment_integration_test.go +++ b/internal/dockerdeploy/private_workload_environment_integration_test.go @@ -115,6 +115,7 @@ printf 'private-mask-pass\n'`, expectedTokenDigest) start, cleanup, container, + plan.Sandbox, environment, RunOptions{}, runCommandWithoutDockerPreflight, diff --git a/internal/dockerdeploy/private_workload_environment_test.go b/internal/dockerdeploy/private_workload_environment_test.go index 4678e304..71454b4f 100644 --- a/internal/dockerdeploy/private_workload_environment_test.go +++ b/internal/dockerdeploy/private_workload_environment_test.go @@ -225,7 +225,7 @@ func TestRenderDockerInputsUsesSecretFreePrivateLauncher(t *testing.T) { compose := string(rendered.Compose) for _, want := range []string{ "stdin_open: true", "reploy_private_environment_ready", "/opt/demo", "serve", - "entrypoint: [/reploy-probe]", "command: [verify-exec, --, /bin/sh, -c", + "entrypoint: [/reploy-probe]", "command: [sandbox-exec, --uid", "--public, deny", "--local, deny", "/bin/sh, -c", "source: /dev/null", "target: /deployment/.env", "read_only: true", "/deployment/.reploy:" + privateRuntimeDirectoryMaskOptionsV1, } { @@ -249,7 +249,8 @@ func TestInjectPrivateWorkloadEnvironmentV1UsesOnlyStdin(t *testing.T) { environment := privateWorkloadEnvironmentV1{Present: true, Payload: []byte("TOKEN=private value\n\n")} var gotSpec CommandSpec var gotInput []byte - err := injectPrivateWorkloadEnvironmentV1(t.Context(), "/usr/bin/docker", "demo", environment, RunOptions{}, func(spec CommandSpec, options RunOptions) error { + sandbox := testApplicationSandboxPlanV1(1000, 1000) + err := injectPrivateWorkloadEnvironmentV1(t.Context(), "/usr/bin/docker", "demo", sandbox, environment, RunOptions{}, func(spec CommandSpec, options RunOptions) error { gotSpec = spec var err error gotInput, err = readAllForPrivateEnvironmentTest(options) @@ -263,7 +264,8 @@ func TestInjectPrivateWorkloadEnvironmentV1UsesOnlyStdin(t *testing.T) { t.Fatalf("command contains secret: %#v", gotSpec) } if !reflect.DeepEqual(gotSpec.Args, []string{ - "exec", "-i", "demo", "/bin/sh", "-c", privateWorkloadEnvironmentRelayV1, + "exec", "-i", "demo", sandbox.StartupVerifier.Path, + "restricted-exec", "--uid", "1000", "--gid", "1000", "--", "/bin/sh", "-c", privateWorkloadEnvironmentRelayV1, "reploy-private-environment", privateWorkloadEnvironmentFIFOPathV1, }) { t.Fatalf("relay command = %#v", gotSpec) @@ -291,6 +293,7 @@ func TestStartAndInjectPrivateWorkloadEnvironmentV1CleansFailedContainer(t *test CommandSpec{Name: "docker", Args: []string{"compose", "up"}}, CommandSpec{Name: "docker", Args: []string{"compose", "down"}}, "demo", + testApplicationSandboxPlanV1(1000, 1000), environment, RunOptions{}, func(spec CommandSpec, _ RunOptions) error { @@ -304,7 +307,7 @@ func TestStartAndInjectPrivateWorkloadEnvironmentV1CleansFailedContainer(t *test if err == nil || !strings.Contains(err.Error(), "one-shot FIFO relay") { t.Fatalf("error = %v", err) } - if len(order) != 3 || order[0] != "compose up" || !strings.HasPrefix(order[1], "exec -i demo /bin/sh -c ") || order[2] != "compose down" { + if len(order) != 3 || order[0] != "compose up" || !strings.HasPrefix(order[1], "exec -i demo "+deploy.ApplicationStartupVerifierPathV1+" restricted-exec ") || order[2] != "compose down" { t.Fatalf("order = %#v", order) } } @@ -438,6 +441,12 @@ func TestPrivateWorkloadEnvironmentRealDockerIsolation(t *testing.T) { t.Skip("set REPLOY_DOCKER_INTEGRATION=1 to run the real-Docker private environment test") } container := "reploy-private-env-test-" + strconv.Itoa(os.Getpid()) + platform, err := blueprint.ParsePlatform("linux/" + runtime.GOARCH) + if err != nil { + t.Fatal(err) + } + probe := buildIntegrationProbeWorkspace(t, platform) + sandbox := testApplicationSandboxPlanV1(65532, 65532) run := func(spec CommandSpec, options RunOptions) error { options.Context = t.Context() return runCommandWithoutDockerPreflight(spec, options) @@ -451,11 +460,14 @@ func TestPrivateWorkloadEnvironmentRealDockerIsolation(t *testing.T) { }) script := privateWorkloadEnvironmentLauncherV1 create := CommandSpec{Name: "docker", Args: []string{ - "create", "--name", container, "-i", "--user", "65532:65532", - "--tmpfs", environmentTemporaryHome + ":rw,noexec,nosuid,nodev,size=64m,mode=1777", + "create", "--name", container, "-i", "--user", "0:0", + "--cap-drop", "ALL", "--cap-add", "NET_ADMIN", "--cap-add", "SETGID", "--cap-add", "SETPCAP", "--cap-add", "SETUID", + "--security-opt", "no-new-privileges=true", + "--tmpfs", environmentTemporaryHome + ":rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=65532,gid=65532", "--env", "HOME=" + environmentTemporaryHome, - "--entrypoint", "/bin/sh", "python:3.11-slim", - "-c", script, "reploy-private-environment", + "--mount", "type=bind,source=" + probe.HostExecutable + ",target=" + sandbox.StartupVerifier.Path + ",readonly", + "--entrypoint", sandbox.StartupVerifier.Path, "python:3.11-slim", + "sandbox-exec", "--uid", "65532", "--gid", "65532", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/bin/sh", "-c", script, "reploy-private-environment", "python", "-c", `import os,time; name="REPLOY_"+"PRIVATE_"+"TEST"; value=os.environ.get(name); data=os.read(0, 1); print("ENV_PRESENT", value is not None, "LENGTH", len(value or ""), "STDIN_EOF", data == b"", flush=True); time.sleep(20)`, }} if err := run(create, RunOptions{}); err != nil { @@ -467,7 +479,7 @@ func TestPrivateWorkloadEnvironmentRealDockerIsolation(t *testing.T) { secret := "private-docker-test-value" environment := privateWorkloadEnvironmentV1{Present: true, Payload: []byte("REPLOY_PRIVATE_TEST=" + secret + "\n\n")} started := time.Now() - if err := injectPrivateWorkloadEnvironmentV1(t.Context(), "docker", container, environment, RunOptions{}, run); err != nil { + if err := injectPrivateWorkloadEnvironmentV1(t.Context(), "docker", container, sandbox, environment, RunOptions{}, run); err != nil { t.Fatal(err) } if elapsed := time.Since(started); elapsed > 5*time.Second { diff --git a/internal/dockerdeploy/provider_graph_validation_test.go b/internal/dockerdeploy/provider_graph_validation_test.go index df8a2344..02562341 100644 --- a/internal/dockerdeploy/provider_graph_validation_test.go +++ b/internal/dockerdeploy/provider_graph_validation_test.go @@ -80,6 +80,7 @@ func TestPrepareProviderGraphValidationInspectsBaseOnlyGraph(t *testing.T) { } policy := deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{}, } result, err := prepareProviderGraphValidation( diff --git a/internal/dockerdeploy/provider_install_host_execute.go b/internal/dockerdeploy/provider_install_host_execute.go index 380f4ca2..6e9a5f72 100644 --- a/internal/dockerdeploy/provider_install_host_execute.go +++ b/internal/dockerdeploy/provider_install_host_execute.go @@ -43,6 +43,7 @@ func startProviderInstallHostV1(ctx context.Context, plan providerInstallationPl commands.Start, cleanup, plan.Docker.ContainerName, + plan.Docker.Sandbox, environment, options, runCommandWithoutDockerPreflight, diff --git a/internal/dockerdeploy/runtime_host_preflight_test.go b/internal/dockerdeploy/runtime_host_preflight_test.go index 383ad3a9..40aa97be 100644 --- a/internal/dockerdeploy/runtime_host_preflight_test.go +++ b/internal/dockerdeploy/runtime_host_preflight_test.go @@ -234,8 +234,9 @@ func TestRuntimeHostSourcesV1IncludesOnlyBindAndExplicitOutputMounts(t *testing. func runtimeHostPolicy(mounts []deploy.RuntimeMountV1) deploy.RuntimePolicyV1 { return deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{{ - ID: "command/check", Mounts: mounts, Executables: []providers.QualifiedOutput{}, + ID: "command/check", InboundTCP: []string{}, Mounts: mounts, Executables: []providers.QualifiedOutput{}, }}, } } diff --git a/internal/dockerdeploy/runtime_logs.go b/internal/dockerdeploy/runtime_logs.go index 62c3f433..f14a2c92 100644 --- a/internal/dockerdeploy/runtime_logs.go +++ b/internal/dockerdeploy/runtime_logs.go @@ -83,10 +83,16 @@ func extractRuntimeStartupLogDiagnostics(logs string) runtimeStartupLogDiagnosti } continue } + cleaned := cleanRuntimeLogSnippetLine(line) + if strings.HasPrefix(cleaned, "reploy-probe: sandbox-exec:") { + diagnostics.Failure = "application sandbox setup failed" + lines = append(lines, cleaned) + continue + } if !capturing { continue } - if cleaned := cleanRuntimeLogSnippetLine(line); cleaned != "" { + if cleaned != "" { lines = append(lines, cleaned) } } diff --git a/internal/dockerdeploy/runtime_plan.go b/internal/dockerdeploy/runtime_plan.go index aa8d054f..b8b09c66 100644 --- a/internal/dockerdeploy/runtime_plan.go +++ b/internal/dockerdeploy/runtime_plan.go @@ -26,6 +26,19 @@ func RuntimePlansV1(document blueprint.Document, dockerPlan DockerExecutionPlan) if err := ValidateApplicationSandboxPlanV1(dockerPlan.Sandbox); err != nil { return nil, fmt.Errorf("runtime application sandbox: %w", err) } + resolvedNetwork := normalizeRuntimeNetworkV1(document.Environment.Runtime.Network) + wantNetwork := ApplicationNetworkPolicyV1{ + Public: resolvedNetwork.Public, + Local: resolvedNetwork.Local, + Ambiguous: resolvedNetwork.Ambiguous, + } + if dockerPlan.Sandbox.Network != wantNetwork { + return nil, fmt.Errorf("runtime application network policy does not match the resolved blueprint") + } + workloadInboundTCP, err := runtimeWorkloadInboundTCPV1(document, dockerPlan) + if err != nil { + return nil, err + } baseMounts, err := runtimeMountsV1(dockerPlan) if err != nil { return nil, err @@ -34,7 +47,7 @@ func RuntimePlansV1(document blueprint.Document, dockerPlan DockerExecutionPlan) Destination: temporaryHomeForPlan(dockerPlan), SourceKind: deploy.RuntimeMountSourceGenerated, }) plans := []deploy.RuntimePlanV1{{ - ID: runtimeShellPlanID, Mounts: cloneRuntimeMountsV1(withHome), Executables: []providers.QualifiedOutput{}, + ID: runtimeShellPlanID, InboundTCP: []string{}, Mounts: cloneRuntimeMountsV1(withHome), Executables: []providers.QualifiedOutput{}, }} commandNames, err := runtimeTransientCommandNamesV1(document) @@ -49,11 +62,11 @@ func RuntimePlansV1(document blueprint.Document, dockerPlan DockerExecutionPlan) } executables := []providers.QualifiedOutput{output} plans = append(plans, deploy.RuntimePlanV1{ - ID: runtimeCommandPlanID(name, false), Mounts: cloneRuntimeMountsV1(withHome), Executables: executables, + ID: runtimeCommandPlanID(name, false), InboundTCP: []string{}, Mounts: cloneRuntimeMountsV1(withHome), Executables: executables, }) if command.NativeCommand || command.DeployedCommand { plans = append(plans, deploy.RuntimePlanV1{ - ID: runtimeCommandPlanID(name, true), + ID: runtimeCommandPlanID(name, true), InboundTCP: []string{}, Mounts: appendRuntimeMountV1(withHome, deploy.RuntimeMountV1{ Destination: runtimeOutputRoot, SourceKind: deploy.RuntimeMountSourceDirectory, }), @@ -68,7 +81,7 @@ func RuntimePlansV1(document blueprint.Document, dockerPlan DockerExecutionPlan) return nil, fmt.Errorf("runtime workload: %w", err) } plans = append(plans, deploy.RuntimePlanV1{ - ID: runtimeWorkloadPlanID, Mounts: cloneRuntimeMountsV1(withHome), + ID: runtimeWorkloadPlanID, InboundTCP: workloadInboundTCP, Mounts: cloneRuntimeMountsV1(withHome), Executables: []providers.QualifiedOutput{output}, }) } @@ -77,6 +90,24 @@ func RuntimePlansV1(document blueprint.Document, dockerPlan DockerExecutionPlan) return plans, nil } +func runtimeWorkloadInboundTCPV1(document blueprint.Document, dockerPlan DockerExecutionPlan) ([]string, error) { + if document.Environment.Workload == nil { + return []string{}, nil + } + if len(document.Environment.Workload.Endpoints) != len(dockerPlan.Workload.Endpoints) { + return nil, fmt.Errorf("runtime workload endpoints do not match the resolved Docker plan") + } + ports := make([]int, 0, len(document.Environment.Workload.Endpoints)) + for name, endpoint := range document.Environment.Workload.Endpoints { + planned, found := dockerPlan.Workload.Endpoints[name] + if !found || planned.ContainerPort != endpoint.Port { + return nil, fmt.Errorf("runtime workload endpoint %q does not match the resolved Docker plan", name) + } + ports = append(ports, endpoint.Port) + } + return deploy.CanonicalRuntimeInboundTCPV1(ports), nil +} + func runtimeCommandPlanID(commandName string, output bool) string { id := "command/" + commandName if output { diff --git a/internal/dockerdeploy/runtime_plan_test.go b/internal/dockerdeploy/runtime_plan_test.go index 31654b39..2d45f33a 100644 --- a/internal/dockerdeploy/runtime_plan_test.go +++ b/internal/dockerdeploy/runtime_plan_test.go @@ -94,6 +94,53 @@ func TestRuntimePlansV1RejectsWorkloadPlanMismatch(t *testing.T) { } } +func TestRuntimePlansV1RejectsNetworkPolicyMismatch(t *testing.T) { + document := runtimePlanDocument() + plan := DockerExecutionPlan{ + Workload: &WorkloadExecutionPlan{}, + Sandbox: newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 1000, GID: 1000, DockerUser: "1000:1000"}, + blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessAllow, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, + ), + } + _, err := RuntimePlansV1(document, plan) + if err == nil || !strings.Contains(err.Error(), "network policy does not match") { + t.Fatalf("network policy mismatch error = %v", err) + } +} + +func TestRuntimePlansV1LocksInboundTCPOnlyForWorkload(t *testing.T) { + document := runtimePlanDocument() + document.Environment.Workload.Endpoints = map[string]blueprint.Endpoint{ + "http": {Scheme: "http", Port: 8080}, + "admin": {Scheme: "http", Port: 8081}, + } + dockerPlan := DockerExecutionPlan{ + Sandbox: testApplicationSandboxPlanV1(1000, 1000), + Workload: &WorkloadExecutionPlan{Endpoints: map[string]EndpointExecutionPlan{ + "http": {ContainerPort: 8080}, + "admin": {ContainerPort: 8081}, + }}, + } + plans, err := RuntimePlansV1(document, dockerPlan) + if err != nil { + t.Fatal(err) + } + for _, plan := range plans { + want := []string{} + if plan.ID == runtimeWorkloadPlanID { + want = []string{"8080", "8081"} + } + if !reflect.DeepEqual(plan.InboundTCP, want) { + t.Fatalf("runtime plan %q inbound TCP = %#v, want %#v", plan.ID, plan.InboundTCP, want) + } + } + dockerPlan.Workload.Endpoints["http"] = EndpointExecutionPlan{ContainerPort: 9090} + if _, err := RuntimePlansV1(document, dockerPlan); err == nil || !strings.Contains(err.Error(), "endpoint \"http\"") { + t.Fatalf("endpoint mismatch error = %v", err) + } +} + func runtimePlanDocument() blueprint.Document { return blueprint.Document{Environment: blueprint.Environment{ Applications: map[string]blueprint.Application{ diff --git a/internal/dockerdeploy/runtime_policy_compile.go b/internal/dockerdeploy/runtime_policy_compile.go index 2cc10bcd..8fde5bd7 100644 --- a/internal/dockerdeploy/runtime_policy_compile.go +++ b/internal/dockerdeploy/runtime_policy_compile.go @@ -2,7 +2,9 @@ package dockerdeploy import ( "fmt" + "slices" "sort" + "strconv" "strings" "github.com/omry/reploy/internal/blueprint" @@ -62,6 +64,11 @@ func compileRuntimePolicyV1( ) (deploy.RuntimePolicyV1, error) { canonicalPlans := append([]deploy.RuntimePlanV1{}, plans...) for index := range canonicalPlans { + inboundTCP, err := canonicalRuntimeInboundTCPV1(canonicalPlans[index].InboundTCP) + if err != nil { + return deploy.RuntimePolicyV1{}, fmt.Errorf("runtime plan %q inbound TCP grants: %w", canonicalPlans[index].ID, err) + } + canonicalPlans[index].InboundTCP = inboundTCP canonicalPlans[index].Mounts = append([]deploy.RuntimeMountV1{}, canonicalPlans[index].Mounts...) canonicalPlans[index].Executables = append([]providers.QualifiedOutput{}, canonicalPlans[index].Executables...) sort.Slice(canonicalPlans[index].Mounts, func(left int, right int) bool { @@ -77,8 +84,12 @@ func compileRuntimePolicyV1( }) } sort.Slice(canonicalPlans, func(left int, right int) bool { return canonicalPlans[left].ID < canonicalPlans[right].ID }) + if err := validateRuntimePolicyInboundTCPV1(document, canonicalPlans); err != nil { + return deploy.RuntimePolicyV1{}, err + } policy := deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: normalizeRuntimeNetworkV1(document.Environment.Runtime.Network), ProtectedPaths: protected, Plans: canonicalPlans, } if err := deploy.ValidateRuntimePolicyV1(policy); err != nil { @@ -93,6 +104,38 @@ func compileRuntimePolicyV1( return policy, nil } +func canonicalRuntimeInboundTCPV1(values []string) ([]string, error) { + ports := make([]int, 0, len(values)) + for _, value := range values { + port, err := strconv.Atoi(value) + if err != nil { + return nil, fmt.Errorf("port %q is not a decimal integer", value) + } + ports = append(ports, port) + } + return deploy.CanonicalRuntimeInboundTCPV1(ports), nil +} + +func validateRuntimePolicyInboundTCPV1(document blueprint.Document, plans []deploy.RuntimePlanV1) error { + wantPorts := []int{} + if workload := document.Environment.Workload; workload != nil { + for _, endpoint := range workload.Endpoints { + wantPorts = append(wantPorts, endpoint.Port) + } + } + want := deploy.CanonicalRuntimeInboundTCPV1(wantPorts) + for _, plan := range plans { + planWant := []string{} + if plan.ID == runtimeWorkloadPlanID { + planWant = want + } + if !slices.Equal(plan.InboundTCP, planWant) { + return fmt.Errorf("runtime plan %q inbound TCP grants do not match the resolved blueprint", plan.ID) + } + } + return nil +} + func providerGraphProtectedPaths(graph providers.GraphExecutionResult, plans []deploy.RuntimePlanV1) ([]deploy.ProtectedPathV1, error) { if graph.Materializations == nil || graph.Bundles == nil || graph.Catalog == nil { return nil, fmt.Errorf("runtime policy graph collections must use arrays") diff --git a/internal/dockerdeploy/runtime_policy_compile_test.go b/internal/dockerdeploy/runtime_policy_compile_test.go index 2043692b..4bdcc7ee 100644 --- a/internal/dockerdeploy/runtime_policy_compile_test.go +++ b/internal/dockerdeploy/runtime_policy_compile_test.go @@ -13,6 +13,11 @@ import ( func TestCompileRuntimePolicyCanonicalizesPlans(t *testing.T) { document := runtimePolicyDocument(t) + document.Environment.Runtime.Network = blueprint.RuntimeNetwork{ + Public: blueprint.NetworkAccessAllow, + Local: blueprint.NetworkAccessDeny, + Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, + } plans := []deploy.RuntimePlanV1{ {ID: "workload", Mounts: []deploy.RuntimeMountV1{ {Destination: "/mnt/config", SourceKind: deploy.RuntimeMountSourceFile, ReadOnly: true}, @@ -33,6 +38,9 @@ func TestCompileRuntimePolicyCanonicalizesPlans(t *testing.T) { if len(policy.Plans) != 2 || policy.Plans[0].ID != "shell" || policy.Plans[1].Mounts[0].Destination != "/data" { t.Fatalf("canonical plans = %#v", policy.Plans) } + if policy.Network != document.Environment.Runtime.Network { + t.Fatalf("network policy = %#v, want %#v", policy.Network, document.Environment.Runtime.Network) + } } func TestCompileRuntimePolicyAllowsAbsoluteTargetsAndRejectsOverlap(t *testing.T) { @@ -64,6 +72,25 @@ func TestCompileRuntimePolicyAllowsAbsoluteTargetsAndRejectsOverlap(t *testing.T } } +func TestCompileRuntimePolicyRejectsInboundTCPOutsideWorkloadContract(t *testing.T) { + document := runtimePolicyDocument(t) + document.Environment.Workload = &blueprint.Workload{Endpoints: map[string]blueprint.Endpoint{ + "http": {Scheme: "http", Port: 8080}, + }} + validPlans := []deploy.RuntimePlanV1{ + {ID: runtimeShellPlanID, InboundTCP: []string{}, Mounts: []deploy.RuntimeMountV1{}, Executables: []providers.QualifiedOutput{}}, + {ID: runtimeWorkloadPlanID, InboundTCP: []string{"8080"}, Mounts: []deploy.RuntimeMountV1{}, Executables: []providers.QualifiedOutput{}}, + } + if _, err := CompileRuntimePolicyV1(document, emptyRuntimePolicyGraph(), validPlans); err != nil { + t.Fatalf("valid workload grant: %v", err) + } + invalid := append([]deploy.RuntimePlanV1{}, validPlans...) + invalid[0].InboundTCP = []string{"8080"} + if _, err := CompileRuntimePolicyV1(document, emptyRuntimePolicyGraph(), invalid); err == nil || !strings.Contains(err.Error(), "shell") { + t.Fatalf("transient inbound grant error = %v", err) + } +} + func TestCompileRuntimePolicyProtectsProviderRootsAndExecutableChains(t *testing.T) { transaction := rendererTransaction() generated := acceptedGeneratedExecutable(transaction) diff --git a/internal/dockerdeploy/runtime_readiness_test.go b/internal/dockerdeploy/runtime_readiness_test.go index 82213ee0..cbeb5b6d 100644 --- a/internal/dockerdeploy/runtime_readiness_test.go +++ b/internal/dockerdeploy/runtime_readiness_test.go @@ -305,6 +305,17 @@ func TestCurrentBuildMatchesRuntimeV1TreatsChangedStateAsStale(t *testing.T) { } } +func TestCurrentBuildMatchesRuntimeV1TreatsChangedLockedNetworkPolicyAsStale(t *testing.T) { + current, buildInput := runtimeCurrentBuildFixture(t) + current.Lock.RuntimePolicy.Network.Public = blueprint.NetworkAccessAllow + refreshCurrentBuildReuseGeneration(t, ¤t) + + matched, err := CurrentBuildMatchesRuntimeV1(current, buildInput.DockerPlan) + if err != nil || matched { + t.Fatalf("changed locked network policy = %v, %v", matched, err) + } +} + func TestCurrentBuildMatchesRuntimeV1RejectsMalformedRuntimePlan(t *testing.T) { current, _ := runtimeCurrentBuildFixture(t) matched, err := CurrentBuildMatchesRuntimeV1(current, DockerExecutionPlan{Workload: &WorkloadExecutionPlan{}}) diff --git a/internal/dockerdeploy/runtime_test.go b/internal/dockerdeploy/runtime_test.go index 4c4ce052..f339a577 100644 --- a/internal/dockerdeploy/runtime_test.go +++ b/internal/dockerdeploy/runtime_test.go @@ -229,6 +229,23 @@ func TestExtractRuntimeStartupLogSnippetUsesOnlyMarkerWindows(t *testing.T) { } } +func TestExtractRuntimeStartupLogDiagnosticsIncludesSandboxSetupFailureBeforeMarkers(t *testing.T) { + logs := strings.Join([]string{ + "environment | unrelated prior output", + "environment | reploy-probe: sandbox-exec: install application network policy: apply nftables transaction: operation not supported", + }, "\n") + diagnostics := extractRuntimeStartupLogDiagnostics(logs) + if diagnostics.Failure != "application sandbox setup failed" { + t.Fatalf("failure = %q", diagnostics.Failure) + } + if !strings.Contains(diagnostics.Snippet, "apply nftables transaction: operation not supported") { + t.Fatalf("snippet = %q", diagnostics.Snippet) + } + if strings.Contains(diagnostics.Snippet, "unrelated prior output") { + t.Fatalf("snippet contains unrelated output: %q", diagnostics.Snippet) + } +} + func runtimeStateEnvelope(t *testing.T, schema string) string { t.Helper() dir := t.TempDir() diff --git a/internal/dockerdeploy/testdata/network_policy_helper/main.go b/internal/dockerdeploy/testdata/network_policy_helper/main.go new file mode 100644 index 00000000..dfc80c4b --- /dev/null +++ b/internal/dockerdeploy/testdata/network_policy_helper/main.go @@ -0,0 +1,340 @@ +//go:build linux + +package main + +import ( + "bufio" + "context" + "encoding/binary" + "fmt" + "io" + "net" + "net/http" + "os" + "strconv" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +func main() { + if len(os.Args) < 2 { + fail("missing mode") + } + switch os.Args[1] { + case "peer": + peer() + case "dial": + dial(os.Args[2:]) + case "dns": + dns(os.Args[2:]) + case "workload": + workload(os.Args[2:]) + default: + fail("unknown mode %q", os.Args[1]) + } +} + +func dns(args []string) { + if len(args) != 3 { + fail("dns requires a name, expected result, and transport") + } + want, err := strconv.ParseBool(args[1]) + if err != nil { + fail("parse DNS expectation: %v", err) + } + checkDNS(args[0], want, args[2]) + fmt.Println("DNS_PASS") +} + +func checkDNS(name string, want bool, transport string) { + if transport != "udp" && transport != "tcp" { + fail("unsupported DNS transport %s", transport) + } + resolver := net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, _, address string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, transport, address) + }, + } + lookupContext, cancel := context.WithTimeout(context.Background(), 750*time.Millisecond) + _, lookupErr := resolver.LookupHost(lookupContext, name) + cancel() + if got := lookupErr == nil; got != want { + fail("DNS %s lookup %s succeeded=%t want=%t err=%v", transport, name, got, want, lookupErr) + } +} + +func dial(args []string) { + if len(args) != 2 { + fail("dial requires an address and expected result") + } + want, err := strconv.ParseBool(args[1]) + if err != nil { + fail("parse dial expectation: %v", err) + } + connection, dialErr := net.DialTimeout("tcp", args[0], 750*time.Millisecond) + if connection != nil { + _ = connection.Close() + } + if got := dialErr == nil; got != want { + fail("dial %s succeeded=%t want=%t err=%v", args[0], got, want, dialErr) + } + fmt.Println("DIAL_PASS") +} + +func peer() { + go serveDNS() + listener, err := net.Listen("tcp", ":9090") + if err != nil { + fail("listen: %v", err) + } + for { + connection, err := listener.Accept() + if err != nil { + fail("accept: %v", err) + } + _ = connection.Close() + } +} + +func serveDNS() { + go serveDNSUDP() + serveDNSTCP() +} + +func serveDNSUDP() { + connection, err := net.ListenPacket("udp", ":53") + if err != nil { + fail("listen for DNS: %v", err) + } + buffer := make([]byte, 4096) + for { + length, address, err := connection.ReadFrom(buffer) + if err != nil { + fail("read DNS query: %v", err) + } + response, ok := dnsResponse(buffer[:length]) + if !ok { + continue + } + if _, err := connection.WriteTo(response, address); err != nil { + fail("write DNS response: %v", err) + } + } +} + +func serveDNSTCP() { + listener, err := net.Listen("tcp", ":53") + if err != nil { + fail("listen for TCP DNS: %v", err) + } + for { + connection, err := listener.Accept() + if err != nil { + fail("accept TCP DNS: %v", err) + } + go answerDNSTCP(connection) + } +} + +func answerDNSTCP(connection net.Conn) { + defer connection.Close() + lengthBuffer := make([]byte, 2) + if _, err := io.ReadFull(connection, lengthBuffer); err != nil { + return + } + length := int(binary.BigEndian.Uint16(lengthBuffer)) + if length == 0 || length > 4096 { + return + } + query := make([]byte, length) + if _, err := io.ReadFull(connection, query); err != nil { + return + } + response, ok := dnsResponse(query) + if !ok { + return + } + framed := make([]byte, 2, len(response)+2) + binary.BigEndian.PutUint16(framed, uint16(len(response))) + framed = append(framed, response...) + for len(framed) != 0 { + written, err := connection.Write(framed) + if err != nil || written == 0 { + return + } + framed = framed[written:] + } +} + +func dnsResponse(query []byte) ([]byte, bool) { + if len(query) < 17 { + return nil, false + } + offset := 12 + for { + if offset >= len(query) { + return nil, false + } + length := int(query[offset]) + offset++ + if length == 0 { + break + } + if length > 63 || offset+length > len(query) { + return nil, false + } + offset += length + } + if offset+4 > len(query) { + return nil, false + } + questionEnd := offset + 4 + kind := binary.BigEndian.Uint16(query[offset : offset+2]) + var answer []byte + switch kind { + case 1: + answer = net.ParseIP("203.0.113.1").To4() + case 28: + answer = net.ParseIP("2001:db8::1").To16() + default: + return nil, false + } + response := append([]byte(nil), query[:questionEnd]...) + binary.BigEndian.PutUint16(response[2:4], 0x8180) + binary.BigEndian.PutUint16(response[6:8], 1) + response = append(response, 0xc0, 0x0c) + response = binary.BigEndian.AppendUint16(response, kind) + response = binary.BigEndian.AppendUint16(response, 1) + response = binary.BigEndian.AppendUint32(response, 0) + response = binary.BigEndian.AppendUint16(response, uint16(len(answer))) + response = append(response, answer...) + return response, true +} + +func workload(args []string) { + if len(args) != 14 { + fail("workload requires UID, six addresses, four expectations, serve mode, DNS name, and DNS expectation") + } + checkStatus(args[0]) + wantLocal, err := strconv.ParseBool(args[7]) + if err != nil { + fail("parse local expectation: %v", err) + } + wantPublic, err := strconv.ParseBool(args[8]) + if err != nil { + fail("parse public expectation: %v", err) + } + wantAmbiguous, err := strconv.ParseBool(args[9]) + if err != nil { + fail("parse ambiguous expectation: %v", err) + } + wantPublicException, err := strconv.ParseBool(args[10]) + if err != nil { + fail("parse public exception expectation: %v", err) + } + for _, item := range []struct { + address string + want bool + }{ + {args[1], wantLocal}, {ipv4MappedAddress(args[1]), wantLocal}, {args[2], wantLocal}, + {args[3], wantPublic}, {ipv4MappedAddress(args[3]), wantPublic}, {args[4], wantPublic}, + {args[5], wantAmbiguous}, {args[6], wantPublicException}, + } { + connection, dialErr := net.DialTimeout("tcp", item.address, 750*time.Millisecond) + if connection != nil { + _ = connection.Close() + } + if got := dialErr == nil; got != item.want { + fail("dial %s succeeded=%t want=%t err=%v", item.address, got, item.want, dialErr) + } + } + wantDNS, err := strconv.ParseBool(args[13]) + if err != nil { + fail("parse DNS expectation: %v", err) + } + for _, transport := range []string{"udp", "tcp"} { + checkDNS(args[12], wantDNS, transport) + } + fmt.Println("NETWORK_POLICY_PASS") + if args[11] != "serve" { + return + } + handler := http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprintln(response, "reploy-network-policy") + }) + declared, err := net.Listen("tcp", ":8080") + if err != nil { + fail("listen on declared endpoint: %v", err) + } + undeclared, err := net.Listen("tcp", ":8081") + if err != nil { + _ = declared.Close() + fail("listen on undeclared endpoint: %v", err) + } + go func() { + if serveErr := http.Serve(undeclared, handler); serveErr != nil { + fail("serve undeclared endpoint: %v", serveErr) + } + }() + if err := http.Serve(declared, handler); err != nil { + fail("serve: %v", err) + } +} + +func ipv4MappedAddress(address string) string { + host, port, err := net.SplitHostPort(address) + if err != nil { + fail("split IPv4 address %s: %v", address, err) + } + ip := net.ParseIP(host).To4() + if ip == nil { + fail("address %s is not IPv4", address) + } + return net.JoinHostPort("::ffff:"+ip.String(), port) +} + +func checkStatus(wantUID string) { + content, err := os.Open("/proc/self/status") + if err != nil { + fail("open status: %v", err) + } + defer content.Close() + seenUID := false + scanner := bufio.NewScanner(content) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "Uid:") { + fields := strings.Fields(line) + seenUID = len(fields) == 5 && fields[1] == wantUID && fields[2] == wantUID && fields[3] == wantUID && fields[4] == wantUID + } + for _, field := range []string{"CapInh:", "CapPrm:", "CapEff:", "CapBnd:", "CapAmb:"} { + if strings.HasPrefix(line, field) && strings.TrimLeft(strings.Fields(line)[1], "0") != "" { + fail("nonempty capability status: %s", line) + } + } + if strings.HasPrefix(line, "NoNewPrivs:") && strings.Fields(line)[1] != "1" { + fail("no-new-privileges missing: %s", line) + } + if strings.HasPrefix(line, "Seccomp:") && strings.Fields(line)[1] != "2" { + fail("seccomp missing: %s", line) + } + } + if err := scanner.Err(); err != nil || !seenUID { + fail("status identity mismatch for UID %s: %v", wantUID, err) + } + raw, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(3))) + if err == nil { + _ = unix.Close(raw) + fail("raw socket unexpectedly succeeded") + } +} + +func htons(value uint16) uint16 { return value<<8 | value>>8 } + +func fail(format string, args ...any) { + fmt.Fprintf(os.Stderr, "FAIL: "+format+"\n", args...) + os.Exit(1) +} diff --git a/internal/dockerdeploy/testdata/resolved_compose.yaml b/internal/dockerdeploy/testdata/resolved_compose.yaml index e31af6bf..448fcc1e 100644 --- a/internal/dockerdeploy/testdata/resolved_compose.yaml +++ b/internal/dockerdeploy/testdata/resolved_compose.yaml @@ -4,15 +4,20 @@ services: image: reploy/demo:staging pull_policy: never container_name: demo-staging-abcd - user: "501:20" + user: "0:0" group_add: [] cap_drop: - ALL + cap_add: + - NET_ADMIN + - SETGID + - SETPCAP + - SETUID security_opt: - no-new-privileges:true - seccomp=builtin entrypoint: [/reploy-probe] - command: [verify-exec, --, /opt/reploy/python/bin/demo, serve] + command: [sandbox-exec, --uid, "501", --gid, "20", --public, deny, --local, deny, --ambiguous, require-both, --inbound-tcp, "8080", --, /opt/reploy/python/bin/demo, serve] volumes: - type: bind source: /tmp/demo/conf diff --git a/internal/probe/main.go b/internal/probe/main.go index 58bfdf7a..a42ac029 100644 --- a/internal/probe/main.go +++ b/internal/probe/main.go @@ -18,6 +18,19 @@ func Main(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) in } return 0 } + if len(args) >= 1 && (args[0] == "sandbox-exec" || args[0] == "restricted-exec") { + installRules := args[0] == "sandbox-exec" + plan, err := parseSandboxExecPlanV1(args[1:], installRules) + if err != nil { + _, _ = fmt.Fprintf(stderr, "reploy-probe: %s: %v\n", args[0], err) + return 2 + } + if err := sandboxAndExecApplicationV1(plan); err != nil { + _, _ = fmt.Fprintf(stderr, "reploy-probe: %s: %v\n", args[0], err) + return 1 + } + return 0 + } return mainWithActions( args, stdin, stdout, stderr, waitForHoldSignal, copyFixedVolumeTree, @@ -76,7 +89,7 @@ func mainWithActions( return 0 } if len(args) != 0 { - _, _ = fmt.Fprintln(stderr, "reploy-probe accepts no arguments for one canonical stdin request, fixed hold mode, fixed copy-volume-tree mode, fixed install-local-account mode, or fixed verify-exec mode") + _, _ = fmt.Fprintln(stderr, "reploy-probe accepts no arguments for one canonical stdin request, fixed hold mode, fixed copy-volume-tree mode, fixed install-local-account mode, fixed verify-exec mode, or a sandbox-exec/restricted-exec contract") return 2 } content, err := io.ReadAll(stdin) diff --git a/internal/probe/network_firewall_linux.go b/internal/probe/network_firewall_linux.go new file mode 100644 index 00000000..d164d973 --- /dev/null +++ b/internal/probe/network_firewall_linux.go @@ -0,0 +1,382 @@ +//go:build linux + +package probe + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "net" + "os" + "slices" + "sort" + "strings" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "golang.org/x/sys/unix" +) + +type applicationNetworkPolicyV1 struct { + AllowPublic bool + AllowLocal bool + AllowAmbiguous bool + InboundTCP []uint16 +} + +const applicationResponseStateMaskV1 uint32 = expr.CtStateBitESTABLISHED + +const applicationResolverConfigurationV1 = "/etc/resolv.conf" +const applicationResolverConfigurationLimitV1 = 64 * 1024 + +var applicationRelatedICMPv4TypesV1 = []byte{3, 11, 12} +var applicationRelatedICMPv6TypesV1 = []byte{1, 2, 3, 4} + +// Keep these destination classes aligned with the IANA special-purpose +// registries. Public exceptions must precede their containing local ranges. +// Translation and tunneling prefixes are ambiguous because they can represent +// either public or local destinations; by default they require both grants. +// IPv4-mapped IPv6 socket addresses are not listed here: Linux emits those as +// IPv4 packets, so the embedded IPv4 destination receives its ordinary class. +// https://www.iana.org/assignments/iana-ipv4-special-registry/ +// https://www.iana.org/assignments/iana-ipv6-special-registry/ +var applicationLocalIPv4CIDRsV1 = []string{ + "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", + "169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.0.2.0/24", + "192.88.99.0/24", "192.168.0.0/16", "198.18.0.0/15", "198.51.100.0/24", "203.0.113.0/24", + "224.0.0.0/4", "240.0.0.0/4", +} + +var applicationPublicIPv4ExceptionsV1 = []string{ + "192.0.0.9/32", "192.0.0.10/32", +} + +var applicationAmbiguousIPv4CIDRsV1 = []string{} + +var applicationLocalIPv6CIDRsV1 = []string{ + "::/128", "::1/128", "100::/64", "100:0:0:1::/64", "2001::/23", + "2001:db8::/32", "3fff::/20", "5f00::/16", + "fc00::/7", "fe80::/10", "ff00::/8", +} + +var applicationPublicIPv6ExceptionsV1 = []string{ + "2001:1::1/128", "2001:1::2/128", "2001:1::3/128", "2001:3::/32", + "2001:4:112::/48", "2001:20::/28", "2001:30::/28", +} + +var applicationAmbiguousIPv6CIDRsV1 = []string{ + "64:ff9b::/96", "64:ff9b:1::/48", "2001::/32", "2002::/16", +} + +func installApplicationNetworkPolicyV1(policy applicationNetworkPolicyV1) error { + ports := append([]uint16(nil), policy.InboundTCP...) + slices.Sort(ports) + ports = slices.Compact(ports) + resolverCIDRs := []string{} + if policy.AllowPublic || policy.AllowLocal { + var err error + resolverCIDRs, err = readApplicationResolverCIDRsV1(applicationResolverConfigurationV1) + if err != nil { + return fmt.Errorf("read application DNS resolvers: %w", err) + } + if len(resolverCIDRs) == 0 { + return fmt.Errorf("read application DNS resolvers: no nameserver entries") + } + } + connection := &nftables.Conn{} + for _, family := range []struct { + tableFamily nftables.TableFamily + local []string + publicExceptions []string + ambiguous []string + addressSize uint32 + }{ + {nftables.TableFamilyIPv4, applicationLocalIPv4CIDRsV1, applicationPublicIPv4ExceptionsV1, applicationAmbiguousIPv4CIDRsV1, 4}, + {nftables.TableFamilyIPv6, applicationLocalIPv6CIDRsV1, applicationPublicIPv6ExceptionsV1, applicationAmbiguousIPv6CIDRsV1, 16}, + } { + if err := replaceApplicationNetworkTableV1(connection, family.tableFamily, family.local, family.publicExceptions, family.ambiguous, resolverCIDRs, family.addressSize, policy, ports); err != nil { + return err + } + } + if err := connection.Flush(); err != nil { + return fmt.Errorf("apply nftables transaction: %w", err) + } + return nil +} + +func replaceApplicationNetworkTableV1(connection *nftables.Conn, family nftables.TableFamily, local []string, publicExceptions []string, ambiguous []string, resolverCIDRs []string, addressSize uint32, policy applicationNetworkPolicyV1, ports []uint16) error { + tables, err := connection.ListTablesOfFamily(family) + if err != nil { + return fmt.Errorf("list nftables family %d: %w", family, err) + } + for _, table := range tables { + if table.Name == "reploy" { + connection.DelTable(table) + } + } + table := connection.AddTable(&nftables.Table{Family: family, Name: "reploy"}) + drop := nftables.ChainPolicyDrop + input := connection.AddChain(&nftables.Chain{ + Name: "input", Table: table, Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookInput, Priority: nftables.ChainPriorityFilter, Policy: &drop, + }) + output := connection.AddChain(&nftables.Chain{ + Name: "output", Table: table, Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookOutput, Priority: nftables.ChainPriorityFilter, Policy: &drop, + }) + forward := connection.AddChain(&nftables.Chain{ + Name: "forward", Table: table, Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookForward, Priority: nftables.ChainPriorityFilter, Policy: &drop, + }) + _ = forward + addInterfaceVerdictV1(connection, table, input, expr.MetaKeyIIFNAME, "lo", expr.VerdictAccept) + addEstablishedVerdictV1(connection, table, input, expr.VerdictAccept) + addRelatedICMPErrorVerdictsV1(connection, table, input, family) + if family == nftables.TableFamilyIPv6 { + addIPv6NeighborDiscoveryRulesV1(connection, table, input) + } + for _, port := range ports { + addTCPPortVerdictV1(connection, table, input, port, expr.VerdictAccept) + } + + if policy.AllowPublic || policy.AllowLocal { + if err := addDNSResolverVerdictsV1(connection, table, output, resolverCIDRs, addressSize); err != nil { + return err + } + } + if family == nftables.TableFamilyIPv4 { + // Docker's embedded resolver is reached through container loopback. + // Permit only its engine-owned DNS service before accepting ordinary + // application loopback traffic. + if err := addCIDRVerdictV1(connection, table, output, "127.0.0.11/32", addressSize, expr.VerdictDrop); err != nil { + return err + } + } + addInterfaceVerdictV1(connection, table, output, expr.MetaKeyOIFNAME, "lo", expr.VerdictAccept) + addEstablishedVerdictV1(connection, table, output, expr.VerdictAccept) + if family == nftables.TableFamilyIPv6 { + addIPv6NeighborDiscoveryRulesV1(connection, table, output) + } + ambiguousVerdict := expr.VerdictDrop + if policy.AllowAmbiguous { + ambiguousVerdict = expr.VerdictAccept + } + if err := addCIDRVerdictsV1(connection, table, output, ambiguous, addressSize, ambiguousVerdict); err != nil { + return err + } + publicVerdict := expr.VerdictDrop + if policy.AllowPublic { + publicVerdict = expr.VerdictAccept + } + if err := addCIDRVerdictsV1(connection, table, output, publicExceptions, addressSize, publicVerdict); err != nil { + return err + } + localVerdict := expr.VerdictDrop + if policy.AllowLocal { + localVerdict = expr.VerdictAccept + } + if err := addCIDRVerdictsV1(connection, table, output, local, addressSize, localVerdict); err != nil { + return err + } + if policy.AllowPublic { + connection.AddRule(&nftables.Rule{Table: table, Chain: output, Exprs: []expr.Any{&expr.Verdict{Kind: expr.VerdictAccept}}}) + } + return nil +} + +func readApplicationResolverCIDRsV1(path string) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + content, err := io.ReadAll(io.LimitReader(file, applicationResolverConfigurationLimitV1+1)) + if err != nil { + return nil, err + } + if len(content) > applicationResolverConfigurationLimitV1 { + return nil, fmt.Errorf("resolver configuration exceeds %d bytes", applicationResolverConfigurationLimitV1) + } + result := map[string]struct{}{} + scanner := bufio.NewScanner(strings.NewReader(string(content))) + for scanner.Scan() { + line := scanner.Text() + if offset := strings.IndexAny(line, "#;"); offset >= 0 { + line = line[:offset] + } + fields := strings.Fields(line) + if len(fields) == 0 || fields[0] != "nameserver" { + continue + } + if len(fields) < 2 { + return nil, fmt.Errorf("nameserver entry has no address") + } + address := fields[1] + if host, _, found := strings.Cut(address, "%"); found { + address = host + } + ip := net.ParseIP(address) + if ip == nil { + return nil, fmt.Errorf("nameserver address %q is not an IP address", fields[1]) + } + bits := 128 + if ipv4 := ip.To4(); ipv4 != nil { + ip = ipv4 + bits = 32 + } + result[fmt.Sprintf("%s/%d", ip.String(), bits)] = struct{}{} + } + if err := scanner.Err(); err != nil { + return nil, err + } + resolverCIDRs := make([]string, 0, len(result)) + for cidr := range result { + resolverCIDRs = append(resolverCIDRs, cidr) + } + sort.Strings(resolverCIDRs) + return resolverCIDRs, nil +} + +func addDNSResolverVerdictsV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, resolverCIDRs []string, addressSize uint32) error { + for _, cidr := range resolverCIDRs { + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("parse application DNS resolver %q: %w", cidr, err) + } + if (addressSize == 4 && ip.To4() == nil) || (addressSize == 16 && ip.To4() != nil) { + continue + } + for _, protocol := range []byte{unix.IPPROTO_UDP, unix.IPPROTO_TCP} { + if err := addTransportPortCIDRVerdictV1(connection, table, chain, cidr, addressSize, protocol, 53, expr.VerdictAccept); err != nil { + return err + } + } + } + return nil +} + +func addTransportPortCIDRVerdictV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, cidr string, addressSize uint32, protocol byte, port uint16, verdict expr.VerdictKind) error { + ip, network, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("parse application network prefix %q: %w", cidr, err) + } + address := ip.To4() + if addressSize == 16 { + address = ip.To16() + } + portData := make([]byte, 2) + binary.BigEndian.PutUint16(portData, port) + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + // Docker DNATs embedded-DNS traffic from port 53 to an internal + // high-numbered socket before the filter hook. Match the original + // conntrack tuple so the narrow resolver grant survives that rewrite. + &expr.Ct{Register: 1, Key: expr.CtKeyDST, Direction: 0}, + &expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: addressSize, Mask: network.Mask, Xor: make([]byte, addressSize)}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: address}, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{protocol}}, + &expr.Ct{Register: 1, Key: expr.CtKeyPROTODST, Direction: 0}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: portData}, + &expr.Verdict{Kind: verdict}, + }}) + return nil +} + +func addCIDRVerdictsV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, cidrs []string, addressSize uint32, verdict expr.VerdictKind) error { + for _, cidr := range cidrs { + if err := addCIDRVerdictV1(connection, table, chain, cidr, addressSize, verdict); err != nil { + return err + } + } + return nil +} + +func addRelatedICMPErrorVerdictsV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, family nftables.TableFamily) { + protocol := byte(unix.IPPROTO_ICMP) + types := applicationRelatedICMPv4TypesV1 + if family == nftables.TableFamilyIPv6 { + protocol = unix.IPPROTO_ICMPV6 + types = applicationRelatedICMPv6TypesV1 + } + for _, kind := range types { + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{protocol}}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{kind}}, + &expr.Ct{Register: 1, Key: expr.CtKeySTATE}, + &expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, + Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitRELATED), + Xor: binaryutil.NativeEndian.PutUint32(0)}, + &expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: []byte{0, 0, 0, 0}}, + &expr.Verdict{Kind: expr.VerdictAccept}, + }}) + } +} + +func addInterfaceVerdictV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, key expr.MetaKey, name string, verdict expr.VerdictKind) { + data := make([]byte, 16) + copy(data, name+"\x00") + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Meta{Key: key, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: data}, + &expr.Verdict{Kind: verdict}, + }}) +} + +func addEstablishedVerdictV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, verdict expr.VerdictKind) { + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Ct{Register: 1, Key: expr.CtKeySTATE}, + &expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, + Mask: binaryutil.NativeEndian.PutUint32(applicationResponseStateMaskV1), + Xor: binaryutil.NativeEndian.PutUint32(0)}, + &expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: []byte{0, 0, 0, 0}}, + &expr.Verdict{Kind: verdict}, + }}) +} + +func addTCPPortVerdictV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, port uint16, verdict expr.VerdictKind) { + data := make([]byte, 2) + binary.BigEndian.PutUint16(data, port) + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: data}, + &expr.Verdict{Kind: verdict}, + }}) +} + +func addIPv6NeighborDiscoveryRulesV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain) { + for _, kind := range []byte{133, 134, 135, 136} { + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_ICMPV6}}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{kind}}, + &expr.Verdict{Kind: expr.VerdictAccept}, + }}) + } +} + +func addCIDRVerdictV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, cidr string, addressSize uint32, verdict expr.VerdictKind) error { + ip, network, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("parse application network prefix %q: %w", cidr, err) + } + offset := uint32(16) + address := ip.To4() + if addressSize == 16 { + offset = 24 + address = ip.To16() + } + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: offset, Len: addressSize}, + &expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: addressSize, Mask: network.Mask, Xor: make([]byte, addressSize)}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: address}, + &expr.Verdict{Kind: verdict}, + }}) + return nil +} diff --git a/internal/probe/network_firewall_linux_test.go b/internal/probe/network_firewall_linux_test.go new file mode 100644 index 00000000..b9d460e0 --- /dev/null +++ b/internal/probe/network_firewall_linux_test.go @@ -0,0 +1,102 @@ +//go:build linux + +package probe + +import ( + "net" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/google/nftables/expr" +) + +func TestReadApplicationResolverCIDRsV1(t *testing.T) { + path := filepath.Join(t.TempDir(), "resolv.conf") + content := "search example.test\n" + + "nameserver 127.0.0.11 # Docker embedded DNS\n" + + "nameserver 8.8.8.8\n" + + "nameserver 2001:4860:4860::8888%eth0\n" + + "nameserver 8.8.8.8 ; duplicate\n" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + got, err := readApplicationResolverCIDRsV1(path) + if err != nil { + t.Fatal(err) + } + want := []string{"127.0.0.11/32", "2001:4860:4860::8888/128", "8.8.8.8/32"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("resolver CIDRs = %#v, want %#v", got, want) + } +} + +func TestReadApplicationResolverCIDRsV1RejectsInvalidNameserver(t *testing.T) { + for _, content := range []string{"nameserver\n", "nameserver resolver.example\n"} { + t.Run(strings.TrimSpace(content), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "resolv.conf") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + if _, err := readApplicationResolverCIDRsV1(path); err == nil { + t.Fatalf("resolver content %q unexpectedly passed", content) + } + }) + } +} + +func TestApplicationResponseTrafficAcceptsOnlyEstablishedConnections(t *testing.T) { + if applicationResponseStateMaskV1 != expr.CtStateBitESTABLISHED || applicationResponseStateMaskV1&expr.CtStateBitRELATED != 0 { + t.Fatalf("response conntrack mask = %#x", applicationResponseStateMaskV1) + } +} + +func TestApplicationRelatedTrafficIsLimitedToICMPNetworkErrors(t *testing.T) { + if !reflect.DeepEqual(applicationRelatedICMPv4TypesV1, []byte{3, 11, 12}) { + t.Fatalf("IPv4 related ICMP types = %#v", applicationRelatedICMPv4TypesV1) + } + if !reflect.DeepEqual(applicationRelatedICMPv6TypesV1, []byte{1, 2, 3, 4}) { + t.Fatalf("IPv6 related ICMP types = %#v", applicationRelatedICMPv6TypesV1) + } +} + +func TestApplicationNetworkCIDRsSeparateLocalAmbiguousAndPublicExceptions(t *testing.T) { + for _, address := range []string{"192.88.99.2", "100:0:0:1::1", "2001:2::1", "3fff::1", "5f00::1"} { + if !applicationCIDRsContainAddressV1(applicationLocalIPv4CIDRsV1, applicationLocalIPv6CIDRsV1, address) { + t.Fatalf("special destination %s is not classified as local", address) + } + } + for _, address := range []string{"64:ff9b::a00:1", "64:ff9b:1::1", "2001::1", "2002:a00:1::1"} { + if !applicationCIDRsContainAddressV1(applicationAmbiguousIPv4CIDRsV1, applicationAmbiguousIPv6CIDRsV1, address) { + t.Fatalf("translation or tunneling destination %s is not classified as ambiguous", address) + } + } + if applicationCIDRsContainAddressV1(applicationAmbiguousIPv4CIDRsV1, applicationAmbiguousIPv6CIDRsV1, "::ffff:10.0.0.1") { + t.Fatal("IPv4-mapped IPv6 destination must use its embedded IPv4 class") + } + for _, address := range []string{"192.0.0.9", "192.0.0.10", "2001:20::1", "2001:30::1"} { + if !applicationCIDRsContainAddressV1(applicationPublicIPv4ExceptionsV1, applicationPublicIPv6ExceptionsV1, address) { + t.Fatalf("globally reachable exception %s is not classified as public", address) + } + } +} + +func applicationCIDRsContainAddressV1(ipv4 []string, ipv6 []string, address string) bool { + ip := net.ParseIP(address) + cidrs := ipv6 + if !strings.Contains(address, ":") { + cidrs = ipv4 + } + for _, cidr := range cidrs { + _, network, err := net.ParseCIDR(cidr) + if err != nil { + panic(err) + } + if network.Contains(ip) { + return true + } + } + return false +} diff --git a/internal/probe/sandbox_exec.go b/internal/probe/sandbox_exec.go new file mode 100644 index 00000000..f5409c67 --- /dev/null +++ b/internal/probe/sandbox_exec.go @@ -0,0 +1,139 @@ +package probe + +import ( + "flag" + "fmt" + "strconv" + "strings" +) + +type sandboxExecPlanV1 struct { + UID int + GID int + Groups []int + AllowPublic bool + AllowLocal bool + AllowAmbiguous bool + InboundTCP []uint16 + InstallRules bool + Argv []string +} + +func parseSandboxExecPlanV1(args []string, installRules bool) (sandboxExecPlanV1, error) { + separator := -1 + for index, argument := range args { + if argument == "--" { + separator = index + break + } + } + if separator < 0 || separator == len(args)-1 { + return sandboxExecPlanV1{}, fmt.Errorf("requires -- followed by an absolute application command") + } + set := flag.NewFlagSet("sandbox-exec", flag.ContinueOnError) + set.SetOutput(new(strings.Builder)) + uid := set.Int("uid", -1, "application UID") + gid := set.Int("gid", -1, "application GID") + groups := set.String("groups", "", "comma-separated supplementary GIDs") + public := set.String("public", "", "public network policy") + local := set.String("local", "", "local network policy") + ambiguous := set.String("ambiguous", "", "ambiguous destination policy") + inbound := set.String("inbound-tcp", "", "comma-separated inbound TCP ports") + if err := set.Parse(args[:separator]); err != nil { + return sandboxExecPlanV1{}, err + } + if len(set.Args()) != 0 { + return sandboxExecPlanV1{}, fmt.Errorf("unexpected positional sandbox arguments") + } + argv := args[separator+1:] + if *uid < 0 || *gid < 0 { + return sandboxExecPlanV1{}, fmt.Errorf("requires non-negative --uid and --gid") + } + parsedGroups, err := parseDecimalListV1(*groups, 0, int(^uint(0)>>1)) + if err != nil { + return sandboxExecPlanV1{}, fmt.Errorf("parse --groups: %w", err) + } + if !installRules && *inbound != "" { + return sandboxExecPlanV1{}, fmt.Errorf("--inbound-tcp is not accepted by restricted-exec") + } + parsedInbound, err := parseDecimalListV1(*inbound, 1, 65535) + if err != nil { + return sandboxExecPlanV1{}, fmt.Errorf("parse --inbound-tcp: %w", err) + } + allowPublic, err := parseNetworkAccessV1("--public", *public, installRules) + if err != nil { + return sandboxExecPlanV1{}, err + } + allowLocal, err := parseNetworkAccessV1("--local", *local, installRules) + if err != nil { + return sandboxExecPlanV1{}, err + } + allowAmbiguous, err := parseAmbiguousNetworkAccessV1(*ambiguous, allowPublic, allowLocal, installRules) + if err != nil { + return sandboxExecPlanV1{}, err + } + ports := make([]uint16, len(parsedInbound)) + for index, value := range parsedInbound { + ports[index] = uint16(value) + } + return sandboxExecPlanV1{ + UID: *uid, GID: *gid, Groups: parsedGroups, + AllowPublic: allowPublic, AllowLocal: allowLocal, AllowAmbiguous: allowAmbiguous, + InboundTCP: ports, InstallRules: installRules, + Argv: append([]string(nil), argv...), + }, nil +} + +func parseAmbiguousNetworkAccessV1(value string, allowPublic bool, allowLocal bool, required bool) (bool, error) { + if !required { + if value != "" { + return false, fmt.Errorf("--ambiguous is not accepted by restricted-exec") + } + return false, nil + } + switch value { + case "require-both": + return allowPublic && allowLocal, nil + case "allow": + return true, nil + default: + return false, fmt.Errorf("--ambiguous must be require-both or allow") + } +} + +func parseNetworkAccessV1(name string, value string, required bool) (bool, error) { + if !required { + if value != "" { + return false, fmt.Errorf("%s is not accepted by restricted-exec", name) + } + return false, nil + } + switch value { + case "allow": + return true, nil + case "deny": + return false, nil + default: + return false, fmt.Errorf("%s must be allow or deny", name) + } +} + +func parseDecimalListV1(value string, minimum int, maximum int) ([]int, error) { + if value == "" { + return []int{}, nil + } + result := []int{} + previous := -1 + for _, item := range strings.Split(value, ",") { + parsed, err := strconv.Atoi(item) + if err != nil || parsed < minimum || parsed > maximum { + return nil, fmt.Errorf("%q is outside %d..%d", item, minimum, maximum) + } + if parsed <= previous { + return nil, fmt.Errorf("values must be unique and sorted") + } + result = append(result, parsed) + previous = parsed + } + return result, nil +} diff --git a/internal/probe/sandbox_exec_linux.go b/internal/probe/sandbox_exec_linux.go new file mode 100644 index 00000000..dde1a156 --- /dev/null +++ b/internal/probe/sandbox_exec_linux.go @@ -0,0 +1,92 @@ +//go:build linux + +package probe + +import ( + "fmt" + "os" + "runtime" + "strconv" + "strings" + + "golang.org/x/sys/unix" +) + +const ( + secureNoRootV1 = 1 << 0 + secureNoRootLockedV1 = 1 << 1 + secureNoSetUIDFixupV1 = 1 << 2 + secureNoSetUIDFixupLockedV1 = 1 << 3 + secureKeepCapsLockedV1 = 1 << 5 + secureNoAmbientRaiseV1 = 1 << 6 + secureNoAmbientRaiseLockedV1 = 1 << 7 +) + +func sandboxAndExecApplicationV1(plan sandboxExecPlanV1) error { + // Linux credentials, securebits, and capability bounding sets are + // thread-scoped. Keep the trusted transition and the final exec on exactly + // one OS thread so the Go scheduler cannot move verification onto a thread + // that still carries the container's setup authority. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + if os.Geteuid() != 0 { + return fmt.Errorf("trusted sandbox setup must begin as container root") + } + if plan.InstallRules { + if err := installApplicationNetworkPolicyV1(applicationNetworkPolicyV1{ + AllowPublic: plan.AllowPublic, + AllowLocal: plan.AllowLocal, + AllowAmbiguous: plan.AllowAmbiguous, + InboundTCP: plan.InboundTCP, + }); err != nil { + return fmt.Errorf("install application network policy: %w", err) + } + } + if err := dropApplicationAuthorityV1(plan.UID, plan.GID, plan.Groups); err != nil { + return err + } + return verifyAndExecApplication(plan.Argv, readApplicationKernelStatus, execApplication) +} + +func dropApplicationAuthorityV1(uid int, gid int, groups []int) error { + lastContent, err := os.ReadFile("/proc/sys/kernel/cap_last_cap") + if err != nil { + return fmt.Errorf("read last Linux capability: %w", err) + } + lastCapability, err := strconv.Atoi(strings.TrimSpace(string(lastContent))) + if err != nil { + return fmt.Errorf("parse last Linux capability: %w", err) + } + for capability := 0; capability <= lastCapability; capability++ { + if err := unix.Prctl(unix.PR_CAPBSET_DROP, uintptr(capability), 0, 0, 0); err != nil { + return fmt.Errorf("drop capability %d from bounding set: %w", capability, err) + } + } + secureBits := uintptr( + secureNoRootV1 | secureNoRootLockedV1 | + secureNoSetUIDFixupV1 | secureNoSetUIDFixupLockedV1 | + secureKeepCapsLockedV1 | + secureNoAmbientRaiseV1 | secureNoAmbientRaiseLockedV1, + ) + if err := unix.Prctl(unix.PR_SET_SECUREBITS, secureBits, 0, 0, 0); err != nil { + return fmt.Errorf("lock application securebits: %w", err) + } + if err := unix.Setgroups(groups); err != nil { + return fmt.Errorf("set application supplementary groups: %w", err) + } + if err := unix.Setresgid(gid, gid, gid); err != nil { + return fmt.Errorf("set application GID %d: %w", gid, err) + } + if err := unix.Setresuid(uid, uid, uid); err != nil { + return fmt.Errorf("set application UID %d: %w", uid, err) + } + if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return fmt.Errorf("set application no-new-privileges: %w", err) + } + header := unix.CapUserHeader{Version: unix.LINUX_CAPABILITY_VERSION_3, Pid: 0} + data := [2]unix.CapUserData{} + if err := unix.Capset(&header, &data[0]); err != nil { + return fmt.Errorf("clear application capabilities: %w", err) + } + return nil +} diff --git a/internal/probe/sandbox_exec_other.go b/internal/probe/sandbox_exec_other.go new file mode 100644 index 00000000..5643acec --- /dev/null +++ b/internal/probe/sandbox_exec_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package probe + +import "fmt" + +func sandboxAndExecApplicationV1(sandboxExecPlanV1) error { + return fmt.Errorf("application sandbox setup is supported only in Linux containers") +} diff --git a/internal/probe/sandbox_exec_test.go b/internal/probe/sandbox_exec_test.go new file mode 100644 index 00000000..eabced64 --- /dev/null +++ b/internal/probe/sandbox_exec_test.go @@ -0,0 +1,91 @@ +package probe + +import ( + "reflect" + "strings" + "testing" +) + +func TestParseSandboxExecPlanV1PreservesPolicyIdentityAndArgv(t *testing.T) { + plan, err := parseSandboxExecPlanV1([]string{ + "--uid", "501", "--gid", "20", "--groups", "33,44", + "--public", "allow", "--local", "deny", "--ambiguous", "allow", "--inbound-tcp", "8080,8443", + "--", "/opt/app", "", "$(not-shell)", + }, true) + if err != nil { + t.Fatal(err) + } + if plan.UID != 501 || plan.GID != 20 || !reflect.DeepEqual(plan.Groups, []int{33, 44}) || + !plan.AllowPublic || plan.AllowLocal || !plan.AllowAmbiguous || !reflect.DeepEqual(plan.InboundTCP, []uint16{8080, 8443}) || + !reflect.DeepEqual(plan.Argv, []string{"/opt/app", "", "$(not-shell)"}) { + t.Fatalf("plan = %#v", plan) + } +} + +func TestParseSandboxExecPlanV1RequireBothAllowsAmbiguousOnlyWithBothGrants(t *testing.T) { + for _, test := range []struct { + name string + public string + local string + wantAllow bool + }{ + {name: "neither", public: "deny", local: "deny"}, + {name: "public only", public: "allow", local: "deny"}, + {name: "local only", public: "deny", local: "allow"}, + {name: "both", public: "allow", local: "allow", wantAllow: true}, + } { + t.Run(test.name, func(t *testing.T) { + plan, err := parseSandboxExecPlanV1([]string{ + "--uid", "501", "--gid", "20", + "--public", test.public, "--local", test.local, "--ambiguous", "require-both", + "--", "/opt/app", + }, true) + if err != nil { + t.Fatal(err) + } + if plan.AllowAmbiguous != test.wantAllow { + t.Fatalf("AllowAmbiguous = %t, want %t", plan.AllowAmbiguous, test.wantAllow) + } + }) + } +} + +func TestParseSandboxExecPlanV1FailsClosed(t *testing.T) { + base := []string{"--uid", "501", "--gid", "20", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/app"} + for _, test := range []struct { + name string + args []string + want string + }{ + {name: "separator", args: base[:len(base)-2], want: "requires --"}, + {name: "identity", args: []string{"--uid", "-1", "--gid", "20", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/app"}, want: "non-negative"}, + {name: "policy", args: []string{"--uid", "1", "--gid", "2", "--public", "yes", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/app"}, want: "--public"}, + {name: "ambiguous policy", args: []string{"--uid", "1", "--gid", "2", "--public", "deny", "--local", "deny", "--ambiguous", "yes", "--", "/opt/app"}, want: "--ambiguous"}, + {name: "groups", args: []string{"--uid", "1", "--gid", "2", "--groups", "44,33", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/app"}, want: "unique and sorted"}, + {name: "port", args: []string{"--uid", "1", "--gid", "2", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--inbound-tcp", "0", "--", "/opt/app"}, want: "outside 1..65535"}, + } { + t.Run(test.name, func(t *testing.T) { + _, err := parseSandboxExecPlanV1(test.args, true) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %v, want %q", err, test.want) + } + }) + } +} + +func TestRestrictedExecDoesNotAcceptNetworkSetupArguments(t *testing.T) { + for _, argument := range []string{"--public", "--local", "--ambiguous", "--inbound-tcp"} { + t.Run(argument, func(t *testing.T) { + value := "allow" + if argument == "--inbound-tcp" { + value = "8080" + } + _, err := parseSandboxExecPlanV1([]string{ + "--uid", "501", "--gid", "20", argument, value, "--", "/bin/true", + }, false) + if err == nil || !strings.Contains(err.Error(), argument) { + t.Fatalf("restricted exec network argument error = %v", err) + } + }) + } +} diff --git a/internal/probe/startup_verifier.go b/internal/probe/startup_verifier.go index 2f3f069e..a3651a9b 100644 --- a/internal/probe/startup_verifier.go +++ b/internal/probe/startup_verifier.go @@ -16,9 +16,11 @@ var requiredApplicationKernelStatusV1 = []struct { want string hex bool }{ + {name: "CapInh", want: "0", hex: true}, {name: "CapBnd", want: "0", hex: true}, {name: "CapEff", want: "0", hex: true}, {name: "CapPrm", want: "0", hex: true}, + {name: "CapAmb", want: "0", hex: true}, {name: "NoNewPrivs", want: "1"}, {name: "Seccomp", want: "2"}, } diff --git a/internal/probe/startup_verifier_test.go b/internal/probe/startup_verifier_test.go index 9c6a169e..47526cd6 100644 --- a/internal/probe/startup_verifier_test.go +++ b/internal/probe/startup_verifier_test.go @@ -9,9 +9,11 @@ import ( ) const validApplicationKernelStatus = `Name: reploy-probe +CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000000000000000 +CapAmb: 0000000000000000 NoNewPrivs: 1 Seccomp: 2 ` @@ -36,6 +38,8 @@ func TestVerifyApplicationKernelStatusFailsClosed(t *testing.T) { {name: "effective capabilities", content: strings.ReplaceAll(validApplicationKernelStatus, "CapEff: 0000000000000000", "CapEff: 0000000000000001"), want: "CapEff is 0000000000000001"}, {name: "permitted capabilities", content: strings.ReplaceAll(validApplicationKernelStatus, "CapPrm: 0000000000000000", "CapPrm: 0000000000000400"), want: "CapPrm is 0000000000000400"}, {name: "bounding capabilities", content: strings.ReplaceAll(validApplicationKernelStatus, "CapBnd: 0000000000000000", "CapBnd: 000001ffffffffff"), want: "CapBnd is 000001ffffffffff"}, + {name: "inheritable capabilities", content: strings.ReplaceAll(validApplicationKernelStatus, "CapInh: 0000000000000000", "CapInh: 0000000000000001"), want: "CapInh is 0000000000000001"}, + {name: "ambient capabilities", content: strings.ReplaceAll(validApplicationKernelStatus, "CapAmb: 0000000000000000", "CapAmb: 0000000000000001"), want: "CapAmb is 0000000000000001"}, {name: "invalid capability", content: strings.ReplaceAll(validApplicationKernelStatus, "CapBnd: 0000000000000000", "CapBnd: not-hex"), want: "not hexadecimal"}, } for _, test := range tests {