diff --git a/.changes/unreleased/+coarse-application-network-policy.yaml b/.changes/unreleased/+coarse-application-network-policy.yaml new file mode 100644 index 00000000..08b3a689 --- /dev/null +++ b/.changes/unreleased/+coarse-application-network-policy.yaml @@ -0,0 +1,2 @@ +kind: Security +body: 'Deny public and local application networking by default, conservatively require both grants for ambiguous translation and tunneling ranges, select host-derived or public DNS from those grants, and enforce the policy for workloads, commands, shells, and lifecycle actions while preserving declared inbound endpoints.' diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a0813e14..20e162df 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -101,6 +101,14 @@ jobs: go test -timeout 10m ./internal/dockerdeploy -run '^(TestPrivateWorkloadEnvironmentDockerIntegrationMasksFilesAndInjectsValues|TestPrivateRuntimeMasksDockerIntegrationProtectTransientContainer|TestPrivateWorkloadEnvironmentRealDockerIsolation)$' + - name: Run application network-policy integration tests + if: matrix.name == 'Linux amd64' + env: + REPLOY_DOCKER_INTEGRATION: "1" + run: >- + go test -timeout 10m ./internal/dockerdeploy + -run '^TestApplicationNetworkPolicyDockerIntegration$' + - name: Run CLI runtime integration if: ${{ matrix.persistent_install == false }} env: diff --git a/docs/APT_PROVIDER_DETAIL_DESIGN.md b/docs/APT_PROVIDER_DETAIL_DESIGN.md index a542b45a..5f0a8263 100644 --- a/docs/APT_PROVIDER_DETAIL_DESIGN.md +++ b/docs/APT_PROVIDER_DETAIL_DESIGN.md @@ -2127,6 +2127,8 @@ intersections with reserved system, Reploy, or provider paths are invalid. ```go type RuntimePolicyV1 struct { Schema string + StartupVerifier ApplicationStartupVerifierV1 + Network RuntimeNetwork ProtectedPaths []ProtectedPathV1 Plans []RuntimePlanV1 } @@ -2139,6 +2141,7 @@ type ProtectedPathV1 struct { type RuntimePlanV1 struct { ID string + InboundTCP []string Mounts []RuntimeMountV1 Executables []QualifiedOutput } @@ -2150,11 +2153,20 @@ type RuntimeMountV1 struct { } ``` -`Schema` is `runtime-policy-v1`. Protected paths are unique normalized absolute -paths sorted by path. Protected kind is `reploy-root`, +`Schema` is `runtime-policy-v1`. `StartupVerifier` identifies the exact trusted +application setup recipe embedded in the runtime image. `Network` records the +effective independent `public` and `local` access values, each `allow` or +`deny`, plus the `ambiguous` translation/tunneling policy, either `require-both` +or `allow`. IPv4-mapped IPv6 sockets follow their embedded IPv4 class because +Linux emits them as IPv4 packets. Protected paths are unique normalized +absolute paths sorted by path. +Protected kind is `reploy-root`, `provider-root`, `provider-leaf`, or `executable-path`; owner is the stable node or qualified-output identity. Plans use the stable command/workload/probe ID and -are sorted by ID. Mounts are sorted by destination; `SourceKind` records only +are sorted by ID. `InboundTCP` records sorted unique canonical decimal port +strings for that exact container shape: declared ports belong only to the +workload plan, while shell and transient command plans use an empty array. Mounts are sorted by +destination; `SourceKind` records only the resolved kind (`file`, `directory`, or `generated`) and never a host source path. Executables are unique and sorted by qualified identity. @@ -2169,10 +2181,13 @@ build identity input. Each one-shot command and `reploy shell` mounts a fresh 64 MiB tmpfs at `/mnt/reploy-home` for `HOME` and `TMPDIR`. The mount is mode `0700`, owned by the selected runtime UID/GID, and removed with the transient container. Docker -starts the resolved executable directly under that final numeric identity; no -root bootstrap helper is involved. Explicit interruption cleanup force-removes -the transient container. Workload containers use the same bounded tmpfs-home -policy at `/mnt/reploy-home`. +starts only the trusted Reploy helper as container root with the minimal setup +capabilities. The helper installs any required application-network rules, +assumes the selected numeric identity, irreversibly drops its setup authority, +verifies the final kernel state, and executes the resolved application argv. +Explicit interruption cleanup force-removes the transient container. Workload +containers use the same setup contract and bounded tmpfs-home policy at +`/mnt/reploy-home`. The policy digest is `canonical.Sum("runtime-policy", "runtime-policy-v1", policy)`. It is recorded diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index 2d1f6bae..35db6179 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -29,17 +29,6 @@ This file is the day-to-day queue for design and implementation gaps. stopping, and whether the next step needs user review, approval, input, or no user action. -## Now - -- [ ] `P1` Implement the initial coarse application-network policy. Preserve - independent public and local policy intent with both denied by default, - apply it consistently to workloads, commands, shells, and lifecycle - commands, and use only proven isolation and endpoint primitives from the - active runtime backend. Permit exact declared inbound endpoints without - granting general local access. Fail closed when a backend cannot realize - a requested combination, and do not represent this slice as destination-, - domain-, or packet-level filtering. - ## Pre-release - [ ] `P1` Accept APT install transaction records with the optional trailing @@ -203,7 +192,13 @@ This file is the day-to-day queue for design and implementation gaps. cases justify its scope, precedence, user/system ownership, validation, and portability. Initial potential use case: overriding the otherwise fixed host-owned limits for controlled-session endpoint streams and - connection-open rates. + connection-open rates. Also record host-owned DNS resolver configuration + used to provide DNS under the coarse application network grants. The + default local-capable path should use the host's configured resolver so + VPN and split-DNS behavior remains available; the public-only path should + use the built-in Google Public DNS profile (`8.8.8.8`, `8.8.4.4`). Allow + host configuration to override either choice. Resolver selection is + machine policy, not blueprint policy. - [ ] `P2` Design and implement a Reploy userland L3 policy gateway. Keep this separate from the initial public/local kill switches and controlled @@ -217,7 +212,9 @@ This file is the day-to-day queue for design and implementation gaps. initial controlled-session host-loopback endpoint publication so only the lease-owned Host Reploy operation can reach the recorded application; include multi-user-host tests proving unrelated local processes cannot - bypass the session endpoint grant. + bypass the session endpoint grant. Replace the temporary, discouraged + `environment.runtime.network.ambiguous: allow` escape hatch with precise + translated-destination policy and deprecate that coarse override. - [ ] `P2` Evaluate and prioritize the Dingo development-environment gaps. Use `docs/DINGO_GAPS.md` as the needs and evidence record for portable diff --git a/docs/BLUEPRINT_ENVIRONMENT_MODEL.md b/docs/BLUEPRINT_ENVIRONMENT_MODEL.md index 9c9b5bfd..266709b1 100644 --- a/docs/BLUEPRINT_ENVIRONMENT_MODEL.md +++ b/docs/BLUEPRINT_ENVIRONMENT_MODEL.md @@ -92,6 +92,10 @@ environment: allow_concurrent: auto # App-command and shell overlap policy. runtime: user: reploy # Container-local account name; defaults to reploy. + network: + public: deny # Public Internet access; defaults to deny. + local: deny # Local/private network access; defaults to deny. + ambiguous: require-both # Translation/tunnel ranges require both grants. terminal: {} # Terminal/color integration. install: {} # Installation target, identity, and success output. mounts: {} # Runtime filesystem contracts. @@ -106,7 +110,11 @@ one to 32 bytes, beginning with a lowercase ASCII letter or underscore, followed only by lowercase ASCII letters, digits, underscores, or hyphens. `root` is reserved and cannot be selected through this field. If the base image already defines the same account name with a different numeric ID, runtime-layer -construction fails rather than rewriting that unrelated account. Backend-specific +construction fails rather than rewriting that unrelated account. +`runtime.network.public` and +`runtime.network.local` independently accept `allow` or `deny`; both default to +`deny`. `runtime.network.ambiguous` accepts `require-both` or the temporary, +discouraged `allow` escape hatch; it defaults to `require-both`. Backend-specific runtime choices remain under the top-level `docker` node. ## Internal Execution Phases @@ -1382,6 +1390,67 @@ contract is implemented. Reploy rejects these combinations before container creation or output-path preparation. Docker-managed volumes and tmpfs remain available because they do not expose a host filesystem path directly. +Application networking is also a portable environment policy rather than a +Docker mode. `public` controls globally routable IP destinations. `local` +controls private, link-local, multicast, reserved, and infrastructure metadata +destinations. Translation and tunneling ranges that can represent either class +are `ambiguous`: by default, `ambiguous: require-both` permits them only when +both `public` and `local` are allowed. IPv4-mapped IPv6 socket addresses use +the class of their embedded IPv4 destination because Linux emits them as IPv4 +packets. Container-local loopback remains available and cannot address host +loopback through the container network namespace. The backend configures the +container's DNS path from the same grants. With neither network class granted, +DNS is unavailable. With only `local`, it uses the host's configured resolver +so local, VPN, and split-DNS behavior remains available. With only `public`, it +uses the built-in Google Public DNS profile (`8.8.8.8` and `8.8.4.4`). With +both, it uses the host resolver, which normally provides both local and public +resolution. + +For Docker, the local-capable path leaves DNS selection to Docker so it derives +the container's resolver path from the host; the public-only path passes the +selected Google Public DNS profile through Docker's per-container DNS +configuration. Docker writes the resulting container resolver configuration. +The default bridge normally exposes host-derived resolver addresses, while a +custom network exposes Docker's embedded resolver at `127.0.0.11` and forwards +to the selected upstreams. Before installing the packet filter, Reploy's +trusted startup helper reads those engine-authored resolver addresses and +admits TCP and UDP port 53 only to them whenever either network class is +granted. This engine-owned exception does not grant general access to the +resolver's address class. Resolver selection is host policy rather than +blueprint policy; future Reploy host configuration may override the default +local and public resolver choices. The backend does not filter DNS answers. +Connections to every resolved address still pass the ordinary destination +policy, so an answer outside the granted address class remains unreachable. + +`ambiguous: allow` is a temporary, discouraged escape hatch for environments +that intentionally need those translation or tunneling ranges. It grants every +range in that coarse class even when either ordinary network class is denied, +so it weakens the isolation expressed by `public` and `local`. Reploy expects to +deprecate this option after the planned L3 policy gateway can classify the real +destination instead of its translated address. + +Declared workload endpoints remain reachable from their explicit host +publication, which uses loopback by default: the application firewall permits +new inbound TCP connections only to declared endpoint ports and permits the +corresponding established response traffic. When DNS is enabled, resolved +connections remain subject to the destination policy. + +The Linux-container backend realizes this policy with a trusted Reploy startup +helper and IPv4/IPv6 nftables rules inside the container network namespace, +including when both egress classes are allowed so undeclared inbound ports +remain closed. Docker starts only that helper as container root with the minimal +setup capabilities. After installing the rules, the helper changes to the +planned application UID/GID, empties every capability set and the capability +bounding set, locks securebits and `no-new-privileges`, verifies seccomp and +the final kernel state, and executes the exact application argv. Reploy-issued +execs into an application container use the same authority-dropping helper; +they never invoke an application command through raw `docker exec`. + +This is coarse IP-class enforcement. It is not domain, URL, DNS-content, +general outbound port policy, or packet auditing, and it does not defend a +container from an operator who already controls the Docker daemon. A backend +that cannot install and verify the requested policy fails closed. + This is a portable blueprint contract with target-specific realization. The current backend writes Linux account databases. A future native-Windows or other target backend may realize the same local identity through different OS diff --git a/docs/CONTROLLED_SESSION_DESIGN.md b/docs/CONTROLLED_SESSION_DESIGN.md index dd8bcfca..7261e1dd 100644 --- a/docs/CONTROLLED_SESSION_DESIGN.md +++ b/docs/CONTROLLED_SESSION_DESIGN.md @@ -836,17 +836,70 @@ All Reploy application runtime containers default to: - local network disabled. These are independent policy switches. A controlled workflow applies them -separately to the controller and workload environments. Local -denial includes host gateways, Docker peers outside the granted operation, -loopback redirection, private and link-local address ranges, IPv6 local ranges, -and infrastructure metadata endpoints. - -The initial implementation preserves this coarse public/local policy intent -and exact declared endpoint grants, using only backend isolation and endpoint -primitives whose behavior Reploy can verify. A backend that cannot realize a -requested combination fails closed. This slice does not introduce a custom -packet gateway and must not claim destination-, port-, domain-, DNS-, or -packet-level enforcement beyond what the selected primitive actually proves. +separately to the controller and workload environments. Local denial includes +host-loopback redirection, private and link-local address ranges, IPv6 local +ranges, and infrastructure metadata endpoints. Translation and tunneling +ranges that can represent either public or local destinations require both +grants by default. The initial coarse classifier follows the destination +address; topology-resistant peer and gateway confinement belongs to the +deferred L3 gateway design. + +The initial Linux/Docker implementation enforces this coarse public/local +policy with IPv4 and IPv6 nftables rules inside each application container's +network namespace. A trusted Reploy helper begins with only the setup +capabilities needed to install those rules and assume the planned application +identity. It then empties every capability set and the capability bounding +set, locks securebits and `no-new-privileges`, verifies seccomp and the final +kernel state, and executes the exact application argv. Reploy-issued execs use +the same guarded authority-drop path. A raw Docker daemon client remains a +trusted host operator outside this sandbox boundary. + +`public` classifies globally routable IP destinations. `local` classifies +private, link-local, multicast, reserved, and infrastructure metadata +destinations. `ambiguous` covers predefined translation and tunneling ranges; +its default `require-both` admits them only when both ordinary classes are +allowed. IPv4-mapped IPv6 socket addresses use the class of their embedded IPv4 +destination because Linux emits them as IPv4 packets. The temporary +`ambiguous: allow` escape hatch admits the remaining ambiguous class +independently. It is intentionally discouraged because an apparently public +translated address may reach a local destination, and it should be deprecated +once the deferred L3 gateway can enforce the underlying destination policy. +Container-local loopback remains available and cannot address host loopback +through the container network namespace. The backend configures the +container's DNS path from the same two grants: + +| Public | Local | DNS path | +| --- | --- | --- | +| deny | deny | no DNS | +| deny | allow | the host's configured resolver, preserving local, VPN, and split-DNS behavior | +| allow | deny | the built-in Google Public DNS profile (`8.8.8.8`, `8.8.4.4`) | +| allow | allow | the host's configured resolver, which normally provides both local and public resolution | + +For Docker, the local-capable path leaves DNS selection to Docker so it derives +the container's resolver path from the host. The public-only path passes the +selected Google Public DNS profile through Docker's per-container DNS +configuration. Docker writes the resulting container resolver configuration: +the default bridge normally receives host-derived resolver addresses, while a +custom network exposes Docker's embedded resolver at `127.0.0.11` and forwards +to the selected upstreams. Before installing the packet filter, the trusted +startup helper reads those engine-authored resolver addresses and admits TCP +and UDP port 53 only to them whenever either network class is granted. This is +an engine-owned DNS exception, not general access to the resolver's address +class. + +Resolver selection is host policy, not blueprint policy. Future Reploy host +configuration may override the default local and public resolver choices. The +backend does not classify or filter DNS answers. A local resolver may return a +public address and a public resolver may return a private address, but every +subsequent connection still passes the ordinary destination policy. When both +ordinary classes are enabled, egress is unrestricted, but the packet filter +remains in place to admit new inbound connections only on declared endpoint +ports. The helper always applies the same identity and authority-drop invariant. +A backend that cannot install and verify the policy fails closed. + +This slice does not introduce the deferred userland L3 gateway and must not +claim domain-, URL-, DNS-content-, general outbound destination-port-, or +audit-level policy. A controller may receive an explicit session-local grant to a declared workload endpoint. That grant is not treated as general local-network @@ -878,12 +931,12 @@ remains a separate prerequisite; this endpoint forwarding path is not a general router, HTTP policy engine, or domain-aware firewall. General network isolation and auditability are a separate design surface. -Future work may include an HTTP/HTTPS proxy, destination policy, controlled -DNS, and agent-sandbox audit records. HTTPS `CONNECT` can filter and audit a +Future work may include an HTTP/HTTPS proxy, destination and DNS-content +policy, and agent-sandbox audit records. HTTPS `CONNECT` can filter and audit a destination hostname without TLS interception, but cannot inspect encrypted URLs or content. Direct egress must be blocked to prevent proxy bypass. -Redirects, DNS rebinding, CDNs, WebSockets, QUIC, and workload-to-network -policy require explicit treatment. +Redirects, DNS rebinding, CDNs, WebSockets, QUIC, and workload-to-network policy +require explicit treatment. Until that design is implemented, rough public and local kill switches must fail closed and must not be described as domain-level isolation. @@ -1120,27 +1173,34 @@ Implementation status: the canonical application sandbox plan and its identity and kernel baseline are implemented for persistent Compose workloads and transient application commands. Reploy now imports canonical supplementary groups, rejects root-group membership for non-root identities, starts transient -commands directly as the final identity, drops all capabilities, enables +commands through the trusted setup helper, drops all capabilities before the +application starts, enables `no-new-privileges`, explicitly selects Docker's built-in seccomp profile, and prohibits privileged mode, host namespaces, and host devices in the common plan. Live Docker tests inspect both runtime paths. Trusted production startup verification is also implemented: Reploy packages the platform-specific probe in a final runtime layer, creates the locked container-local account there, records that layer outside the provider graph, and uses its fixed -verify-and-exec contract as the outermost process for persistent +sandbox-and-exec contract as the outermost process for persistent workloads, transient commands, shells, and lifecycle commands. The verifier fails closed unless `/proc/self/status` reports seccomp filtering, -`no-new-privileges`, and empty effective, permitted, and bounding capability +`no-new-privileges`, and empty inheritable, effective, permitted, bounding, and +ambient capability sets, then directly executes the exact application argv. Private-environment workloads use one additional fixed Reploy step: after verification, the probe executes the environment injector, which imports the private variables and then -executes the unchanged application argv. Network denial and resource limits -remain separate prerequisite slices. Root host authority is now enforced at -runtime: host sources are classified as input, shared state, or explicit -output; UID 0 is rejected for all three before container creation; and root -output options are rejected before host-path preparation. Docker-managed -volumes and tmpfs remain available to root. Ordinary binds also reject -canonical host root, `/proc`, +executes the unchanged application argv. The same helper now installs the +default-deny application-network policy for persistent and transient containers +while preserving exact declared inbound endpoints. Live Docker coverage +exercises all four public/local combinations over IPv4 and IPv6, strict and +escaped ambiguous-range handling, a globally reachable exception nested inside +a reserved parent range, a root default-denial case, non-root authority removal, +guarded exec, and host-loopback endpoint publication. Resource limits remain a +separate prerequisite slice. Root host authority is now enforced at runtime: +host sources are classified as input, shared state, or explicit output; UID 0 is +rejected for all three before container creation; and root output options are +rejected before host-path preparation. Docker-managed volumes and tmpfs remain +available to root. Ordinary binds also reject canonical host root, `/proc`, `/dev`, and `/sys` sources plus equivalent protected filesystem mounts detected through native filesystem identity. Explicit non-root directory binds intentionally grant access to their remaining unmasked contents, including @@ -1217,16 +1277,16 @@ lease protocol. ### Network Isolation and Audit -After the coarse public/local kill switches, define a separate Reploy userland -L3 policy gateway for finer network control. Its design should cover a -capability-free application network namespace, one-shot route initialization, -an isolated data path whose only peer is the gateway, private gateway control, -root-resistant route invariants, direct-egress prevention, destination and -port grants, DNS and IPv6 policy, metadata protection, auditing, resource -limits, failure behavior, reconciliation, and portable Docker/Podman -integration. The one-way, exact endpoint forwarding used by the initial -controlled session remains intentionally narrower and does not depend on this -later gateway. +After the implemented coarse public/local kill switches, define a separate +Reploy userland L3 policy gateway for finer network control. Its design should +cover a capability-free application network namespace, one-shot route +initialization, an isolated data path whose only peer is the gateway, private +gateway control, root-resistant route invariants, direct-egress prevention, +destination and port grants, DNS and IPv6 policy, metadata protection, +auditing, resource limits, failure behavior, reconciliation, and portable +Docker/Podman integration. The one-way, exact endpoint forwarding used by the +initial controlled session remains intentionally narrower and does not depend +on this later gateway. ### Disposable Writable Workspaces diff --git a/go.mod b/go.mod index ce7e23f5..3c647c8a 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/charmbracelet/x/term v0.2.2 github.com/distribution/reference v0.6.0 github.com/go-git/go-git/v5 v5.19.1 + github.com/google/nftables v0.3.0 github.com/pelletier/go-toml/v2 v2.4.3 github.com/yusufpapurcu/wmi v1.2.4 golang.org/x/sys v0.43.0 @@ -37,6 +38,7 @@ require ( github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect @@ -44,6 +46,8 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect + github.com/mdlayher/socket v0.5.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect @@ -56,6 +60,7 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/crypto v0.50.0 // indirect golang.org/x/net v0.53.0 // indirect + golang.org/x/sync v0.20.0 // indirect golang.org/x/text v0.36.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index f1cb8ae9..50e4105a 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,8 @@ github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8J github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg= +github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -89,6 +91,10 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg= +github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o= +github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI= +github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= @@ -121,6 +127,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= @@ -135,6 +143,8 @@ golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aI golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/blueprint/model.go b/internal/blueprint/model.go index b9a9248d..08f2d3c6 100644 --- a/internal/blueprint/model.go +++ b/internal/blueprint/model.go @@ -41,7 +41,28 @@ type Environment struct { } type EnvironmentRuntime struct { - User string + User string + Network RuntimeNetwork +} + +type NetworkAccess string + +const ( + NetworkAccessDeny NetworkAccess = "deny" + NetworkAccessAllow NetworkAccess = "allow" +) + +type AmbiguousNetworkAccess string + +const ( + AmbiguousNetworkAccessRequireBoth AmbiguousNetworkAccess = "require-both" + AmbiguousNetworkAccessAllow AmbiguousNetworkAccess = "allow" +) + +type RuntimeNetwork struct { + Public NetworkAccess + Local NetworkAccess + Ambiguous AmbiguousNetworkAccess } type EnvironmentPackages struct { diff --git a/internal/blueprint/resolve.go b/internal/blueprint/resolve.go index 2df9526b..8f385d1f 100644 --- a/internal/blueprint/resolve.go +++ b/internal/blueprint/resolve.go @@ -40,6 +40,10 @@ func Resolve(source Syntax) (Document, error) { if err != nil { return Document{}, err } + runtimeNetwork, err := resolveRuntimeNetwork(source.Environment.Runtime.Network) + if err != nil { + return Document{}, err + } extended, err := resolveExtends(source) if err != nil { return Document{}, err @@ -59,7 +63,7 @@ func Resolve(source Syntax) (Document, error) { Applications: map[string]Application{}, Components: map[string]Component{}, AllowConcurrent: allowConcurrent, - Runtime: EnvironmentRuntime{User: runtimeUser}, + Runtime: EnvironmentRuntime{User: runtimeUser, Network: runtimeNetwork}, Terminal: Terminal{ColorEnv: strings.TrimSpace(source.Environment.Terminal.ColorEnv)}, Install: resolveInstallSyntax(source.Environment.Install, variables), Mounts: map[string]EnvironmentMount{}, @@ -87,6 +91,48 @@ func Resolve(source Syntax) (Document, error) { return document, nil } +func resolveRuntimeNetwork(source RuntimeNetworkSyntax) (RuntimeNetwork, error) { + public, err := resolveNetworkAccess("environment.runtime.network.public", source.Public) + if err != nil { + return RuntimeNetwork{}, err + } + local, err := resolveNetworkAccess("environment.runtime.network.local", source.Local) + if err != nil { + return RuntimeNetwork{}, err + } + ambiguous, err := resolveAmbiguousNetworkAccess(source.Ambiguous) + if err != nil { + return RuntimeNetwork{}, err + } + return RuntimeNetwork{Public: public, Local: local, Ambiguous: ambiguous}, nil +} + +func resolveNetworkAccess(field string, value string) (NetworkAccess, error) { + access := NetworkAccess(strings.TrimSpace(value)) + if access == "" { + return NetworkAccessDeny, nil + } + switch access { + case NetworkAccessDeny, NetworkAccessAllow: + return access, nil + default: + return "", fmt.Errorf("%s must be allow or deny", field) + } +} + +func resolveAmbiguousNetworkAccess(value string) (AmbiguousNetworkAccess, error) { + access := AmbiguousNetworkAccess(strings.TrimSpace(value)) + if access == "" { + return AmbiguousNetworkAccessRequireBoth, nil + } + switch access { + case AmbiguousNetworkAccessRequireBoth, AmbiguousNetworkAccessAllow: + return access, nil + default: + return "", fmt.Errorf("environment.runtime.network.ambiguous must be require-both or allow") + } +} + func resolveRuntimeUser(value string) (string, error) { value = strings.TrimSpace(value) if value == "" { diff --git a/internal/blueprint/resolve_test.go b/internal/blueprint/resolve_test.go index 5deffb58..475bd9c5 100644 --- a/internal/blueprint/resolve_test.go +++ b/internal/blueprint/resolve_test.go @@ -26,6 +26,9 @@ func TestResolveProducesTypedEnvironment(t *testing.T) { if document.Environment.Runtime.User != DefaultRuntimeUser { t.Fatalf("runtime user = %q", document.Environment.Runtime.User) } + if document.Environment.Runtime.Network != (RuntimeNetwork{Public: NetworkAccessDeny, Local: NetworkAccessDeny, Ambiguous: AmbiguousNetworkAccessRequireBoth}) { + t.Fatalf("runtime network = %#v", document.Environment.Runtime.Network) + } if got := document.Blueprint.Compatibility.Platforms; !reflect.DeepEqual(got, []Platform{ {OS: "linux", Architecture: "amd64", Canonical: "linux/amd64"}, {OS: "linux", Architecture: "arm64", Canonical: "linux/arm64"}, @@ -51,6 +54,37 @@ func TestResolveProducesTypedEnvironment(t *testing.T) { } } +func TestResolveRuntimeNetwork(t *testing.T) { + for _, test := range []struct { + name string + source RuntimeNetworkSyntax + want RuntimeNetwork + field string + }{ + {name: "default", want: RuntimeNetwork{Public: NetworkAccessDeny, Local: NetworkAccessDeny, Ambiguous: AmbiguousNetworkAccessRequireBoth}}, + {name: "public only", source: RuntimeNetworkSyntax{Public: "allow"}, want: RuntimeNetwork{Public: NetworkAccessAllow, Local: NetworkAccessDeny, Ambiguous: AmbiguousNetworkAccessRequireBoth}}, + {name: "local only", source: RuntimeNetworkSyntax{Local: "allow"}, want: RuntimeNetwork{Public: NetworkAccessDeny, Local: NetworkAccessAllow, Ambiguous: AmbiguousNetworkAccessRequireBoth}}, + {name: "both", source: RuntimeNetworkSyntax{Public: "allow", Local: "allow"}, want: RuntimeNetwork{Public: NetworkAccessAllow, Local: NetworkAccessAllow, Ambiguous: AmbiguousNetworkAccessRequireBoth}}, + {name: "ambiguous escape hatch", source: RuntimeNetworkSyntax{Ambiguous: "allow"}, want: RuntimeNetwork{Public: NetworkAccessDeny, Local: NetworkAccessDeny, Ambiguous: AmbiguousNetworkAccessAllow}}, + {name: "invalid public", source: RuntimeNetworkSyntax{Public: "yes"}, field: "environment.runtime.network.public"}, + {name: "invalid local", source: RuntimeNetworkSyntax{Local: "none"}, field: "environment.runtime.network.local"}, + {name: "invalid ambiguous", source: RuntimeNetworkSyntax{Ambiguous: "deny"}, field: "environment.runtime.network.ambiguous"}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := resolveRuntimeNetwork(test.source) + if test.field == "" { + if err != nil || got != test.want { + t.Fatalf("runtime network = %#v, %v; want %#v", got, err, test.want) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.field) { + t.Fatalf("runtime network error = %v", err) + } + }) + } +} + func TestResolveRuntimeUser(t *testing.T) { for _, test := range []struct { name string diff --git a/internal/blueprint/syntax.go b/internal/blueprint/syntax.go index 0aa1f943..42770f6d 100644 --- a/internal/blueprint/syntax.go +++ b/internal/blueprint/syntax.go @@ -44,7 +44,14 @@ type EnvironmentSyntax struct { } type EnvironmentRuntimeSyntax struct { - User string `yaml:"user"` + User string `yaml:"user"` + Network RuntimeNetworkSyntax `yaml:"network"` +} + +type RuntimeNetworkSyntax struct { + Public string `yaml:"public"` + Local string `yaml:"local"` + Ambiguous string `yaml:"ambiguous"` } type TerminalSyntax struct { diff --git a/internal/deploy/runtime_policy.go b/internal/deploy/runtime_policy.go index f4459562..3480d9d4 100644 --- a/internal/deploy/runtime_policy.go +++ b/internal/deploy/runtime_policy.go @@ -3,6 +3,8 @@ package deploy import ( "fmt" "path" + "slices" + "strconv" "strings" "github.com/omry/reploy/internal/blueprint" @@ -28,6 +30,7 @@ const ( type RuntimePolicyV1 struct { Schema string `json:"schema"` StartupVerifier ApplicationStartupVerifierV1 `json:"startup_verifier"` + Network blueprint.RuntimeNetwork `json:"network"` ProtectedPaths []ProtectedPathV1 `json:"protected_paths"` Plans []RuntimePlanV1 `json:"plans"` } @@ -40,6 +43,7 @@ type ProtectedPathV1 struct { type RuntimePlanV1 struct { ID string `json:"id"` + InboundTCP []string `json:"inbound_tcp"` Mounts []RuntimeMountV1 `json:"mounts"` Executables []providers.QualifiedOutput `json:"executables"` } @@ -67,6 +71,15 @@ func ValidateRuntimePolicyV1(policy RuntimePolicyV1) error { if err := ValidateApplicationStartupVerifierV1(policy.StartupVerifier, false); err != nil { return fmt.Errorf("runtime policy startup verifier: %w", err) } + if err := validateRuntimeNetworkAccessV1("public", policy.Network.Public); err != nil { + return err + } + if err := validateRuntimeNetworkAccessV1("local", policy.Network.Local); err != nil { + return err + } + if err := validateRuntimeAmbiguousNetworkAccessV1(policy.Network.Ambiguous); err != nil { + return err + } for index, protected := range policy.ProtectedPaths { if err := validateRuntimeAbsolutePath("protected path", protected.Path); err != nil { return err @@ -90,9 +103,23 @@ func ValidateRuntimePolicyV1(policy RuntimePolicyV1) error { if index > 0 && policy.Plans[index-1].ID >= plan.ID { return fmt.Errorf("runtime plans must be unique and sorted by ID") } - if plan.Mounts == nil || plan.Executables == nil { + if plan.InboundTCP == nil || plan.Mounts == nil || plan.Executables == nil { return fmt.Errorf("runtime plan %q collections must use arrays", plan.ID) } + previousPort := 0 + for portIndex, rawPort := range plan.InboundTCP { + port, err := strconv.Atoi(rawPort) + if err != nil || strconv.Itoa(port) != rawPort { + return fmt.Errorf("runtime plan %q inbound TCP ports must use canonical decimal strings", plan.ID) + } + if port < 1 || port > 65535 { + return fmt.Errorf("runtime plan %q inbound TCP port must be between 1 and 65535", plan.ID) + } + if portIndex > 0 && previousPort >= port { + return fmt.Errorf("runtime plan %q inbound TCP ports must be unique and sorted", plan.ID) + } + previousPort = port + } for mountIndex, mount := range plan.Mounts { if err := validateRuntimeAbsolutePath("mount destination", mount.Destination); err != nil { return fmt.Errorf("runtime plan %q: %w", plan.ID, err) @@ -124,6 +151,35 @@ func ValidateRuntimePolicyV1(policy RuntimePolicyV1) error { return nil } +func CanonicalRuntimeInboundTCPV1(values []int) []string { + ports := append([]int{}, values...) + slices.Sort(ports) + ports = slices.Compact(ports) + result := make([]string, len(ports)) + for index, port := range ports { + result[index] = strconv.Itoa(port) + } + return result +} + +func validateRuntimeNetworkAccessV1(name string, access blueprint.NetworkAccess) error { + switch access { + case blueprint.NetworkAccessDeny, blueprint.NetworkAccessAllow: + return nil + default: + return fmt.Errorf("runtime policy %s network access must be allow or deny", name) + } +} + +func validateRuntimeAmbiguousNetworkAccessV1(access blueprint.AmbiguousNetworkAccess) error { + switch access { + case blueprint.AmbiguousNetworkAccessRequireBoth, blueprint.AmbiguousNetworkAccessAllow: + return nil + default: + return fmt.Errorf("runtime policy ambiguous network access must be require-both or allow") + } +} + func validateRuntimeReservedDestination(destination string) error { if destination == "/" { return fmt.Errorf("mount destination must not be the container filesystem root") diff --git a/internal/deploy/runtime_policy_test.go b/internal/deploy/runtime_policy_test.go index 88ea5222..7325fce5 100644 --- a/internal/deploy/runtime_policy_test.go +++ b/internal/deploy/runtime_policy_test.go @@ -4,18 +4,20 @@ import ( "strings" "testing" + "github.com/omry/reploy/internal/blueprint" "github.com/omry/reploy/internal/providers" ) func validRuntimePolicy() RuntimePolicyV1 { return RuntimePolicyV1{ Schema: RuntimePolicySchemaV1, StartupVerifier: ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []ProtectedPathV1{ {Path: "/.reploy", Kind: ProtectedPathReployRoot, Owner: "reploy"}, {Path: "/opt/app/bin/tool", Kind: ProtectedPathExecutablePath, Owner: "app.tool"}, }, Plans: []RuntimePlanV1{{ - ID: "serve", + ID: "serve", InboundTCP: []string{"8080"}, Mounts: []RuntimeMountV1{ {Destination: "/mnt/config", SourceKind: RuntimeMountSourceFile, ReadOnly: true}, {Destination: "/workspace/output", SourceKind: RuntimeMountSourceDirectory}, @@ -38,6 +40,23 @@ func TestRuntimePolicyDigestV1IsStable(t *testing.T) { if first != second || first == "" { t.Fatalf("runtime policy digests = %q, %q", first, second) } + policy.Network.Public = blueprint.NetworkAccessAllow + changed, err := RuntimePolicyDigestV1(policy) + if err != nil { + t.Fatal(err) + } + if changed == first { + t.Fatal("runtime policy digest ignored network policy") + } + policy.Network.Public = blueprint.NetworkAccessDeny + policy.Plans[0].InboundTCP = []string{"8081"} + changed, err = RuntimePolicyDigestV1(policy) + if err != nil { + t.Fatal(err) + } + if changed == first { + t.Fatal("runtime policy digest ignored inbound TCP grants") + } } func TestValidateRuntimePolicyV1RejectsNoncanonicalStructure(t *testing.T) { @@ -51,6 +70,11 @@ func TestValidateRuntimePolicyV1RejectsNoncanonicalStructure(t *testing.T) { value.ProtectedPaths[0], value.ProtectedPaths[1] = value.ProtectedPaths[1], value.ProtectedPaths[0] }, want: "protected paths"}, {name: "unsafe owner", mutate: func(value *RuntimePolicyV1) { value.ProtectedPaths[0].Owner = "\n" }, want: "owner"}, + {name: "invalid public network", mutate: func(value *RuntimePolicyV1) { value.Network.Public = "sometimes" }, want: "public network"}, + {name: "invalid local network", mutate: func(value *RuntimePolicyV1) { value.Network.Local = "sometimes" }, want: "local network"}, + {name: "nil inbound TCP", mutate: func(value *RuntimePolicyV1) { value.Plans[0].InboundTCP = nil }, want: "collections"}, + {name: "unsorted inbound TCP", mutate: func(value *RuntimePolicyV1) { value.Plans[0].InboundTCP = []string{"8081", "8080"} }, want: "inbound TCP ports"}, + {name: "invalid inbound TCP", mutate: func(value *RuntimePolicyV1) { value.Plans[0].InboundTCP = []string{"65536"} }, want: "between 1 and 65535"}, {name: "nil mounts", mutate: func(value *RuntimePolicyV1) { value.Plans[0].Mounts = nil }, want: "collections"}, {name: "unsorted mounts", mutate: func(value *RuntimePolicyV1) { value.Plans[0].Mounts[0], value.Plans[0].Mounts[1] = value.Plans[0].Mounts[1], value.Plans[0].Mounts[0] diff --git a/internal/deploy/runtime_verifier.go b/internal/deploy/runtime_verifier.go index 3e25b532..55576d73 100644 --- a/internal/deploy/runtime_verifier.go +++ b/internal/deploy/runtime_verifier.go @@ -11,7 +11,7 @@ import ( const ( ApplicationStartupVerifierSchemaV1 = "application-startup-verifier-v1" - ApplicationStartupVerifierRecipeV1 = "linux-proc-status-verify-exec-v1" + ApplicationStartupVerifierRecipeV1 = "linux-network-policy-sandbox-exec-v1" ApplicationStartupVerifierPathV1 = "/reploy-probe" ApplicationRuntimeLayerSchemaV1 = "application-runtime-layer-v1" ApplicationLocalAccountSchemaV1 = "application-local-account-v1" diff --git a/internal/dockerdeploy/application_network_policy_integration_test.go b/internal/dockerdeploy/application_network_policy_integration_test.go new file mode 100644 index 00000000..03acbb5c --- /dev/null +++ b/internal/dockerdeploy/application_network_policy_integration_test.go @@ -0,0 +1,445 @@ +package dockerdeploy + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/omry/reploy/internal/blueprint" +) + +func TestApplicationNetworkPolicyDockerIntegration(t *testing.T) { + if os.Getenv("REPLOY_DOCKER_INTEGRATION") != "1" { + t.Skip("set REPLOY_DOCKER_INTEGRATION=1 to run Docker integration evidence") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + image, _ := buildApplicationStartupVerifierIntegrationImage(t, ctx) + helper := buildNetworkPolicyIntegrationHelper(t, ctx) + localNetwork, publicNetwork, publicExceptionNetwork, ambiguousNetwork, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, dnsName := createNetworkPolicyIntegrationPeers(t, ctx, image, helper) + directResolver := createNetworkPolicyDirectResolver(t, ctx, image, helper) + + for _, test := range []struct { + name string + public blueprint.NetworkAccess + local blueprint.NetworkAccess + want bool + }{ + {name: "deny both", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny}, + {name: "public only", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessDeny, want: true}, + {name: "local only", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessAllow, want: true}, + {name: "allow both", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessAllow, want: true}, + } { + for _, transport := range []string{"udp", "tcp"} { + t.Run("engine-selected DNS "+transport+" "+test.name, func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, test.public, test.local, blueprint.AmbiguousNetworkAccessRequireBoth) + command := ResolvedEnvironmentCommand{Argv: []string{"/network-test", "dns", "reploy.test", strconv.FormatBool(test.want), transport}} + execution, err := PlanTransientContainerExecutionV1(plan, command, nil, "run-0000000000000001", false, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = exec.CommandContext(context.Background(), "docker", execution.Cleanup.Args...).Run() }) + runDockerIntegration(t, ctx, dockerCreateWithDNSV1(t, execution.Create.Args, net.JoinHostPort(directResolver, "53"))...) + output := runDockerIntegration(t, ctx, execution.Start.Args...) + if !strings.Contains(output, "DNS_PASS") { + t.Fatalf("direct local DNS output = %q", output) + } + }) + } + } + + t.Run("public-only default DNS profile", func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, blueprint.NetworkAccessAllow, blueprint.NetworkAccessDeny, blueprint.AmbiguousNetworkAccessRequireBoth) + command := ResolvedEnvironmentCommand{Argv: []string{"/network-test", "dns", "example.com", "true", "udp"}} + execution, err := PlanTransientContainerExecutionV1(plan, command, nil, "run-0000000000000001", false, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = exec.CommandContext(context.Background(), "docker", execution.Cleanup.Args...).Run() }) + runDockerIntegration(t, ctx, execution.Create.Args...) + output := runDockerIntegration(t, ctx, execution.Start.Args...) + if !strings.Contains(output, "DNS_PASS") { + t.Fatalf("public-only default DNS output = %q", output) + } + }) + + for _, test := range []struct { + name string + public blueprint.NetworkAccess + local blueprint.NetworkAccess + ambiguous blueprint.AmbiguousNetworkAccess + wantPublic bool + wantLocal bool + wantAmbiguous bool + }{ + {name: "deny-deny", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, + {name: "allow-deny", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, wantPublic: true}, + {name: "deny-allow", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessAllow, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, wantLocal: true}, + {name: "ambiguous-escape-hatch", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessAllow, wantAmbiguous: true}, + {name: "allow-allow", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessAllow, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, wantPublic: true, wantLocal: true, wantAmbiguous: true}, + } { + t.Run("transient "+test.name, func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, test.public, test.local, test.ambiguous) + command := ResolvedEnvironmentCommand{Argv: networkPolicyIntegrationWorkloadArgv( + plan.Sandbox.RuntimeUser.UID, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, + test.wantLocal, test.wantPublic, test.wantAmbiguous, test.wantPublic, false, dnsName, + )} + execution, err := PlanTransientContainerExecutionV1(plan, command, nil, "run-0000000000000001", false, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = exec.CommandContext(context.Background(), "docker", execution.Cleanup.Args...).Run() }) + runDockerIntegration(t, ctx, dockerCreateWithDNSV1(t, execution.Create.Args, localAddresses[0])...) + runDockerIntegration(t, ctx, "network", "connect", localNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", publicNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", publicExceptionNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", ambiguousNetwork, execution.Container) + output := runDockerIntegration(t, ctx, execution.Start.Args...) + if !strings.Contains(output, "NETWORK_POLICY_PASS") { + t.Fatalf("transient network-policy output = %q", output) + } + }) + } + + t.Run("transient root deny-deny", func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, blueprint.NetworkAccessDeny, blueprint.NetworkAccessDeny, blueprint.AmbiguousNetworkAccessRequireBoth) + plan.Sandbox = newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 0, GID: 0, DockerUser: "0:0"}, + blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, + ) + command := ResolvedEnvironmentCommand{Argv: networkPolicyIntegrationWorkloadArgv( + 0, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, false, false, false, false, false, dnsName, + )} + execution, err := PlanTransientContainerExecutionV1(plan, command, nil, "run-0000000000000001", false, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = exec.CommandContext(context.Background(), "docker", execution.Cleanup.Args...).Run() }) + runDockerIntegration(t, ctx, dockerCreateWithDNSV1(t, execution.Create.Args, localAddresses[0])...) + runDockerIntegration(t, ctx, "network", "connect", localNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", publicNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", publicExceptionNetwork, execution.Container) + runDockerIntegration(t, ctx, "network", "connect", ambiguousNetwork, execution.Container) + output := runDockerIntegration(t, ctx, execution.Start.Args...) + if !strings.Contains(output, "NETWORK_POLICY_PASS") { + t.Fatalf("root network-policy output = %q", output) + } + }) + + for _, test := range []struct { + name string + public blueprint.NetworkAccess + local blueprint.NetworkAccess + wantPublic bool + wantLocal bool + }{ + {name: "deny-deny", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny}, + {name: "allow-deny", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessDeny, wantPublic: true}, + {name: "deny-allow", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessAllow, wantLocal: true}, + {name: "allow-allow", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessAllow, wantPublic: true, wantLocal: true}, + } { + t.Run("persistent "+test.name, func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, test.public, test.local, blueprint.AmbiguousNetworkAccessRequireBoth) + plan.Workload = &WorkloadExecutionPlan{ + Argv: networkPolicyIntegrationWorkloadArgv( + plan.Sandbox.RuntimeUser.UID, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, + test.wantLocal, test.wantPublic, test.wantLocal && test.wantPublic, test.wantPublic, false, dnsName, + ), + Endpoints: map[string]EndpointExecutionPlan{}, + } + rendered, err := RenderDockerInputs(plan, "network-policy") + if err != nil { + t.Fatal(err) + } + composePath := filepath.Join(t.TempDir(), "compose.yaml") + if err := os.WriteFile(composePath, rendered.Compose, 0o600); err != nil { + t.Fatal(err) + } + composeArgs := []string{"compose", "--project-name", plan.NetworkName, "-f", composePath} + t.Cleanup(func() { + args := append(append([]string(nil), composeArgs...), "down", "--remove-orphans") + _ = exec.CommandContext(context.Background(), "docker", args...).Run() + }) + runDockerIntegration(t, ctx, append(composeArgs, "create", "--pull", "never")...) + runDockerIntegration(t, ctx, "network", "connect", localNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicExceptionNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", ambiguousNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, append(composeArgs, "start")...) + runDockerIntegration(t, ctx, "wait", plan.ContainerName) + logs := runDockerIntegration(t, ctx, append(composeArgs, "logs")...) + if !strings.Contains(logs, "NETWORK_POLICY_PASS") { + t.Fatalf("persistent network-policy logs = %q", logs) + } + }) + } + + t.Run("persistent endpoint survives default denial", func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, blueprint.NetworkAccessDeny, blueprint.NetworkAccessDeny, blueprint.AmbiguousNetworkAccessRequireBoth) + plan.Workload = &WorkloadExecutionPlan{ + Argv: networkPolicyIntegrationWorkloadArgv(plan.Sandbox.RuntimeUser.UID, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, false, false, false, false, true, dnsName), + Endpoints: map[string]EndpointExecutionPlan{ + "http": {Scheme: "http", PublishAddress: "127.0.0.1", PublishedPort: reserveNetworkPolicyIntegrationPort(t), ContainerPort: 8080}, + }, + } + rendered, err := RenderDockerInputs(plan, "network-policy") + if err != nil { + t.Fatal(err) + } + composePath := filepath.Join(t.TempDir(), "compose.yaml") + if err := os.WriteFile(composePath, rendered.Compose, 0o600); err != nil { + t.Fatal(err) + } + composeArgs := []string{"compose", "--project-name", plan.NetworkName, "-f", composePath} + t.Cleanup(func() { + args := append(append([]string(nil), composeArgs...), "down", "--remove-orphans") + _ = exec.CommandContext(context.Background(), "docker", args...).Run() + }) + runDockerIntegration(t, ctx, append(composeArgs, "create", "--pull", "never")...) + runDockerIntegration(t, ctx, "network", "connect", localNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicExceptionNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", ambiguousNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, append(composeArgs, "start")...) + url := fmt.Sprintf("http://127.0.0.1:%d/", plan.Workload.Endpoints["http"].PublishedPort) + waitNetworkPolicyIntegrationEndpoint(t, url, func() string { return runDockerIntegration(t, ctx, append(composeArgs, "logs")...) }) + runDockerIntegration(t, ctx, "restart", plan.ContainerName) + waitNetworkPolicyIntegrationEndpoint(t, url, func() string { return runDockerIntegration(t, ctx, append(composeArgs, "logs")...) }) + logs := runDockerIntegration(t, ctx, append(composeArgs, "logs")...) + if !strings.Contains(logs, "NETWORK_POLICY_PASS") { + t.Fatalf("persistent network-policy logs = %q", logs) + } + }) + + t.Run("persistent unrestricted egress still limits inbound", func(t *testing.T) { + plan := networkPolicyIntegrationPlan(t, image, helper, blueprint.NetworkAccessAllow, blueprint.NetworkAccessAllow, blueprint.AmbiguousNetworkAccessRequireBoth) + plan.Workload = &WorkloadExecutionPlan{ + Argv: networkPolicyIntegrationWorkloadArgv(plan.Sandbox.RuntimeUser.UID, localAddresses, publicAddresses, ambiguousAddress, publicExceptionAddress, true, true, true, true, true, dnsName), + Endpoints: map[string]EndpointExecutionPlan{ + "http": {Scheme: "http", PublishAddress: "127.0.0.1", PublishedPort: reserveNetworkPolicyIntegrationPort(t), ContainerPort: 8080}, + }, + } + rendered, err := RenderDockerInputs(plan, "network-policy") + if err != nil { + t.Fatal(err) + } + composePath := filepath.Join(t.TempDir(), "compose.yaml") + if err := os.WriteFile(composePath, rendered.Compose, 0o600); err != nil { + t.Fatal(err) + } + composeArgs := []string{"compose", "--project-name", plan.NetworkName, "-f", composePath} + t.Cleanup(func() { + args := append(append([]string(nil), composeArgs...), "down", "--remove-orphans") + _ = exec.CommandContext(context.Background(), "docker", args...).Run() + }) + runDockerIntegration(t, ctx, append(composeArgs, "create", "--pull", "never")...) + runDockerIntegration(t, ctx, "network", "connect", localNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", publicExceptionNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, "network", "connect", ambiguousNetwork, plan.ContainerName) + runDockerIntegration(t, ctx, append(composeArgs, "start")...) + url := fmt.Sprintf("http://127.0.0.1:%d/", plan.Workload.Endpoints["http"].PublishedPort) + waitNetworkPolicyIntegrationEndpoint(t, url, func() string { return runDockerIntegration(t, ctx, append(composeArgs, "logs")...) }) + runDockerIntegration(t, ctx, "network", "connect", plan.NetworkName, dnsName) + t.Cleanup(func() { + _ = exec.CommandContext(context.Background(), "docker", "network", "disconnect", "--force", plan.NetworkName, dnsName).Run() + }) + runDockerIntegration(t, ctx, "exec", dnsName, "/network-test", "dial", plan.ContainerName+":8080", "true") + runDockerIntegration(t, ctx, "exec", dnsName, "/network-test", "dial", plan.ContainerName+":8081", "false") + }) +} + +func dockerCreateWithDNSV1(t *testing.T, args []string, resolver string) []string { + t.Helper() + host, _, err := net.SplitHostPort(resolver) + if err != nil { + t.Fatalf("split integration resolver %q: %v", resolver, err) + } + entrypoint := -1 + for index, argument := range args { + if argument == "--entrypoint" { + entrypoint = index + break + } + } + if entrypoint < 0 { + t.Fatal("transient Docker create command has no entrypoint boundary") + } + result := make([]string, 0, len(args)+2) + for index := 0; index < entrypoint; index++ { + if args[index] == "--dns" { + if index+1 >= entrypoint { + t.Fatal("transient Docker create command has an incomplete DNS option") + } + index++ + continue + } + result = append(result, args[index]) + } + result = append(result, "--dns", host) + return append(result, args[entrypoint:]...) +} + +func waitNetworkPolicyIntegrationEndpoint(t *testing.T, url string, logs func() string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + var requestErr error + for { + response, err := http.Get(url) + requestErr = err + if err == nil { + _ = response.Body.Close() + if response.StatusCode == http.StatusOK { + return + } + } + if time.Now().After(deadline) { + t.Fatalf("persistent endpoint did not become ready: %v\n%s", requestErr, logs()) + } + time.Sleep(100 * time.Millisecond) + } +} + +func networkPolicyIntegrationPlan(t *testing.T, image string, helper string, public blueprint.NetworkAccess, local blueprint.NetworkAccess, ambiguous blueprint.AmbiguousNetworkAccess) DockerExecutionPlan { + t.Helper() + return DockerExecutionPlan{ + EnvironmentID: "network-policy", DeploymentDir: t.TempDir(), Phase: blueprint.PhaseStaged, + Image: image, ContainerName: uniqueDockerIntegrationName("reploy-network-policy"), + NetworkName: uniqueDockerIntegrationName("reploy-network-policy-network"), + Sandbox: newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 12345, GID: 23456, DockerUser: "12345:23456"}, + blueprint.RuntimeNetwork{Public: public, Local: local, Ambiguous: ambiguous}, + ), + Mounts: []MountExecutionPlan{{ + Name: "network-test", Mode: blueprint.MountBind, Source: helper, + SourceKind: "file", Target: "/network-test", ReadOnly: true, + }}, + } +} + +func networkPolicyIntegrationWorkloadArgv(uid int, local []string, public []string, ambiguous string, publicException string, wantLocal bool, wantPublic bool, wantAmbiguous bool, wantPublicException bool, serve bool, dnsName string) []string { + mode := "exit" + if serve { + mode = "serve" + } + return []string{ + "/network-test", "workload", strconv.Itoa(uid), + local[0], local[1], public[0], public[1], ambiguous, publicException, + strconv.FormatBool(wantLocal), strconv.FormatBool(wantPublic), strconv.FormatBool(wantAmbiguous), strconv.FormatBool(wantPublicException), mode, + dnsName, strconv.FormatBool(wantPublic || wantLocal), + } +} + +func buildNetworkPolicyIntegrationHelper(t *testing.T, ctx context.Context) string { + t.Helper() + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skipf("network-policy Docker integration does not build a helper for %s", runtime.GOARCH) + } + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + output := filepath.Join(dockerIntegrationSharedTempDir(t), "network-policy-helper") + command := exec.CommandContext(ctx, "go", "build", "-buildvcs=false", "-o", output, "./internal/dockerdeploy/testdata/network_policy_helper") + command.Dir = root + command.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux", "GOARCH="+runtime.GOARCH, "GOCACHE="+filepath.Join(t.TempDir(), "go-cache")) + if content, err := command.CombinedOutput(); err != nil { + t.Fatalf("build network-policy integration helper: %v\n%s", err, content) + } + return output +} + +func createNetworkPolicyIntegrationPeers(t *testing.T, ctx context.Context, image string, helper string) (string, string, string, string, []string, []string, string, string, string) { + t.Helper() + seed := int(time.Now().UnixNano()%180) + 40 + localNetwork := uniqueDockerIntegrationName("reploy-network-local") + publicNetwork := uniqueDockerIntegrationName("reploy-network-public") + publicExceptionNetwork := uniqueDockerIntegrationName("reploy-network-public-exception") + ambiguousNetwork := uniqueDockerIntegrationName("reploy-network-ambiguous") + localV4 := fmt.Sprintf("172.30.%d.3", seed) + localV6 := fmt.Sprintf("fd30:%x::3", seed) + publicV4 := fmt.Sprintf("11.%d.0.3", seed) + publicV6 := fmt.Sprintf("2001:4860:%x::3", seed) + publicExceptionV6 := fmt.Sprintf("2001:20:%x::3", seed) + ambiguousV4 := fmt.Sprintf("172.29.%d.3", seed) + ambiguousV6 := fmt.Sprintf("64:ff9b:1:%x::3", seed) + runDockerIntegration(t, ctx, "network", "create", "--ipv6", "--subnet", fmt.Sprintf("172.30.%d.0/24", seed), "--subnet", fmt.Sprintf("fd30:%x::/64", seed), localNetwork) + runDockerIntegration(t, ctx, "network", "create", "--ipv6", "--subnet", fmt.Sprintf("11.%d.0.0/24", seed), "--subnet", fmt.Sprintf("2001:4860:%x::/64", seed), publicNetwork) + runDockerIntegration(t, ctx, "network", "create", "--ipv6", "--subnet", fmt.Sprintf("172.28.%d.0/24", seed), "--subnet", fmt.Sprintf("2001:20:%x::/64", seed), publicExceptionNetwork) + runDockerIntegration(t, ctx, "network", "create", "--ipv6", "--subnet", fmt.Sprintf("172.29.%d.0/24", seed), "--subnet", fmt.Sprintf("64:ff9b:1:%x::/64", seed), ambiguousNetwork) + t.Cleanup(func() { + _ = exec.CommandContext(context.Background(), "docker", "network", "rm", localNetwork, publicNetwork, publicExceptionNetwork, ambiguousNetwork).Run() + }) + publicPeerName := "" + for _, peer := range []struct { + name string + network string + ipv4 string + ipv6 string + }{ + {name: uniqueDockerIntegrationName("reploy-network-local-peer"), network: localNetwork, ipv4: localV4, ipv6: localV6}, + {name: uniqueDockerIntegrationName("reploy-network-public-peer"), network: publicNetwork, ipv4: publicV4, ipv6: publicV6}, + {name: uniqueDockerIntegrationName("reploy-network-public-exception-peer"), network: publicExceptionNetwork, ipv6: publicExceptionV6}, + {name: uniqueDockerIntegrationName("reploy-network-ambiguous-peer"), network: ambiguousNetwork, ipv4: ambiguousV4, ipv6: ambiguousV6}, + } { + if peer.network == publicNetwork && publicPeerName == "" { + publicPeerName = peer.name + } + args := []string{ + "run", "--detach", "--pull", "never", "--name", peer.name, + "--network", peer.network, + } + if peer.ipv4 != "" { + args = append(args, "--ip", peer.ipv4) + } + args = append(args, + "--ip6", peer.ipv6, + "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges=true", + "--mount", "type=bind,source="+helper+",target=/network-test,readonly", + "--entrypoint", "/network-test", image, "peer", + ) + runDockerIntegration(t, ctx, args...) + t.Cleanup(func() { + _ = exec.CommandContext(context.Background(), "docker", "container", "rm", "--force", peer.name).Run() + }) + } + time.Sleep(250 * time.Millisecond) + return localNetwork, publicNetwork, publicExceptionNetwork, ambiguousNetwork, + []string{localV4 + ":9090", "[" + localV6 + "]:9090"}, + []string{publicV4 + ":9090", "[" + publicV6 + "]:9090"}, "[" + ambiguousV6 + "]:9090", "[" + publicExceptionV6 + "]:9090", publicPeerName +} + +func createNetworkPolicyDirectResolver(t *testing.T, ctx context.Context, image string, helper string) string { + t.Helper() + name := uniqueDockerIntegrationName("reploy-network-direct-dns") + runDockerIntegration(t, ctx, + "run", "--detach", "--name", name, "--network", "bridge", + "--mount", "type=bind,source="+helper+",target=/network-test,readonly", + "--entrypoint", "/network-test", image, "peer", + ) + t.Cleanup(func() { _ = exec.CommandContext(context.Background(), "docker", "rm", "--force", name).Run() }) + address := strings.TrimSpace(runDockerIntegration(t, ctx, "inspect", "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", name)) + if net.ParseIP(address) == nil { + t.Fatalf("direct DNS peer address = %q", address) + } + return address +} + +func reserveNetworkPolicyIntegrationPort(t *testing.T) int { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + return listener.Addr().(*net.TCPAddr).Port +} diff --git a/internal/dockerdeploy/application_sandbox_plan.go b/internal/dockerdeploy/application_sandbox_plan.go index c9ec2f15..fe29f013 100644 --- a/internal/dockerdeploy/application_sandbox_plan.go +++ b/internal/dockerdeploy/application_sandbox_plan.go @@ -6,10 +6,13 @@ import ( "slices" "strconv" + "github.com/omry/reploy/internal/blueprint" "github.com/omry/reploy/internal/deploy" ) const applicationSeccompProfileBuiltinV1 = "builtin" +const applicationGooglePublicDNSPrimaryV1 = "8.8.8.8" +const applicationGooglePublicDNSSecondaryV1 = "8.8.4.4" type ApplicationKernelPolicyV1 struct { DropAllCapabilities bool @@ -26,18 +29,31 @@ type ApplicationKernelPolicyV1 struct { // renderer-specific flags. type ApplicationSandboxPlanV1 struct { RuntimeUser RuntimeUserPlan + Network ApplicationNetworkPolicyV1 ReadOnlyRoot bool TemporaryHome string StartupVerifier deploy.ApplicationStartupVerifierV1 Kernel ApplicationKernelPolicyV1 } +type ApplicationNetworkPolicyV1 struct { + Public blueprint.NetworkAccess + Local blueprint.NetworkAccess + Ambiguous blueprint.AmbiguousNetworkAccess +} + func newApplicationSandboxPlanV1(runtimeUser RuntimeUserPlan) ApplicationSandboxPlanV1 { + return newApplicationSandboxPlanWithNetworkV1(runtimeUser, blueprint.RuntimeNetwork{}) +} + +func newApplicationSandboxPlanWithNetworkV1(runtimeUser RuntimeUserPlan, network blueprint.RuntimeNetwork) ApplicationSandboxPlanV1 { if runtimeUser.LocalUser == "" { runtimeUser.LocalUser = runtimeLocalUserNameV1("", runtimeUser.UID) } + network = normalizeRuntimeNetworkV1(network) return ApplicationSandboxPlanV1{ RuntimeUser: runtimeUser, + Network: ApplicationNetworkPolicyV1{Public: network.Public, Local: network.Local, Ambiguous: network.Ambiguous}, ReadOnlyRoot: true, TemporaryHome: environmentTemporaryHome, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), @@ -51,10 +67,39 @@ func newApplicationSandboxPlanV1(runtimeUser RuntimeUserPlan) ApplicationSandbox } } +func normalizeRuntimeNetworkV1(network blueprint.RuntimeNetwork) blueprint.RuntimeNetwork { + if network.Public == "" { + network.Public = blueprint.NetworkAccessDeny + } + if network.Local == "" { + network.Local = blueprint.NetworkAccessDeny + } + if network.Ambiguous == "" { + network.Ambiguous = blueprint.AmbiguousNetworkAccessRequireBoth + } + return network +} + +func applicationDockerDNSResolversV1(network ApplicationNetworkPolicyV1) []string { + if network.Public == blueprint.NetworkAccessAllow && network.Local == blueprint.NetworkAccessDeny { + return []string{applicationGooglePublicDNSPrimaryV1, applicationGooglePublicDNSSecondaryV1} + } + return nil +} + func ValidateApplicationSandboxPlanV1(plan ApplicationSandboxPlanV1) error { if plan.RuntimeUser.UID < 0 || plan.RuntimeUser.GID < 0 { return fmt.Errorf("application sandbox requires a non-negative numeric UID and GID") } + if err := validateApplicationNetworkAccessV1("public", plan.Network.Public); err != nil { + return err + } + if err := validateApplicationNetworkAccessV1("local", plan.Network.Local); err != nil { + return err + } + if err := validateApplicationAmbiguousNetworkAccessV1(plan.Network.Ambiguous); err != nil { + return err + } wantUser := strconv.Itoa(plan.RuntimeUser.UID) + ":" + strconv.Itoa(plan.RuntimeUser.GID) if plan.RuntimeUser.DockerUser != wantUser { return fmt.Errorf("application sandbox Docker user must match its numeric UID and GID") @@ -110,6 +155,24 @@ func ValidateApplicationSandboxPlanV1(plan ApplicationSandboxPlanV1) error { return nil } +func validateApplicationNetworkAccessV1(name string, access blueprint.NetworkAccess) error { + switch access { + case blueprint.NetworkAccessDeny, blueprint.NetworkAccessAllow: + return nil + default: + return fmt.Errorf("application sandbox %s network access must be allow or deny", name) + } +} + +func validateApplicationAmbiguousNetworkAccessV1(access blueprint.AmbiguousNetworkAccess) error { + switch access { + case blueprint.AmbiguousNetworkAccessRequireBoth, blueprint.AmbiguousNetworkAccessAllow: + return nil + default: + return fmt.Errorf("application sandbox ambiguous network access must be require-both or allow") + } +} + func applicationLocalAccountV1(plan ApplicationSandboxPlanV1) (deploy.ApplicationLocalAccountV1, error) { if err := ValidateApplicationSandboxPlanV1(plan); err != nil { return deploy.ApplicationLocalAccountV1{}, err diff --git a/internal/dockerdeploy/application_sandbox_plan_test.go b/internal/dockerdeploy/application_sandbox_plan_test.go index 88a448d1..b811e99f 100644 --- a/internal/dockerdeploy/application_sandbox_plan_test.go +++ b/internal/dockerdeploy/application_sandbox_plan_test.go @@ -20,6 +20,7 @@ func TestApplicationRenderersConsumeCanonicalSandboxPlan(t *testing.T) { Sandbox: newApplicationSandboxPlanV1(RuntimeUserPlan{ UID: 501, GID: 20, SupplementaryGIDs: []int{33, 44}, DockerUser: "501:20", }), + Workload: &WorkloadExecutionPlan{Argv: []string{"/bin/true"}, Endpoints: map[string]EndpointExecutionPlan{}}, } persistent, err := RenderDockerInputs(plan, "demo") @@ -31,11 +32,13 @@ func TestApplicationRenderersConsumeCanonicalSandboxPlan(t *testing.T) { t.Fatal(err) } service := compose.Services["environment"] - if service.User != plan.Sandbox.RuntimeUser.DockerUser || !service.ReadOnly { + if service.User != "0:0" || !service.ReadOnly { t.Fatalf("persistent sandbox identity/read-only = user %q, read-only %t", service.User, service.ReadOnly) } - if !slices.Equal(service.GroupAdd, []string{"33", "44"}) || !slices.Equal(service.CapDrop, []string{"ALL"}) || !slices.Equal(service.SecurityOpt, []string{"no-new-privileges:true", "seccomp=builtin"}) { - t.Fatalf("persistent kernel sandbox = groups %#v, caps %#v, security %#v", service.GroupAdd, service.CapDrop, service.SecurityOpt) + if len(service.GroupAdd) != 0 || !slices.Equal(service.CapDrop, []string{"ALL"}) || + !slices.Equal(service.CapAdd, []string{"NET_ADMIN", "SETGID", "SETPCAP", "SETUID"}) || + !slices.Equal(service.SecurityOpt, []string{"no-new-privileges:true", "seccomp=builtin"}) { + t.Fatalf("persistent kernel sandbox = groups %#v, drop %#v, add %#v, security %#v", service.GroupAdd, service.CapDrop, service.CapAdd, service.SecurityOpt) } if service.Environment["HOME"] != plan.Sandbox.TemporaryHome || service.Environment["TMPDIR"] != plan.Sandbox.TemporaryHome { t.Fatalf("persistent sandbox environment = %#v", service.Environment) @@ -43,6 +46,20 @@ func TestApplicationRenderersConsumeCanonicalSandboxPlan(t *testing.T) { if !containsString(service.Tmpfs, temporaryHomeMountForPlan(plan)) { t.Fatalf("persistent sandbox temporary home = %#v", service.Tmpfs) } + baseOnly := plan + baseOnly.Workload = nil + baseRendered, err := RenderDockerInputs(baseOnly, "demo") + if err != nil { + t.Fatal(err) + } + var baseCompose composePlanDocument + if err := yaml.Unmarshal(baseRendered.Compose, &baseCompose); err != nil { + t.Fatal(err) + } + baseService := baseCompose.Services["environment"] + if baseService.User != plan.Sandbox.RuntimeUser.DockerUser || len(baseService.CapAdd) != 0 || !slices.Equal(baseService.GroupAdd, []string{"33", "44"}) { + t.Fatalf("base-only dormant service authority = user %q, groups %#v, caps %#v", baseService.User, baseService.GroupAdd, baseService.CapAdd) + } transient, err := TransientCommandSpec( plan, @@ -63,10 +80,10 @@ func TestApplicationRenderersConsumeCanonicalSandboxPlan(t *testing.T) { }) { t.Fatalf("transient sandbox environment = %#v", transient.Args) } - if !containsInOrder(transient.Args, []string{"--user", "501:20", "--cap-drop", "ALL"}) || - !containsInOrder(transient.Args, []string{"--group-add", "33", "--group-add", "44"}) || + if !containsInOrder(transient.Args, []string{"--user", "0:0", "--cap-drop", "ALL"}) || + !containsInOrder(transient.Args, []string{"--cap-add", "NET_ADMIN", "--cap-add", "SETGID", "--cap-add", "SETPCAP", "--cap-add", "SETUID"}) || !containsInOrder(transient.Args, []string{"--security-opt", "no-new-privileges=true", "--security-opt", "seccomp=builtin"}) || - !containsInOrder(transient.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "verify-exec", "--", "/bin/true"}) { + !containsInOrder(transient.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "sandbox-exec", "--uid", "501", "--gid", "20", "--groups", "33,44", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/bin/true"}) { t.Fatalf("transient sandbox runtime identity = %#v", transient.Args) } @@ -118,6 +135,122 @@ func TestApplicationSandboxPlanRejectsIdentityAndKernelEscapes(t *testing.T) { } } +func TestApplicationNetworkPolicyControlsOnlySetupCapability(t *testing.T) { + for _, test := range []struct { + name string + public blueprint.NetworkAccess + local blueprint.NetworkAccess + ambiguous blueprint.AmbiguousNetworkAccess + netAdmin bool + }{ + {name: "deny both", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, netAdmin: true}, + {name: "public only", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, netAdmin: true}, + {name: "local only", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessAllow, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, netAdmin: true}, + {name: "ambiguous escape hatch", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny, ambiguous: blueprint.AmbiguousNetworkAccessAllow, netAdmin: true}, + {name: "allow both", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessAllow, ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, netAdmin: true}, + } { + t.Run(test.name, func(t *testing.T) { + plan := newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 501, GID: 20, DockerUser: "501:20"}, + blueprint.RuntimeNetwork{Public: test.public, Local: test.local, Ambiguous: test.ambiguous}, + ) + capabilities := applicationSetupCapabilitiesV1(plan) + if containsString(capabilities, "NET_ADMIN") != test.netAdmin { + t.Fatalf("setup capabilities = %#v", capabilities) + } + argv := sandboxApplicationArgvV1(DockerExecutionPlan{ + Sandbox: plan, + Workload: &WorkloadExecutionPlan{Endpoints: map[string]EndpointExecutionPlan{ + "http": {ContainerPort: 8080}, + }}, + }, []string{"/bin/true"}, true, []int{8080}) + if !containsInOrder(argv, []string{"--public", string(test.public), "--local", string(test.local), "--ambiguous", string(test.ambiguous), "--inbound-tcp", "8080", "--", "/bin/true"}) { + t.Fatalf("sandbox argv = %#v", argv) + } + }) + } +} + +func TestApplicationDockerDNSResolversFollowNetworkPolicy(t *testing.T) { + for _, test := range []struct { + name string + public blueprint.NetworkAccess + local blueprint.NetworkAccess + want []string + }{ + {name: "deny both", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessDeny}, + {name: "public only", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessDeny, want: []string{"8.8.8.8", "8.8.4.4"}}, + {name: "local only", public: blueprint.NetworkAccessDeny, local: blueprint.NetworkAccessAllow}, + {name: "allow both", public: blueprint.NetworkAccessAllow, local: blueprint.NetworkAccessAllow}, + } { + t.Run(test.name, func(t *testing.T) { + plan := newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 501, GID: 20, DockerUser: "501:20"}, + blueprint.RuntimeNetwork{Public: test.public, Local: test.local}, + ) + if got := applicationDockerDNSResolversV1(plan.Network); !slices.Equal(got, test.want) { + t.Fatalf("Docker DNS resolvers = %#v, want %#v", got, test.want) + } + }) + } +} + +func TestApplicationRenderersConfigurePublicOnlyDNS(t *testing.T) { + plan := DockerExecutionPlan{ + EnvironmentID: "demo", DeploymentDir: t.TempDir(), Phase: blueprint.PhaseStaged, + Image: "reploy/demo:staging", ContainerName: "demo", NetworkName: "demo", + Sandbox: newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 501, GID: 20, DockerUser: "501:20"}, + blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessAllow, Local: blueprint.NetworkAccessDeny}, + ), + Workload: &WorkloadExecutionPlan{Argv: []string{"/bin/true"}, Endpoints: map[string]EndpointExecutionPlan{}}, + } + persistent, err := RenderDockerInputs(plan, "demo") + if err != nil { + t.Fatal(err) + } + var compose composePlanDocument + if err := yaml.Unmarshal(persistent.Compose, &compose); err != nil { + t.Fatal(err) + } + want := []string{"8.8.8.8", "8.8.4.4"} + if got := compose.Services["environment"].DNS; !slices.Equal(got, want) { + t.Fatalf("persistent DNS = %#v, want %#v", got, want) + } + transient, err := TransientCommandSpec(plan, ResolvedEnvironmentCommand{Argv: []string{"/bin/true"}}, nil, false, false) + if err != nil { + t.Fatal(err) + } + if !containsInOrder(transient.Args, []string{"--dns", "8.8.8.8", "--dns", "8.8.4.4"}) { + t.Fatalf("transient DNS arguments = %#v", transient.Args) + } +} + +func TestRootApplicationSetupCanClearSupplementaryGroups(t *testing.T) { + plan := newApplicationSandboxPlanV1(RuntimeUserPlan{UID: 0, GID: 0, DockerUser: "0:0"}) + capabilities := applicationSetupCapabilitiesV1(plan) + if !containsString(capabilities, "SETGID") || !containsString(capabilities, "SETPCAP") || !containsString(capabilities, "NET_ADMIN") || containsString(capabilities, "SETUID") { + t.Fatalf("root setup capabilities = %#v", capabilities) + } +} + +func TestTransientApplicationDoesNotInheritWorkloadEndpointGrants(t *testing.T) { + plan := DockerExecutionPlan{ + DeploymentDir: t.TempDir(), Image: "reploy/demo:staging", ContainerName: "demo", + Sandbox: newApplicationSandboxPlanV1(RuntimeUserPlan{UID: 501, GID: 20, DockerUser: "501:20"}), + Workload: &WorkloadExecutionPlan{Endpoints: map[string]EndpointExecutionPlan{ + "http": {ContainerPort: 8080}, + }}, + } + spec, err := TransientCommandSpec(plan, ResolvedEnvironmentCommand{Argv: []string{"/bin/true"}}, nil, false, false) + if err != nil { + t.Fatal(err) + } + if containsString(spec.Args, "--inbound-tcp") || containsString(spec.Args, "8080") { + t.Fatalf("transient command inherited workload endpoint grant: %#v", spec.Args) + } +} + func containsString(values []string, want string) bool { for _, value := range values { if value == want { diff --git a/internal/dockerdeploy/build_publication_test.go b/internal/dockerdeploy/build_publication_test.go index 00fcacb7..cce209ba 100644 --- a/internal/dockerdeploy/build_publication_test.go +++ b/internal/dockerdeploy/build_publication_test.go @@ -407,6 +407,7 @@ func publicationLockFixture(t *testing.T, dir string, imageChar string, configCh image := providers.RealizedImageV1{Digest: rendererDigest(imageChar), ConfigDigest: rendererDigest(configChar), RootFSSubject: runtimeRootFS} policy := deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{}, } policyDigest, err := deploy.RuntimePolicyDigestV1(policy) diff --git a/internal/dockerdeploy/command_execution.go b/internal/dockerdeploy/command_execution.go index 239fd200..2c9c3976 100644 --- a/internal/dockerdeploy/command_execution.go +++ b/internal/dockerdeploy/command_execution.go @@ -216,13 +216,16 @@ func transientContainerCommandSpecV1(operation string, container string, plan Do home := temporaryHomeForPlan(plan) args := []string{ operation, "--pull", "never", "--rm", "--name", container, - "--user", plan.Sandbox.RuntimeUser.DockerUser, + "--user", "0:0", "--cap-drop", "ALL", "--security-opt", "no-new-privileges=true", "--security-opt", "seccomp=" + plan.Sandbox.Kernel.SeccompProfile, } - for _, group := range dockerSupplementaryGroupsV1(plan.Sandbox.RuntimeUser.SupplementaryGIDs) { - args = append(args, "--group-add", group) + for _, capability := range applicationSetupCapabilitiesV1(plan.Sandbox) { + args = append(args, "--cap-add", capability) + } + for _, resolver := range applicationDockerDNSResolversV1(plan.Sandbox.Network) { + args = append(args, "--dns", resolver) } if plan.Sandbox.ReadOnlyRoot { args = append(args, "--read-only") @@ -286,7 +289,7 @@ func transientContainerCommandSpecV1(operation string, container string, plan Do "--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, ) - args = append(args, verifiedApplicationArgvV1(command.Argv)...) + args = append(args, sandboxApplicationArgvV1(plan, command.Argv, true, []int{})...) return CommandSpec{Name: "docker", Args: args}, nil } diff --git a/internal/dockerdeploy/command_execution_test.go b/internal/dockerdeploy/command_execution_test.go index 6e1e44ca..97ef6681 100644 --- a/internal/dockerdeploy/command_execution_test.go +++ b/internal/dockerdeploy/command_execution_test.go @@ -85,7 +85,7 @@ func TestTransientAndShellCommandsUseDockerExecArgv(t *testing.T) { t.Fatal(err) } joined := strings.Join(spec.Args, "|") - if strings.Contains(joined, "sh|-c") || !containsInOrder(spec.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "verify-exec", "--", "/opt/demo", ";rm", "$(touch pwned)"}) { + if strings.Contains(joined, "sh|-c") || !containsInOrder(spec.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "sandbox-exec", "--uid", "501", "--gid", "20", "--groups", "33,44", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/demo", ";rm", "$(touch pwned)"}) { t.Fatalf("spec = %#v", spec) } if !containsInOrder(spec.Args, []string{"--mount", "type=bind,source=" + outputDir + ",target=" + runtimeOutputRoot, "--env", runtimeOutputFileVariable + "=" + runtimeOutputRoot + "/output"}) { @@ -94,13 +94,13 @@ func TestTransientAndShellCommandsUseDockerExecArgv(t *testing.T) { if !containsAdjacent(spec.Args, "--pull", "never") { t.Fatalf("transient command permits image pulls: %#v", spec.Args) } - if !containsInOrder(spec.Args, []string{"--user", "501:20", "--cap-drop", "ALL"}) || - !containsInOrder(spec.Args, []string{"--group-add", "33", "--group-add", "44"}) || - !containsInOrder(spec.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "verify-exec", "--", "/opt/demo", ";rm", "$(touch pwned)"}) { + if !containsInOrder(spec.Args, []string{"--user", "0:0", "--cap-drop", "ALL"}) || + !containsInOrder(spec.Args, []string{"--cap-add", "NET_ADMIN", "--cap-add", "SETGID", "--cap-add", "SETPCAP", "--cap-add", "SETUID"}) || + !containsInOrder(spec.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "sandbox-exec", "--uid", "501", "--gid", "20", "--groups", "33,44", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/demo", ";rm", "$(touch pwned)"}) { t.Fatalf("transient command does not start directly with its final identity and command: %#v", spec.Args) } shell := ShellCommandSpec(plan, true, true) - if !strings.Contains(strings.Join(shell.Args, " "), "--interactive --tty") || !containsInOrder(shell.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "verify-exec", "--", "/bin/sh"}) { + if !strings.Contains(strings.Join(shell.Args, " "), "--interactive --tty") || !containsInOrder(shell.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "sandbox-exec", "--uid", "501", "--gid", "20", "--groups", "33,44", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/bin/sh"}) { t.Fatalf("shell = %#v", shell) } if !containsInOrder(shell.Args, []string{"--read-only", "--tmpfs", transientHomeMountForPlan(plan)}) || @@ -204,7 +204,7 @@ func TestPlanTransientContainerExecutionV1SeparatesCreateStartAndCleanup(t *test t.Fatalf("create prefix = %#v", execution.Create.Args) } if !containsInOrder(execution.Create.Args, []string{"--interactive", "--tty"}) || - !containsInOrder(execution.Create.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "verify-exec", "--", "/opt/demo", "export"}) { + !containsInOrder(execution.Create.Args, []string{"--entrypoint", plan.Sandbox.StartupVerifier.Path, plan.Image, "sandbox-exec", "--uid", "501", "--gid", "20", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/demo", "export"}) { t.Fatalf("create args = %#v", execution.Create.Args) } if !reflect.DeepEqual(execution.Start.Args, []string{"start", "--attach", "--interactive", wantContainer}) { diff --git a/internal/dockerdeploy/current_workload_lifecycle.go b/internal/dockerdeploy/current_workload_lifecycle.go index 34c2c80e..15ddbd53 100644 --- a/internal/dockerdeploy/current_workload_lifecycle.go +++ b/internal/dockerdeploy/current_workload_lifecycle.go @@ -44,7 +44,7 @@ type currentWorkloadLifecycleBackendV1 struct { cleanup func(string) CommandSpec runTemporary func(temporaryCommandRunner, CommandSpec, CommandSpec, RunOptions) error runCommand func(CommandSpec, RunOptions) error - inject func(context.Context, string, string, privateWorkloadEnvironmentV1, RunOptions, commandRunner) error + inject func(context.Context, string, string, ApplicationSandboxPlanV1, privateWorkloadEnvironmentV1, RunOptions, commandRunner) error readiness func(context.Context, EndpointExecutionPlan, func(context.Context) error) error serviceCheck func(string, string, time.Duration) error } @@ -222,6 +222,7 @@ func runCurrentWorkloadLifecycleV1(ctx context.Context, input CurrentWorkloadLif startCtx, spec.Name, input.Plan.Docker.ContainerName, + input.Plan.Docker.Sandbox, input.PrivateEnvironment, runOptions, backend.runCommand, diff --git a/internal/dockerdeploy/current_workload_lifecycle_test.go b/internal/dockerdeploy/current_workload_lifecycle_test.go index bfac86ed..571e72e9 100644 --- a/internal/dockerdeploy/current_workload_lifecycle_test.go +++ b/internal/dockerdeploy/current_workload_lifecycle_test.go @@ -469,7 +469,7 @@ func currentWorkloadLifecycleTestBackend(t *testing.T, lifecycle LifecyclePlan, *order = append(*order, "run "+spec.Name) return nil }, - inject: func(context.Context, string, string, privateWorkloadEnvironmentV1, RunOptions, commandRunner) error { + inject: func(context.Context, string, string, ApplicationSandboxPlanV1, privateWorkloadEnvironmentV1, RunOptions, commandRunner) error { *order = append(*order, "inject private environment") return nil }, diff --git a/internal/dockerdeploy/execution_plan.go b/internal/dockerdeploy/execution_plan.go index 12eb8b23..bb62a369 100644 --- a/internal/dockerdeploy/execution_plan.go +++ b/internal/dockerdeploy/execution_plan.go @@ -143,7 +143,7 @@ func PlanDockerExecution(document blueprint.Document, context DockerPlanContext) if err != nil { return DockerExecutionPlan{}, err } - plan.Sandbox = newApplicationSandboxPlanV1(runtimeUser) + plan.Sandbox = newApplicationSandboxPlanWithNetworkV1(runtimeUser, document.Environment.Runtime.Network) if err := ValidateApplicationSandboxPlanV1(plan.Sandbox); err != nil { return DockerExecutionPlan{}, err } diff --git a/internal/dockerdeploy/execution_render.go b/internal/dockerdeploy/execution_render.go index 16c71277..e7a55c6a 100644 --- a/internal/dockerdeploy/execution_render.go +++ b/internal/dockerdeploy/execution_render.go @@ -2,6 +2,7 @@ package dockerdeploy import ( "fmt" + "slices" "sort" "strconv" "strings" @@ -52,6 +53,7 @@ type composePlanService struct { User string `yaml:"user"` GroupAdd []string `yaml:"group_add"` CapDrop []string `yaml:"cap_drop"` + CapAdd []string `yaml:"cap_add"` SecurityOpt []string `yaml:"security_opt"` Restart string `yaml:"restart,omitempty"` Entrypoint []string `yaml:"entrypoint,omitempty,flow"` @@ -62,6 +64,7 @@ type composePlanService struct { ReadOnly bool `yaml:"read_only"` Environment map[string]string `yaml:"environment"` Tmpfs []string `yaml:"tmpfs"` + DNS []string `yaml:"dns,omitempty"` } type composePlanMount struct { @@ -108,10 +111,14 @@ func RenderDockerInputs(plan DockerExecutionPlan, controlScript string) (DockerR CapDrop: []string{"ALL"}, SecurityOpt: []string{"no-new-privileges:true", "seccomp=" + plan.Sandbox.Kernel.SeccompProfile}, ReadOnly: plan.Sandbox.ReadOnlyRoot, Environment: temporaryEnvironmentForPlan(plan), Tmpfs: []string{temporaryHomeMountForPlan(plan)}, + DNS: applicationDockerDNSResolversV1(plan.Sandbox.Network), } if plan.Workload != nil { + service.User = "0:0" + service.GroupAdd = []string{} + service.CapAdd = applicationSetupCapabilitiesV1(plan.Sandbox) service.Entrypoint = []string{plan.Sandbox.StartupVerifier.Path} - service.Command = verifiedApplicationArgvV1(plan.Workload.Argv) + service.Command = sandboxApplicationArgvV1(plan, plan.Workload.Argv, true, applicationInboundTCPPortsV1(plan)) } if plan.PrivateEnvironment { if plan.Workload == nil { @@ -123,7 +130,7 @@ func RenderDockerInputs(plan DockerExecutionPlan, controlScript string) (DockerR composeLauncher := strings.ReplaceAll(privateWorkloadEnvironmentLauncherV1, "$", "$$") launcher := []string{"/bin/sh", "-c", composeLauncher, "reploy-private-environment"} launcher = append(launcher, plan.Workload.Argv...) - service.Command = verifiedApplicationArgvV1(launcher) + service.Command = sandboxApplicationArgvV1(plan, launcher, true, applicationInboundTCPPortsV1(plan)) service.StdinOpen = true } volumes := map[string]any{} @@ -213,11 +220,63 @@ func RenderDockerInputs(plan DockerExecutionPlan, controlScript string) (DockerR }, nil } -func verifiedApplicationArgvV1(argv []string) []string { - result := []string{"verify-exec", "--"} +func sandboxApplicationArgvV1(plan DockerExecutionPlan, argv []string, installNetwork bool, inboundTCP []int) []string { + mode := "restricted-exec" + if installNetwork { + mode = "sandbox-exec" + } + user := plan.Sandbox.RuntimeUser + result := []string{mode, "--uid", strconv.Itoa(user.UID), "--gid", strconv.Itoa(user.GID)} + if len(user.SupplementaryGIDs) != 0 { + result = append(result, "--groups", joinDecimalValuesV1(user.SupplementaryGIDs)) + } + if installNetwork { + result = append(result, + "--public", string(plan.Sandbox.Network.Public), + "--local", string(plan.Sandbox.Network.Local), + "--ambiguous", string(plan.Sandbox.Network.Ambiguous), + ) + ports := append([]int{}, inboundTCP...) + sort.Ints(ports) + ports = slices.Compact(ports) + if len(ports) != 0 { + result = append(result, "--inbound-tcp", joinDecimalValuesV1(ports)) + } + } + result = append(result, "--") return append(result, argv...) } +func applicationSetupCapabilitiesV1(plan ApplicationSandboxPlanV1) []string { + result := []string{"NET_ADMIN", "SETGID", "SETPCAP"} + if plan.RuntimeUser.UID != 0 { + result = append(result, "SETUID") + } + sort.Strings(result) + return result +} + +func applicationInboundTCPPortsV1(plan DockerExecutionPlan) []int { + if plan.Workload == nil { + return []int{} + } + ports := make([]int, 0, len(plan.Workload.Endpoints)) + for _, endpoint := range plan.Workload.Endpoints { + ports = append(ports, endpoint.ContainerPort) + } + sort.Ints(ports) + ports = slices.Compact(ports) + return ports +} + +func joinDecimalValuesV1(values []int) string { + items := make([]string, len(values)) + for index, value := range values { + items[index] = strconv.Itoa(value) + } + return strings.Join(items, ",") +} + func temporaryHomeForPlan(plan DockerExecutionPlan) string { return plan.Sandbox.TemporaryHome } diff --git a/internal/dockerdeploy/execution_render_test.go b/internal/dockerdeploy/execution_render_test.go index d4da5db2..c7c77476 100644 --- a/internal/dockerdeploy/execution_render_test.go +++ b/internal/dockerdeploy/execution_render_test.go @@ -30,7 +30,7 @@ func TestRenderDockerInputsFromResolvedPlan(t *testing.T) { if compose != normalizedGolden { t.Fatalf("compose golden mismatch\nactual:\n%s\nwant:\n%s", compose, wantGolden) } - for _, want := range []string{"image: reploy/demo:staging", "pull_policy: never", `user: "501:20"`, "cap_drop:", "- ALL", "no-new-privileges:true", "seccomp=builtin", "read_only: true", "HOME: /mnt/reploy-home", "TMPDIR: /mnt/reploy-home", "- /mnt/reploy-home:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=501,gid=20", "type: bind", "127.0.0.1:18080:8080", "/opt/reploy/python/bin/demo", "name: demo-staging-abcd"} { + for _, want := range []string{"image: reploy/demo:staging", "pull_policy: never", `user: "0:0"`, "cap_drop:", "- ALL", "cap_add:", "- NET_ADMIN", "- SETGID", "- SETPCAP", "- SETUID", "sandbox-exec", "--public", "deny", "--local", "deny", "--inbound-tcp", "no-new-privileges:true", "seccomp=builtin", "read_only: true", "HOME: /mnt/reploy-home", "TMPDIR: /mnt/reploy-home", "- /mnt/reploy-home:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=501,gid=20", "type: bind", "127.0.0.1:18080:8080", "/opt/reploy/python/bin/demo", "name: demo-staging-abcd"} { if !strings.Contains(compose, want) { t.Fatalf("compose missing %q:\n%s", want, compose) } diff --git a/internal/dockerdeploy/full_validation_test.go b/internal/dockerdeploy/full_validation_test.go index faf1f333..4687fd62 100644 --- a/internal/dockerdeploy/full_validation_test.go +++ b/internal/dockerdeploy/full_validation_test.go @@ -27,6 +27,7 @@ func fullValidationInput(t *testing.T, digestChar string) FullImageValidationInp Image: request.Source, Profiles: []providers.RequirementProfile{}, Outputs: []providers.RealizedOutput{}, RuntimePolicy: deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{}, }, } diff --git a/internal/dockerdeploy/installed_service_container.go b/internal/dockerdeploy/installed_service_container.go index f4653d8b..9c3ddc57 100644 --- a/internal/dockerdeploy/installed_service_container.go +++ b/internal/dockerdeploy/installed_service_container.go @@ -103,6 +103,7 @@ func RunInstalledServiceContainerV1(ctx context.Context, deploymentDir string, a start, cleanup, plan.Docker.ContainerName, + plan.Docker.Sandbox, environment, options, runDockerCommand, diff --git a/internal/dockerdeploy/prepared_python_graph_reuse_test.go b/internal/dockerdeploy/prepared_python_graph_reuse_test.go index 24425241..6ed01e16 100644 --- a/internal/dockerdeploy/prepared_python_graph_reuse_test.go +++ b/internal/dockerdeploy/prepared_python_graph_reuse_test.go @@ -463,6 +463,7 @@ func newPreparedPythonGraphReuseFixtureWithManifest(t *testing.T, sourceManifest Catalog: append([]providers.RealizedOutput{}, request.EarlierCatalog...), RuntimePolicy: deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{}, }, RuntimeLayer: testApplicationRuntimeLayerV1(t, request.Platform, resultImage, providers.RealizedImageV1{ @@ -581,6 +582,7 @@ func newPreparedAPTGraphReuseFixture(t *testing.T) ( Catalog: []providers.RealizedOutput{}, RuntimePolicy: deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{}, }, RuntimeLayer: testApplicationRuntimeLayerV1(t, descriptor.Platform, resultImage, providers.RealizedImageV1{ diff --git a/internal/dockerdeploy/private_workload_environment_inject.go b/internal/dockerdeploy/private_workload_environment_inject.go index 978cf400..ffe50007 100644 --- a/internal/dockerdeploy/private_workload_environment_inject.go +++ b/internal/dockerdeploy/private_workload_environment_inject.go @@ -20,6 +20,7 @@ func injectPrivateWorkloadEnvironmentV1( ctx context.Context, dockerPath string, containerName string, + sandbox ApplicationSandboxPlanV1, environment privateWorkloadEnvironmentV1, options RunOptions, run commandRunner, @@ -47,13 +48,12 @@ func injectPrivateWorkloadEnvironmentV1( "exec", "-i", containerName, - "/bin/sh", - "-c", - privateWorkloadEnvironmentRelayV1, - "reploy-private-environment", - privateWorkloadEnvironmentFIFOPathV1, }, } + relay := []string{"/bin/sh", "-c", privateWorkloadEnvironmentRelayV1, "reploy-private-environment", privateWorkloadEnvironmentFIFOPathV1} + wrapperPlan := DockerExecutionPlan{Sandbox: sandbox} + spec.Args = append(spec.Args, sandbox.StartupVerifier.Path) + spec.Args = append(spec.Args, sandboxApplicationArgvV1(wrapperPlan, relay, false, []int{})...) if err := run(spec, runOptions); err != nil { return fmt.Errorf("inject private workload environment through one-shot FIFO relay: %w", err) } @@ -65,6 +65,7 @@ func startAndInjectPrivateWorkloadEnvironmentV1( start CommandSpec, cleanup CommandSpec, containerName string, + sandbox ApplicationSandboxPlanV1, environment privateWorkloadEnvironmentV1, options RunOptions, run commandRunner, @@ -78,7 +79,7 @@ func startAndInjectPrivateWorkloadEnvironmentV1( if !environment.Present { return nil } - if err := injectPrivateWorkloadEnvironmentV1(ctx, start.Name, containerName, environment, options, run); err != nil { + if err := injectPrivateWorkloadEnvironmentV1(ctx, start.Name, containerName, sandbox, environment, options, run); err != nil { cleanupOptions := options cleanupOptions.Context = context.WithoutCancel(ctx) cleanupOptions.Stdin = nil diff --git a/internal/dockerdeploy/private_workload_environment_integration_test.go b/internal/dockerdeploy/private_workload_environment_integration_test.go index d567808f..76ea4738 100644 --- a/internal/dockerdeploy/private_workload_environment_integration_test.go +++ b/internal/dockerdeploy/private_workload_environment_integration_test.go @@ -115,6 +115,7 @@ printf 'private-mask-pass\n'`, expectedTokenDigest) start, cleanup, container, + plan.Sandbox, environment, RunOptions{}, runCommandWithoutDockerPreflight, diff --git a/internal/dockerdeploy/private_workload_environment_test.go b/internal/dockerdeploy/private_workload_environment_test.go index 4678e304..71454b4f 100644 --- a/internal/dockerdeploy/private_workload_environment_test.go +++ b/internal/dockerdeploy/private_workload_environment_test.go @@ -225,7 +225,7 @@ func TestRenderDockerInputsUsesSecretFreePrivateLauncher(t *testing.T) { compose := string(rendered.Compose) for _, want := range []string{ "stdin_open: true", "reploy_private_environment_ready", "/opt/demo", "serve", - "entrypoint: [/reploy-probe]", "command: [verify-exec, --, /bin/sh, -c", + "entrypoint: [/reploy-probe]", "command: [sandbox-exec, --uid", "--public, deny", "--local, deny", "/bin/sh, -c", "source: /dev/null", "target: /deployment/.env", "read_only: true", "/deployment/.reploy:" + privateRuntimeDirectoryMaskOptionsV1, } { @@ -249,7 +249,8 @@ func TestInjectPrivateWorkloadEnvironmentV1UsesOnlyStdin(t *testing.T) { environment := privateWorkloadEnvironmentV1{Present: true, Payload: []byte("TOKEN=private value\n\n")} var gotSpec CommandSpec var gotInput []byte - err := injectPrivateWorkloadEnvironmentV1(t.Context(), "/usr/bin/docker", "demo", environment, RunOptions{}, func(spec CommandSpec, options RunOptions) error { + sandbox := testApplicationSandboxPlanV1(1000, 1000) + err := injectPrivateWorkloadEnvironmentV1(t.Context(), "/usr/bin/docker", "demo", sandbox, environment, RunOptions{}, func(spec CommandSpec, options RunOptions) error { gotSpec = spec var err error gotInput, err = readAllForPrivateEnvironmentTest(options) @@ -263,7 +264,8 @@ func TestInjectPrivateWorkloadEnvironmentV1UsesOnlyStdin(t *testing.T) { t.Fatalf("command contains secret: %#v", gotSpec) } if !reflect.DeepEqual(gotSpec.Args, []string{ - "exec", "-i", "demo", "/bin/sh", "-c", privateWorkloadEnvironmentRelayV1, + "exec", "-i", "demo", sandbox.StartupVerifier.Path, + "restricted-exec", "--uid", "1000", "--gid", "1000", "--", "/bin/sh", "-c", privateWorkloadEnvironmentRelayV1, "reploy-private-environment", privateWorkloadEnvironmentFIFOPathV1, }) { t.Fatalf("relay command = %#v", gotSpec) @@ -291,6 +293,7 @@ func TestStartAndInjectPrivateWorkloadEnvironmentV1CleansFailedContainer(t *test CommandSpec{Name: "docker", Args: []string{"compose", "up"}}, CommandSpec{Name: "docker", Args: []string{"compose", "down"}}, "demo", + testApplicationSandboxPlanV1(1000, 1000), environment, RunOptions{}, func(spec CommandSpec, _ RunOptions) error { @@ -304,7 +307,7 @@ func TestStartAndInjectPrivateWorkloadEnvironmentV1CleansFailedContainer(t *test if err == nil || !strings.Contains(err.Error(), "one-shot FIFO relay") { t.Fatalf("error = %v", err) } - if len(order) != 3 || order[0] != "compose up" || !strings.HasPrefix(order[1], "exec -i demo /bin/sh -c ") || order[2] != "compose down" { + if len(order) != 3 || order[0] != "compose up" || !strings.HasPrefix(order[1], "exec -i demo "+deploy.ApplicationStartupVerifierPathV1+" restricted-exec ") || order[2] != "compose down" { t.Fatalf("order = %#v", order) } } @@ -438,6 +441,12 @@ func TestPrivateWorkloadEnvironmentRealDockerIsolation(t *testing.T) { t.Skip("set REPLOY_DOCKER_INTEGRATION=1 to run the real-Docker private environment test") } container := "reploy-private-env-test-" + strconv.Itoa(os.Getpid()) + platform, err := blueprint.ParsePlatform("linux/" + runtime.GOARCH) + if err != nil { + t.Fatal(err) + } + probe := buildIntegrationProbeWorkspace(t, platform) + sandbox := testApplicationSandboxPlanV1(65532, 65532) run := func(spec CommandSpec, options RunOptions) error { options.Context = t.Context() return runCommandWithoutDockerPreflight(spec, options) @@ -451,11 +460,14 @@ func TestPrivateWorkloadEnvironmentRealDockerIsolation(t *testing.T) { }) script := privateWorkloadEnvironmentLauncherV1 create := CommandSpec{Name: "docker", Args: []string{ - "create", "--name", container, "-i", "--user", "65532:65532", - "--tmpfs", environmentTemporaryHome + ":rw,noexec,nosuid,nodev,size=64m,mode=1777", + "create", "--name", container, "-i", "--user", "0:0", + "--cap-drop", "ALL", "--cap-add", "NET_ADMIN", "--cap-add", "SETGID", "--cap-add", "SETPCAP", "--cap-add", "SETUID", + "--security-opt", "no-new-privileges=true", + "--tmpfs", environmentTemporaryHome + ":rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=65532,gid=65532", "--env", "HOME=" + environmentTemporaryHome, - "--entrypoint", "/bin/sh", "python:3.11-slim", - "-c", script, "reploy-private-environment", + "--mount", "type=bind,source=" + probe.HostExecutable + ",target=" + sandbox.StartupVerifier.Path + ",readonly", + "--entrypoint", sandbox.StartupVerifier.Path, "python:3.11-slim", + "sandbox-exec", "--uid", "65532", "--gid", "65532", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/bin/sh", "-c", script, "reploy-private-environment", "python", "-c", `import os,time; name="REPLOY_"+"PRIVATE_"+"TEST"; value=os.environ.get(name); data=os.read(0, 1); print("ENV_PRESENT", value is not None, "LENGTH", len(value or ""), "STDIN_EOF", data == b"", flush=True); time.sleep(20)`, }} if err := run(create, RunOptions{}); err != nil { @@ -467,7 +479,7 @@ func TestPrivateWorkloadEnvironmentRealDockerIsolation(t *testing.T) { secret := "private-docker-test-value" environment := privateWorkloadEnvironmentV1{Present: true, Payload: []byte("REPLOY_PRIVATE_TEST=" + secret + "\n\n")} started := time.Now() - if err := injectPrivateWorkloadEnvironmentV1(t.Context(), "docker", container, environment, RunOptions{}, run); err != nil { + if err := injectPrivateWorkloadEnvironmentV1(t.Context(), "docker", container, sandbox, environment, RunOptions{}, run); err != nil { t.Fatal(err) } if elapsed := time.Since(started); elapsed > 5*time.Second { diff --git a/internal/dockerdeploy/provider_graph_validation_test.go b/internal/dockerdeploy/provider_graph_validation_test.go index df8a2344..02562341 100644 --- a/internal/dockerdeploy/provider_graph_validation_test.go +++ b/internal/dockerdeploy/provider_graph_validation_test.go @@ -80,6 +80,7 @@ func TestPrepareProviderGraphValidationInspectsBaseOnlyGraph(t *testing.T) { } policy := deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{}, } result, err := prepareProviderGraphValidation( diff --git a/internal/dockerdeploy/provider_install_host_execute.go b/internal/dockerdeploy/provider_install_host_execute.go index 380f4ca2..6e9a5f72 100644 --- a/internal/dockerdeploy/provider_install_host_execute.go +++ b/internal/dockerdeploy/provider_install_host_execute.go @@ -43,6 +43,7 @@ func startProviderInstallHostV1(ctx context.Context, plan providerInstallationPl commands.Start, cleanup, plan.Docker.ContainerName, + plan.Docker.Sandbox, environment, options, runCommandWithoutDockerPreflight, diff --git a/internal/dockerdeploy/runtime_host_preflight_test.go b/internal/dockerdeploy/runtime_host_preflight_test.go index 383ad3a9..40aa97be 100644 --- a/internal/dockerdeploy/runtime_host_preflight_test.go +++ b/internal/dockerdeploy/runtime_host_preflight_test.go @@ -234,8 +234,9 @@ func TestRuntimeHostSourcesV1IncludesOnlyBindAndExplicitOutputMounts(t *testing. func runtimeHostPolicy(mounts []deploy.RuntimeMountV1) deploy.RuntimePolicyV1 { return deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessDeny, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, ProtectedPaths: []deploy.ProtectedPathV1{}, Plans: []deploy.RuntimePlanV1{{ - ID: "command/check", Mounts: mounts, Executables: []providers.QualifiedOutput{}, + ID: "command/check", InboundTCP: []string{}, Mounts: mounts, Executables: []providers.QualifiedOutput{}, }}, } } diff --git a/internal/dockerdeploy/runtime_logs.go b/internal/dockerdeploy/runtime_logs.go index 62c3f433..f14a2c92 100644 --- a/internal/dockerdeploy/runtime_logs.go +++ b/internal/dockerdeploy/runtime_logs.go @@ -83,10 +83,16 @@ func extractRuntimeStartupLogDiagnostics(logs string) runtimeStartupLogDiagnosti } continue } + cleaned := cleanRuntimeLogSnippetLine(line) + if strings.HasPrefix(cleaned, "reploy-probe: sandbox-exec:") { + diagnostics.Failure = "application sandbox setup failed" + lines = append(lines, cleaned) + continue + } if !capturing { continue } - if cleaned := cleanRuntimeLogSnippetLine(line); cleaned != "" { + if cleaned != "" { lines = append(lines, cleaned) } } diff --git a/internal/dockerdeploy/runtime_plan.go b/internal/dockerdeploy/runtime_plan.go index aa8d054f..b8b09c66 100644 --- a/internal/dockerdeploy/runtime_plan.go +++ b/internal/dockerdeploy/runtime_plan.go @@ -26,6 +26,19 @@ func RuntimePlansV1(document blueprint.Document, dockerPlan DockerExecutionPlan) if err := ValidateApplicationSandboxPlanV1(dockerPlan.Sandbox); err != nil { return nil, fmt.Errorf("runtime application sandbox: %w", err) } + resolvedNetwork := normalizeRuntimeNetworkV1(document.Environment.Runtime.Network) + wantNetwork := ApplicationNetworkPolicyV1{ + Public: resolvedNetwork.Public, + Local: resolvedNetwork.Local, + Ambiguous: resolvedNetwork.Ambiguous, + } + if dockerPlan.Sandbox.Network != wantNetwork { + return nil, fmt.Errorf("runtime application network policy does not match the resolved blueprint") + } + workloadInboundTCP, err := runtimeWorkloadInboundTCPV1(document, dockerPlan) + if err != nil { + return nil, err + } baseMounts, err := runtimeMountsV1(dockerPlan) if err != nil { return nil, err @@ -34,7 +47,7 @@ func RuntimePlansV1(document blueprint.Document, dockerPlan DockerExecutionPlan) Destination: temporaryHomeForPlan(dockerPlan), SourceKind: deploy.RuntimeMountSourceGenerated, }) plans := []deploy.RuntimePlanV1{{ - ID: runtimeShellPlanID, Mounts: cloneRuntimeMountsV1(withHome), Executables: []providers.QualifiedOutput{}, + ID: runtimeShellPlanID, InboundTCP: []string{}, Mounts: cloneRuntimeMountsV1(withHome), Executables: []providers.QualifiedOutput{}, }} commandNames, err := runtimeTransientCommandNamesV1(document) @@ -49,11 +62,11 @@ func RuntimePlansV1(document blueprint.Document, dockerPlan DockerExecutionPlan) } executables := []providers.QualifiedOutput{output} plans = append(plans, deploy.RuntimePlanV1{ - ID: runtimeCommandPlanID(name, false), Mounts: cloneRuntimeMountsV1(withHome), Executables: executables, + ID: runtimeCommandPlanID(name, false), InboundTCP: []string{}, Mounts: cloneRuntimeMountsV1(withHome), Executables: executables, }) if command.NativeCommand || command.DeployedCommand { plans = append(plans, deploy.RuntimePlanV1{ - ID: runtimeCommandPlanID(name, true), + ID: runtimeCommandPlanID(name, true), InboundTCP: []string{}, Mounts: appendRuntimeMountV1(withHome, deploy.RuntimeMountV1{ Destination: runtimeOutputRoot, SourceKind: deploy.RuntimeMountSourceDirectory, }), @@ -68,7 +81,7 @@ func RuntimePlansV1(document blueprint.Document, dockerPlan DockerExecutionPlan) return nil, fmt.Errorf("runtime workload: %w", err) } plans = append(plans, deploy.RuntimePlanV1{ - ID: runtimeWorkloadPlanID, Mounts: cloneRuntimeMountsV1(withHome), + ID: runtimeWorkloadPlanID, InboundTCP: workloadInboundTCP, Mounts: cloneRuntimeMountsV1(withHome), Executables: []providers.QualifiedOutput{output}, }) } @@ -77,6 +90,24 @@ func RuntimePlansV1(document blueprint.Document, dockerPlan DockerExecutionPlan) return plans, nil } +func runtimeWorkloadInboundTCPV1(document blueprint.Document, dockerPlan DockerExecutionPlan) ([]string, error) { + if document.Environment.Workload == nil { + return []string{}, nil + } + if len(document.Environment.Workload.Endpoints) != len(dockerPlan.Workload.Endpoints) { + return nil, fmt.Errorf("runtime workload endpoints do not match the resolved Docker plan") + } + ports := make([]int, 0, len(document.Environment.Workload.Endpoints)) + for name, endpoint := range document.Environment.Workload.Endpoints { + planned, found := dockerPlan.Workload.Endpoints[name] + if !found || planned.ContainerPort != endpoint.Port { + return nil, fmt.Errorf("runtime workload endpoint %q does not match the resolved Docker plan", name) + } + ports = append(ports, endpoint.Port) + } + return deploy.CanonicalRuntimeInboundTCPV1(ports), nil +} + func runtimeCommandPlanID(commandName string, output bool) string { id := "command/" + commandName if output { diff --git a/internal/dockerdeploy/runtime_plan_test.go b/internal/dockerdeploy/runtime_plan_test.go index 31654b39..2d45f33a 100644 --- a/internal/dockerdeploy/runtime_plan_test.go +++ b/internal/dockerdeploy/runtime_plan_test.go @@ -94,6 +94,53 @@ func TestRuntimePlansV1RejectsWorkloadPlanMismatch(t *testing.T) { } } +func TestRuntimePlansV1RejectsNetworkPolicyMismatch(t *testing.T) { + document := runtimePlanDocument() + plan := DockerExecutionPlan{ + Workload: &WorkloadExecutionPlan{}, + Sandbox: newApplicationSandboxPlanWithNetworkV1( + RuntimeUserPlan{UID: 1000, GID: 1000, DockerUser: "1000:1000"}, + blueprint.RuntimeNetwork{Public: blueprint.NetworkAccessAllow, Local: blueprint.NetworkAccessDeny, Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth}, + ), + } + _, err := RuntimePlansV1(document, plan) + if err == nil || !strings.Contains(err.Error(), "network policy does not match") { + t.Fatalf("network policy mismatch error = %v", err) + } +} + +func TestRuntimePlansV1LocksInboundTCPOnlyForWorkload(t *testing.T) { + document := runtimePlanDocument() + document.Environment.Workload.Endpoints = map[string]blueprint.Endpoint{ + "http": {Scheme: "http", Port: 8080}, + "admin": {Scheme: "http", Port: 8081}, + } + dockerPlan := DockerExecutionPlan{ + Sandbox: testApplicationSandboxPlanV1(1000, 1000), + Workload: &WorkloadExecutionPlan{Endpoints: map[string]EndpointExecutionPlan{ + "http": {ContainerPort: 8080}, + "admin": {ContainerPort: 8081}, + }}, + } + plans, err := RuntimePlansV1(document, dockerPlan) + if err != nil { + t.Fatal(err) + } + for _, plan := range plans { + want := []string{} + if plan.ID == runtimeWorkloadPlanID { + want = []string{"8080", "8081"} + } + if !reflect.DeepEqual(plan.InboundTCP, want) { + t.Fatalf("runtime plan %q inbound TCP = %#v, want %#v", plan.ID, plan.InboundTCP, want) + } + } + dockerPlan.Workload.Endpoints["http"] = EndpointExecutionPlan{ContainerPort: 9090} + if _, err := RuntimePlansV1(document, dockerPlan); err == nil || !strings.Contains(err.Error(), "endpoint \"http\"") { + t.Fatalf("endpoint mismatch error = %v", err) + } +} + func runtimePlanDocument() blueprint.Document { return blueprint.Document{Environment: blueprint.Environment{ Applications: map[string]blueprint.Application{ diff --git a/internal/dockerdeploy/runtime_policy_compile.go b/internal/dockerdeploy/runtime_policy_compile.go index 2cc10bcd..8fde5bd7 100644 --- a/internal/dockerdeploy/runtime_policy_compile.go +++ b/internal/dockerdeploy/runtime_policy_compile.go @@ -2,7 +2,9 @@ package dockerdeploy import ( "fmt" + "slices" "sort" + "strconv" "strings" "github.com/omry/reploy/internal/blueprint" @@ -62,6 +64,11 @@ func compileRuntimePolicyV1( ) (deploy.RuntimePolicyV1, error) { canonicalPlans := append([]deploy.RuntimePlanV1{}, plans...) for index := range canonicalPlans { + inboundTCP, err := canonicalRuntimeInboundTCPV1(canonicalPlans[index].InboundTCP) + if err != nil { + return deploy.RuntimePolicyV1{}, fmt.Errorf("runtime plan %q inbound TCP grants: %w", canonicalPlans[index].ID, err) + } + canonicalPlans[index].InboundTCP = inboundTCP canonicalPlans[index].Mounts = append([]deploy.RuntimeMountV1{}, canonicalPlans[index].Mounts...) canonicalPlans[index].Executables = append([]providers.QualifiedOutput{}, canonicalPlans[index].Executables...) sort.Slice(canonicalPlans[index].Mounts, func(left int, right int) bool { @@ -77,8 +84,12 @@ func compileRuntimePolicyV1( }) } sort.Slice(canonicalPlans, func(left int, right int) bool { return canonicalPlans[left].ID < canonicalPlans[right].ID }) + if err := validateRuntimePolicyInboundTCPV1(document, canonicalPlans); err != nil { + return deploy.RuntimePolicyV1{}, err + } policy := deploy.RuntimePolicyV1{ Schema: deploy.RuntimePolicySchemaV1, StartupVerifier: deploy.ApplicationStartupVerifierContractV1(), + Network: normalizeRuntimeNetworkV1(document.Environment.Runtime.Network), ProtectedPaths: protected, Plans: canonicalPlans, } if err := deploy.ValidateRuntimePolicyV1(policy); err != nil { @@ -93,6 +104,38 @@ func compileRuntimePolicyV1( return policy, nil } +func canonicalRuntimeInboundTCPV1(values []string) ([]string, error) { + ports := make([]int, 0, len(values)) + for _, value := range values { + port, err := strconv.Atoi(value) + if err != nil { + return nil, fmt.Errorf("port %q is not a decimal integer", value) + } + ports = append(ports, port) + } + return deploy.CanonicalRuntimeInboundTCPV1(ports), nil +} + +func validateRuntimePolicyInboundTCPV1(document blueprint.Document, plans []deploy.RuntimePlanV1) error { + wantPorts := []int{} + if workload := document.Environment.Workload; workload != nil { + for _, endpoint := range workload.Endpoints { + wantPorts = append(wantPorts, endpoint.Port) + } + } + want := deploy.CanonicalRuntimeInboundTCPV1(wantPorts) + for _, plan := range plans { + planWant := []string{} + if plan.ID == runtimeWorkloadPlanID { + planWant = want + } + if !slices.Equal(plan.InboundTCP, planWant) { + return fmt.Errorf("runtime plan %q inbound TCP grants do not match the resolved blueprint", plan.ID) + } + } + return nil +} + func providerGraphProtectedPaths(graph providers.GraphExecutionResult, plans []deploy.RuntimePlanV1) ([]deploy.ProtectedPathV1, error) { if graph.Materializations == nil || graph.Bundles == nil || graph.Catalog == nil { return nil, fmt.Errorf("runtime policy graph collections must use arrays") diff --git a/internal/dockerdeploy/runtime_policy_compile_test.go b/internal/dockerdeploy/runtime_policy_compile_test.go index 2043692b..4bdcc7ee 100644 --- a/internal/dockerdeploy/runtime_policy_compile_test.go +++ b/internal/dockerdeploy/runtime_policy_compile_test.go @@ -13,6 +13,11 @@ import ( func TestCompileRuntimePolicyCanonicalizesPlans(t *testing.T) { document := runtimePolicyDocument(t) + document.Environment.Runtime.Network = blueprint.RuntimeNetwork{ + Public: blueprint.NetworkAccessAllow, + Local: blueprint.NetworkAccessDeny, + Ambiguous: blueprint.AmbiguousNetworkAccessRequireBoth, + } plans := []deploy.RuntimePlanV1{ {ID: "workload", Mounts: []deploy.RuntimeMountV1{ {Destination: "/mnt/config", SourceKind: deploy.RuntimeMountSourceFile, ReadOnly: true}, @@ -33,6 +38,9 @@ func TestCompileRuntimePolicyCanonicalizesPlans(t *testing.T) { if len(policy.Plans) != 2 || policy.Plans[0].ID != "shell" || policy.Plans[1].Mounts[0].Destination != "/data" { t.Fatalf("canonical plans = %#v", policy.Plans) } + if policy.Network != document.Environment.Runtime.Network { + t.Fatalf("network policy = %#v, want %#v", policy.Network, document.Environment.Runtime.Network) + } } func TestCompileRuntimePolicyAllowsAbsoluteTargetsAndRejectsOverlap(t *testing.T) { @@ -64,6 +72,25 @@ func TestCompileRuntimePolicyAllowsAbsoluteTargetsAndRejectsOverlap(t *testing.T } } +func TestCompileRuntimePolicyRejectsInboundTCPOutsideWorkloadContract(t *testing.T) { + document := runtimePolicyDocument(t) + document.Environment.Workload = &blueprint.Workload{Endpoints: map[string]blueprint.Endpoint{ + "http": {Scheme: "http", Port: 8080}, + }} + validPlans := []deploy.RuntimePlanV1{ + {ID: runtimeShellPlanID, InboundTCP: []string{}, Mounts: []deploy.RuntimeMountV1{}, Executables: []providers.QualifiedOutput{}}, + {ID: runtimeWorkloadPlanID, InboundTCP: []string{"8080"}, Mounts: []deploy.RuntimeMountV1{}, Executables: []providers.QualifiedOutput{}}, + } + if _, err := CompileRuntimePolicyV1(document, emptyRuntimePolicyGraph(), validPlans); err != nil { + t.Fatalf("valid workload grant: %v", err) + } + invalid := append([]deploy.RuntimePlanV1{}, validPlans...) + invalid[0].InboundTCP = []string{"8080"} + if _, err := CompileRuntimePolicyV1(document, emptyRuntimePolicyGraph(), invalid); err == nil || !strings.Contains(err.Error(), "shell") { + t.Fatalf("transient inbound grant error = %v", err) + } +} + func TestCompileRuntimePolicyProtectsProviderRootsAndExecutableChains(t *testing.T) { transaction := rendererTransaction() generated := acceptedGeneratedExecutable(transaction) diff --git a/internal/dockerdeploy/runtime_readiness_test.go b/internal/dockerdeploy/runtime_readiness_test.go index 82213ee0..cbeb5b6d 100644 --- a/internal/dockerdeploy/runtime_readiness_test.go +++ b/internal/dockerdeploy/runtime_readiness_test.go @@ -305,6 +305,17 @@ func TestCurrentBuildMatchesRuntimeV1TreatsChangedStateAsStale(t *testing.T) { } } +func TestCurrentBuildMatchesRuntimeV1TreatsChangedLockedNetworkPolicyAsStale(t *testing.T) { + current, buildInput := runtimeCurrentBuildFixture(t) + current.Lock.RuntimePolicy.Network.Public = blueprint.NetworkAccessAllow + refreshCurrentBuildReuseGeneration(t, ¤t) + + matched, err := CurrentBuildMatchesRuntimeV1(current, buildInput.DockerPlan) + if err != nil || matched { + t.Fatalf("changed locked network policy = %v, %v", matched, err) + } +} + func TestCurrentBuildMatchesRuntimeV1RejectsMalformedRuntimePlan(t *testing.T) { current, _ := runtimeCurrentBuildFixture(t) matched, err := CurrentBuildMatchesRuntimeV1(current, DockerExecutionPlan{Workload: &WorkloadExecutionPlan{}}) diff --git a/internal/dockerdeploy/runtime_test.go b/internal/dockerdeploy/runtime_test.go index 4c4ce052..f339a577 100644 --- a/internal/dockerdeploy/runtime_test.go +++ b/internal/dockerdeploy/runtime_test.go @@ -229,6 +229,23 @@ func TestExtractRuntimeStartupLogSnippetUsesOnlyMarkerWindows(t *testing.T) { } } +func TestExtractRuntimeStartupLogDiagnosticsIncludesSandboxSetupFailureBeforeMarkers(t *testing.T) { + logs := strings.Join([]string{ + "environment | unrelated prior output", + "environment | reploy-probe: sandbox-exec: install application network policy: apply nftables transaction: operation not supported", + }, "\n") + diagnostics := extractRuntimeStartupLogDiagnostics(logs) + if diagnostics.Failure != "application sandbox setup failed" { + t.Fatalf("failure = %q", diagnostics.Failure) + } + if !strings.Contains(diagnostics.Snippet, "apply nftables transaction: operation not supported") { + t.Fatalf("snippet = %q", diagnostics.Snippet) + } + if strings.Contains(diagnostics.Snippet, "unrelated prior output") { + t.Fatalf("snippet contains unrelated output: %q", diagnostics.Snippet) + } +} + func runtimeStateEnvelope(t *testing.T, schema string) string { t.Helper() dir := t.TempDir() diff --git a/internal/dockerdeploy/testdata/network_policy_helper/main.go b/internal/dockerdeploy/testdata/network_policy_helper/main.go new file mode 100644 index 00000000..dfc80c4b --- /dev/null +++ b/internal/dockerdeploy/testdata/network_policy_helper/main.go @@ -0,0 +1,340 @@ +//go:build linux + +package main + +import ( + "bufio" + "context" + "encoding/binary" + "fmt" + "io" + "net" + "net/http" + "os" + "strconv" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +func main() { + if len(os.Args) < 2 { + fail("missing mode") + } + switch os.Args[1] { + case "peer": + peer() + case "dial": + dial(os.Args[2:]) + case "dns": + dns(os.Args[2:]) + case "workload": + workload(os.Args[2:]) + default: + fail("unknown mode %q", os.Args[1]) + } +} + +func dns(args []string) { + if len(args) != 3 { + fail("dns requires a name, expected result, and transport") + } + want, err := strconv.ParseBool(args[1]) + if err != nil { + fail("parse DNS expectation: %v", err) + } + checkDNS(args[0], want, args[2]) + fmt.Println("DNS_PASS") +} + +func checkDNS(name string, want bool, transport string) { + if transport != "udp" && transport != "tcp" { + fail("unsupported DNS transport %s", transport) + } + resolver := net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, _, address string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, transport, address) + }, + } + lookupContext, cancel := context.WithTimeout(context.Background(), 750*time.Millisecond) + _, lookupErr := resolver.LookupHost(lookupContext, name) + cancel() + if got := lookupErr == nil; got != want { + fail("DNS %s lookup %s succeeded=%t want=%t err=%v", transport, name, got, want, lookupErr) + } +} + +func dial(args []string) { + if len(args) != 2 { + fail("dial requires an address and expected result") + } + want, err := strconv.ParseBool(args[1]) + if err != nil { + fail("parse dial expectation: %v", err) + } + connection, dialErr := net.DialTimeout("tcp", args[0], 750*time.Millisecond) + if connection != nil { + _ = connection.Close() + } + if got := dialErr == nil; got != want { + fail("dial %s succeeded=%t want=%t err=%v", args[0], got, want, dialErr) + } + fmt.Println("DIAL_PASS") +} + +func peer() { + go serveDNS() + listener, err := net.Listen("tcp", ":9090") + if err != nil { + fail("listen: %v", err) + } + for { + connection, err := listener.Accept() + if err != nil { + fail("accept: %v", err) + } + _ = connection.Close() + } +} + +func serveDNS() { + go serveDNSUDP() + serveDNSTCP() +} + +func serveDNSUDP() { + connection, err := net.ListenPacket("udp", ":53") + if err != nil { + fail("listen for DNS: %v", err) + } + buffer := make([]byte, 4096) + for { + length, address, err := connection.ReadFrom(buffer) + if err != nil { + fail("read DNS query: %v", err) + } + response, ok := dnsResponse(buffer[:length]) + if !ok { + continue + } + if _, err := connection.WriteTo(response, address); err != nil { + fail("write DNS response: %v", err) + } + } +} + +func serveDNSTCP() { + listener, err := net.Listen("tcp", ":53") + if err != nil { + fail("listen for TCP DNS: %v", err) + } + for { + connection, err := listener.Accept() + if err != nil { + fail("accept TCP DNS: %v", err) + } + go answerDNSTCP(connection) + } +} + +func answerDNSTCP(connection net.Conn) { + defer connection.Close() + lengthBuffer := make([]byte, 2) + if _, err := io.ReadFull(connection, lengthBuffer); err != nil { + return + } + length := int(binary.BigEndian.Uint16(lengthBuffer)) + if length == 0 || length > 4096 { + return + } + query := make([]byte, length) + if _, err := io.ReadFull(connection, query); err != nil { + return + } + response, ok := dnsResponse(query) + if !ok { + return + } + framed := make([]byte, 2, len(response)+2) + binary.BigEndian.PutUint16(framed, uint16(len(response))) + framed = append(framed, response...) + for len(framed) != 0 { + written, err := connection.Write(framed) + if err != nil || written == 0 { + return + } + framed = framed[written:] + } +} + +func dnsResponse(query []byte) ([]byte, bool) { + if len(query) < 17 { + return nil, false + } + offset := 12 + for { + if offset >= len(query) { + return nil, false + } + length := int(query[offset]) + offset++ + if length == 0 { + break + } + if length > 63 || offset+length > len(query) { + return nil, false + } + offset += length + } + if offset+4 > len(query) { + return nil, false + } + questionEnd := offset + 4 + kind := binary.BigEndian.Uint16(query[offset : offset+2]) + var answer []byte + switch kind { + case 1: + answer = net.ParseIP("203.0.113.1").To4() + case 28: + answer = net.ParseIP("2001:db8::1").To16() + default: + return nil, false + } + response := append([]byte(nil), query[:questionEnd]...) + binary.BigEndian.PutUint16(response[2:4], 0x8180) + binary.BigEndian.PutUint16(response[6:8], 1) + response = append(response, 0xc0, 0x0c) + response = binary.BigEndian.AppendUint16(response, kind) + response = binary.BigEndian.AppendUint16(response, 1) + response = binary.BigEndian.AppendUint32(response, 0) + response = binary.BigEndian.AppendUint16(response, uint16(len(answer))) + response = append(response, answer...) + return response, true +} + +func workload(args []string) { + if len(args) != 14 { + fail("workload requires UID, six addresses, four expectations, serve mode, DNS name, and DNS expectation") + } + checkStatus(args[0]) + wantLocal, err := strconv.ParseBool(args[7]) + if err != nil { + fail("parse local expectation: %v", err) + } + wantPublic, err := strconv.ParseBool(args[8]) + if err != nil { + fail("parse public expectation: %v", err) + } + wantAmbiguous, err := strconv.ParseBool(args[9]) + if err != nil { + fail("parse ambiguous expectation: %v", err) + } + wantPublicException, err := strconv.ParseBool(args[10]) + if err != nil { + fail("parse public exception expectation: %v", err) + } + for _, item := range []struct { + address string + want bool + }{ + {args[1], wantLocal}, {ipv4MappedAddress(args[1]), wantLocal}, {args[2], wantLocal}, + {args[3], wantPublic}, {ipv4MappedAddress(args[3]), wantPublic}, {args[4], wantPublic}, + {args[5], wantAmbiguous}, {args[6], wantPublicException}, + } { + connection, dialErr := net.DialTimeout("tcp", item.address, 750*time.Millisecond) + if connection != nil { + _ = connection.Close() + } + if got := dialErr == nil; got != item.want { + fail("dial %s succeeded=%t want=%t err=%v", item.address, got, item.want, dialErr) + } + } + wantDNS, err := strconv.ParseBool(args[13]) + if err != nil { + fail("parse DNS expectation: %v", err) + } + for _, transport := range []string{"udp", "tcp"} { + checkDNS(args[12], wantDNS, transport) + } + fmt.Println("NETWORK_POLICY_PASS") + if args[11] != "serve" { + return + } + handler := http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprintln(response, "reploy-network-policy") + }) + declared, err := net.Listen("tcp", ":8080") + if err != nil { + fail("listen on declared endpoint: %v", err) + } + undeclared, err := net.Listen("tcp", ":8081") + if err != nil { + _ = declared.Close() + fail("listen on undeclared endpoint: %v", err) + } + go func() { + if serveErr := http.Serve(undeclared, handler); serveErr != nil { + fail("serve undeclared endpoint: %v", serveErr) + } + }() + if err := http.Serve(declared, handler); err != nil { + fail("serve: %v", err) + } +} + +func ipv4MappedAddress(address string) string { + host, port, err := net.SplitHostPort(address) + if err != nil { + fail("split IPv4 address %s: %v", address, err) + } + ip := net.ParseIP(host).To4() + if ip == nil { + fail("address %s is not IPv4", address) + } + return net.JoinHostPort("::ffff:"+ip.String(), port) +} + +func checkStatus(wantUID string) { + content, err := os.Open("/proc/self/status") + if err != nil { + fail("open status: %v", err) + } + defer content.Close() + seenUID := false + scanner := bufio.NewScanner(content) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "Uid:") { + fields := strings.Fields(line) + seenUID = len(fields) == 5 && fields[1] == wantUID && fields[2] == wantUID && fields[3] == wantUID && fields[4] == wantUID + } + for _, field := range []string{"CapInh:", "CapPrm:", "CapEff:", "CapBnd:", "CapAmb:"} { + if strings.HasPrefix(line, field) && strings.TrimLeft(strings.Fields(line)[1], "0") != "" { + fail("nonempty capability status: %s", line) + } + } + if strings.HasPrefix(line, "NoNewPrivs:") && strings.Fields(line)[1] != "1" { + fail("no-new-privileges missing: %s", line) + } + if strings.HasPrefix(line, "Seccomp:") && strings.Fields(line)[1] != "2" { + fail("seccomp missing: %s", line) + } + } + if err := scanner.Err(); err != nil || !seenUID { + fail("status identity mismatch for UID %s: %v", wantUID, err) + } + raw, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(3))) + if err == nil { + _ = unix.Close(raw) + fail("raw socket unexpectedly succeeded") + } +} + +func htons(value uint16) uint16 { return value<<8 | value>>8 } + +func fail(format string, args ...any) { + fmt.Fprintf(os.Stderr, "FAIL: "+format+"\n", args...) + os.Exit(1) +} diff --git a/internal/dockerdeploy/testdata/resolved_compose.yaml b/internal/dockerdeploy/testdata/resolved_compose.yaml index e31af6bf..448fcc1e 100644 --- a/internal/dockerdeploy/testdata/resolved_compose.yaml +++ b/internal/dockerdeploy/testdata/resolved_compose.yaml @@ -4,15 +4,20 @@ services: image: reploy/demo:staging pull_policy: never container_name: demo-staging-abcd - user: "501:20" + user: "0:0" group_add: [] cap_drop: - ALL + cap_add: + - NET_ADMIN + - SETGID + - SETPCAP + - SETUID security_opt: - no-new-privileges:true - seccomp=builtin entrypoint: [/reploy-probe] - command: [verify-exec, --, /opt/reploy/python/bin/demo, serve] + command: [sandbox-exec, --uid, "501", --gid, "20", --public, deny, --local, deny, --ambiguous, require-both, --inbound-tcp, "8080", --, /opt/reploy/python/bin/demo, serve] volumes: - type: bind source: /tmp/demo/conf diff --git a/internal/probe/main.go b/internal/probe/main.go index 58bfdf7a..a42ac029 100644 --- a/internal/probe/main.go +++ b/internal/probe/main.go @@ -18,6 +18,19 @@ func Main(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) in } return 0 } + if len(args) >= 1 && (args[0] == "sandbox-exec" || args[0] == "restricted-exec") { + installRules := args[0] == "sandbox-exec" + plan, err := parseSandboxExecPlanV1(args[1:], installRules) + if err != nil { + _, _ = fmt.Fprintf(stderr, "reploy-probe: %s: %v\n", args[0], err) + return 2 + } + if err := sandboxAndExecApplicationV1(plan); err != nil { + _, _ = fmt.Fprintf(stderr, "reploy-probe: %s: %v\n", args[0], err) + return 1 + } + return 0 + } return mainWithActions( args, stdin, stdout, stderr, waitForHoldSignal, copyFixedVolumeTree, @@ -76,7 +89,7 @@ func mainWithActions( return 0 } if len(args) != 0 { - _, _ = fmt.Fprintln(stderr, "reploy-probe accepts no arguments for one canonical stdin request, fixed hold mode, fixed copy-volume-tree mode, fixed install-local-account mode, or fixed verify-exec mode") + _, _ = fmt.Fprintln(stderr, "reploy-probe accepts no arguments for one canonical stdin request, fixed hold mode, fixed copy-volume-tree mode, fixed install-local-account mode, fixed verify-exec mode, or a sandbox-exec/restricted-exec contract") return 2 } content, err := io.ReadAll(stdin) diff --git a/internal/probe/network_firewall_linux.go b/internal/probe/network_firewall_linux.go new file mode 100644 index 00000000..d164d973 --- /dev/null +++ b/internal/probe/network_firewall_linux.go @@ -0,0 +1,382 @@ +//go:build linux + +package probe + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "net" + "os" + "slices" + "sort" + "strings" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "golang.org/x/sys/unix" +) + +type applicationNetworkPolicyV1 struct { + AllowPublic bool + AllowLocal bool + AllowAmbiguous bool + InboundTCP []uint16 +} + +const applicationResponseStateMaskV1 uint32 = expr.CtStateBitESTABLISHED + +const applicationResolverConfigurationV1 = "/etc/resolv.conf" +const applicationResolverConfigurationLimitV1 = 64 * 1024 + +var applicationRelatedICMPv4TypesV1 = []byte{3, 11, 12} +var applicationRelatedICMPv6TypesV1 = []byte{1, 2, 3, 4} + +// Keep these destination classes aligned with the IANA special-purpose +// registries. Public exceptions must precede their containing local ranges. +// Translation and tunneling prefixes are ambiguous because they can represent +// either public or local destinations; by default they require both grants. +// IPv4-mapped IPv6 socket addresses are not listed here: Linux emits those as +// IPv4 packets, so the embedded IPv4 destination receives its ordinary class. +// https://www.iana.org/assignments/iana-ipv4-special-registry/ +// https://www.iana.org/assignments/iana-ipv6-special-registry/ +var applicationLocalIPv4CIDRsV1 = []string{ + "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", + "169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.0.2.0/24", + "192.88.99.0/24", "192.168.0.0/16", "198.18.0.0/15", "198.51.100.0/24", "203.0.113.0/24", + "224.0.0.0/4", "240.0.0.0/4", +} + +var applicationPublicIPv4ExceptionsV1 = []string{ + "192.0.0.9/32", "192.0.0.10/32", +} + +var applicationAmbiguousIPv4CIDRsV1 = []string{} + +var applicationLocalIPv6CIDRsV1 = []string{ + "::/128", "::1/128", "100::/64", "100:0:0:1::/64", "2001::/23", + "2001:db8::/32", "3fff::/20", "5f00::/16", + "fc00::/7", "fe80::/10", "ff00::/8", +} + +var applicationPublicIPv6ExceptionsV1 = []string{ + "2001:1::1/128", "2001:1::2/128", "2001:1::3/128", "2001:3::/32", + "2001:4:112::/48", "2001:20::/28", "2001:30::/28", +} + +var applicationAmbiguousIPv6CIDRsV1 = []string{ + "64:ff9b::/96", "64:ff9b:1::/48", "2001::/32", "2002::/16", +} + +func installApplicationNetworkPolicyV1(policy applicationNetworkPolicyV1) error { + ports := append([]uint16(nil), policy.InboundTCP...) + slices.Sort(ports) + ports = slices.Compact(ports) + resolverCIDRs := []string{} + if policy.AllowPublic || policy.AllowLocal { + var err error + resolverCIDRs, err = readApplicationResolverCIDRsV1(applicationResolverConfigurationV1) + if err != nil { + return fmt.Errorf("read application DNS resolvers: %w", err) + } + if len(resolverCIDRs) == 0 { + return fmt.Errorf("read application DNS resolvers: no nameserver entries") + } + } + connection := &nftables.Conn{} + for _, family := range []struct { + tableFamily nftables.TableFamily + local []string + publicExceptions []string + ambiguous []string + addressSize uint32 + }{ + {nftables.TableFamilyIPv4, applicationLocalIPv4CIDRsV1, applicationPublicIPv4ExceptionsV1, applicationAmbiguousIPv4CIDRsV1, 4}, + {nftables.TableFamilyIPv6, applicationLocalIPv6CIDRsV1, applicationPublicIPv6ExceptionsV1, applicationAmbiguousIPv6CIDRsV1, 16}, + } { + if err := replaceApplicationNetworkTableV1(connection, family.tableFamily, family.local, family.publicExceptions, family.ambiguous, resolverCIDRs, family.addressSize, policy, ports); err != nil { + return err + } + } + if err := connection.Flush(); err != nil { + return fmt.Errorf("apply nftables transaction: %w", err) + } + return nil +} + +func replaceApplicationNetworkTableV1(connection *nftables.Conn, family nftables.TableFamily, local []string, publicExceptions []string, ambiguous []string, resolverCIDRs []string, addressSize uint32, policy applicationNetworkPolicyV1, ports []uint16) error { + tables, err := connection.ListTablesOfFamily(family) + if err != nil { + return fmt.Errorf("list nftables family %d: %w", family, err) + } + for _, table := range tables { + if table.Name == "reploy" { + connection.DelTable(table) + } + } + table := connection.AddTable(&nftables.Table{Family: family, Name: "reploy"}) + drop := nftables.ChainPolicyDrop + input := connection.AddChain(&nftables.Chain{ + Name: "input", Table: table, Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookInput, Priority: nftables.ChainPriorityFilter, Policy: &drop, + }) + output := connection.AddChain(&nftables.Chain{ + Name: "output", Table: table, Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookOutput, Priority: nftables.ChainPriorityFilter, Policy: &drop, + }) + forward := connection.AddChain(&nftables.Chain{ + Name: "forward", Table: table, Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookForward, Priority: nftables.ChainPriorityFilter, Policy: &drop, + }) + _ = forward + addInterfaceVerdictV1(connection, table, input, expr.MetaKeyIIFNAME, "lo", expr.VerdictAccept) + addEstablishedVerdictV1(connection, table, input, expr.VerdictAccept) + addRelatedICMPErrorVerdictsV1(connection, table, input, family) + if family == nftables.TableFamilyIPv6 { + addIPv6NeighborDiscoveryRulesV1(connection, table, input) + } + for _, port := range ports { + addTCPPortVerdictV1(connection, table, input, port, expr.VerdictAccept) + } + + if policy.AllowPublic || policy.AllowLocal { + if err := addDNSResolverVerdictsV1(connection, table, output, resolverCIDRs, addressSize); err != nil { + return err + } + } + if family == nftables.TableFamilyIPv4 { + // Docker's embedded resolver is reached through container loopback. + // Permit only its engine-owned DNS service before accepting ordinary + // application loopback traffic. + if err := addCIDRVerdictV1(connection, table, output, "127.0.0.11/32", addressSize, expr.VerdictDrop); err != nil { + return err + } + } + addInterfaceVerdictV1(connection, table, output, expr.MetaKeyOIFNAME, "lo", expr.VerdictAccept) + addEstablishedVerdictV1(connection, table, output, expr.VerdictAccept) + if family == nftables.TableFamilyIPv6 { + addIPv6NeighborDiscoveryRulesV1(connection, table, output) + } + ambiguousVerdict := expr.VerdictDrop + if policy.AllowAmbiguous { + ambiguousVerdict = expr.VerdictAccept + } + if err := addCIDRVerdictsV1(connection, table, output, ambiguous, addressSize, ambiguousVerdict); err != nil { + return err + } + publicVerdict := expr.VerdictDrop + if policy.AllowPublic { + publicVerdict = expr.VerdictAccept + } + if err := addCIDRVerdictsV1(connection, table, output, publicExceptions, addressSize, publicVerdict); err != nil { + return err + } + localVerdict := expr.VerdictDrop + if policy.AllowLocal { + localVerdict = expr.VerdictAccept + } + if err := addCIDRVerdictsV1(connection, table, output, local, addressSize, localVerdict); err != nil { + return err + } + if policy.AllowPublic { + connection.AddRule(&nftables.Rule{Table: table, Chain: output, Exprs: []expr.Any{&expr.Verdict{Kind: expr.VerdictAccept}}}) + } + return nil +} + +func readApplicationResolverCIDRsV1(path string) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + content, err := io.ReadAll(io.LimitReader(file, applicationResolverConfigurationLimitV1+1)) + if err != nil { + return nil, err + } + if len(content) > applicationResolverConfigurationLimitV1 { + return nil, fmt.Errorf("resolver configuration exceeds %d bytes", applicationResolverConfigurationLimitV1) + } + result := map[string]struct{}{} + scanner := bufio.NewScanner(strings.NewReader(string(content))) + for scanner.Scan() { + line := scanner.Text() + if offset := strings.IndexAny(line, "#;"); offset >= 0 { + line = line[:offset] + } + fields := strings.Fields(line) + if len(fields) == 0 || fields[0] != "nameserver" { + continue + } + if len(fields) < 2 { + return nil, fmt.Errorf("nameserver entry has no address") + } + address := fields[1] + if host, _, found := strings.Cut(address, "%"); found { + address = host + } + ip := net.ParseIP(address) + if ip == nil { + return nil, fmt.Errorf("nameserver address %q is not an IP address", fields[1]) + } + bits := 128 + if ipv4 := ip.To4(); ipv4 != nil { + ip = ipv4 + bits = 32 + } + result[fmt.Sprintf("%s/%d", ip.String(), bits)] = struct{}{} + } + if err := scanner.Err(); err != nil { + return nil, err + } + resolverCIDRs := make([]string, 0, len(result)) + for cidr := range result { + resolverCIDRs = append(resolverCIDRs, cidr) + } + sort.Strings(resolverCIDRs) + return resolverCIDRs, nil +} + +func addDNSResolverVerdictsV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, resolverCIDRs []string, addressSize uint32) error { + for _, cidr := range resolverCIDRs { + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("parse application DNS resolver %q: %w", cidr, err) + } + if (addressSize == 4 && ip.To4() == nil) || (addressSize == 16 && ip.To4() != nil) { + continue + } + for _, protocol := range []byte{unix.IPPROTO_UDP, unix.IPPROTO_TCP} { + if err := addTransportPortCIDRVerdictV1(connection, table, chain, cidr, addressSize, protocol, 53, expr.VerdictAccept); err != nil { + return err + } + } + } + return nil +} + +func addTransportPortCIDRVerdictV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, cidr string, addressSize uint32, protocol byte, port uint16, verdict expr.VerdictKind) error { + ip, network, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("parse application network prefix %q: %w", cidr, err) + } + address := ip.To4() + if addressSize == 16 { + address = ip.To16() + } + portData := make([]byte, 2) + binary.BigEndian.PutUint16(portData, port) + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + // Docker DNATs embedded-DNS traffic from port 53 to an internal + // high-numbered socket before the filter hook. Match the original + // conntrack tuple so the narrow resolver grant survives that rewrite. + &expr.Ct{Register: 1, Key: expr.CtKeyDST, Direction: 0}, + &expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: addressSize, Mask: network.Mask, Xor: make([]byte, addressSize)}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: address}, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{protocol}}, + &expr.Ct{Register: 1, Key: expr.CtKeyPROTODST, Direction: 0}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: portData}, + &expr.Verdict{Kind: verdict}, + }}) + return nil +} + +func addCIDRVerdictsV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, cidrs []string, addressSize uint32, verdict expr.VerdictKind) error { + for _, cidr := range cidrs { + if err := addCIDRVerdictV1(connection, table, chain, cidr, addressSize, verdict); err != nil { + return err + } + } + return nil +} + +func addRelatedICMPErrorVerdictsV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, family nftables.TableFamily) { + protocol := byte(unix.IPPROTO_ICMP) + types := applicationRelatedICMPv4TypesV1 + if family == nftables.TableFamilyIPv6 { + protocol = unix.IPPROTO_ICMPV6 + types = applicationRelatedICMPv6TypesV1 + } + for _, kind := range types { + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{protocol}}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{kind}}, + &expr.Ct{Register: 1, Key: expr.CtKeySTATE}, + &expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, + Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitRELATED), + Xor: binaryutil.NativeEndian.PutUint32(0)}, + &expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: []byte{0, 0, 0, 0}}, + &expr.Verdict{Kind: expr.VerdictAccept}, + }}) + } +} + +func addInterfaceVerdictV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, key expr.MetaKey, name string, verdict expr.VerdictKind) { + data := make([]byte, 16) + copy(data, name+"\x00") + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Meta{Key: key, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: data}, + &expr.Verdict{Kind: verdict}, + }}) +} + +func addEstablishedVerdictV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, verdict expr.VerdictKind) { + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Ct{Register: 1, Key: expr.CtKeySTATE}, + &expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, + Mask: binaryutil.NativeEndian.PutUint32(applicationResponseStateMaskV1), + Xor: binaryutil.NativeEndian.PutUint32(0)}, + &expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: []byte{0, 0, 0, 0}}, + &expr.Verdict{Kind: verdict}, + }}) +} + +func addTCPPortVerdictV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, port uint16, verdict expr.VerdictKind) { + data := make([]byte, 2) + binary.BigEndian.PutUint16(data, port) + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: data}, + &expr.Verdict{Kind: verdict}, + }}) +} + +func addIPv6NeighborDiscoveryRulesV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain) { + for _, kind := range []byte{133, 134, 135, 136} { + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_ICMPV6}}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{kind}}, + &expr.Verdict{Kind: expr.VerdictAccept}, + }}) + } +} + +func addCIDRVerdictV1(connection *nftables.Conn, table *nftables.Table, chain *nftables.Chain, cidr string, addressSize uint32, verdict expr.VerdictKind) error { + ip, network, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("parse application network prefix %q: %w", cidr, err) + } + offset := uint32(16) + address := ip.To4() + if addressSize == 16 { + offset = 24 + address = ip.To16() + } + connection.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: []expr.Any{ + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: offset, Len: addressSize}, + &expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: addressSize, Mask: network.Mask, Xor: make([]byte, addressSize)}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: address}, + &expr.Verdict{Kind: verdict}, + }}) + return nil +} diff --git a/internal/probe/network_firewall_linux_test.go b/internal/probe/network_firewall_linux_test.go new file mode 100644 index 00000000..b9d460e0 --- /dev/null +++ b/internal/probe/network_firewall_linux_test.go @@ -0,0 +1,102 @@ +//go:build linux + +package probe + +import ( + "net" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/google/nftables/expr" +) + +func TestReadApplicationResolverCIDRsV1(t *testing.T) { + path := filepath.Join(t.TempDir(), "resolv.conf") + content := "search example.test\n" + + "nameserver 127.0.0.11 # Docker embedded DNS\n" + + "nameserver 8.8.8.8\n" + + "nameserver 2001:4860:4860::8888%eth0\n" + + "nameserver 8.8.8.8 ; duplicate\n" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + got, err := readApplicationResolverCIDRsV1(path) + if err != nil { + t.Fatal(err) + } + want := []string{"127.0.0.11/32", "2001:4860:4860::8888/128", "8.8.8.8/32"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("resolver CIDRs = %#v, want %#v", got, want) + } +} + +func TestReadApplicationResolverCIDRsV1RejectsInvalidNameserver(t *testing.T) { + for _, content := range []string{"nameserver\n", "nameserver resolver.example\n"} { + t.Run(strings.TrimSpace(content), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "resolv.conf") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + if _, err := readApplicationResolverCIDRsV1(path); err == nil { + t.Fatalf("resolver content %q unexpectedly passed", content) + } + }) + } +} + +func TestApplicationResponseTrafficAcceptsOnlyEstablishedConnections(t *testing.T) { + if applicationResponseStateMaskV1 != expr.CtStateBitESTABLISHED || applicationResponseStateMaskV1&expr.CtStateBitRELATED != 0 { + t.Fatalf("response conntrack mask = %#x", applicationResponseStateMaskV1) + } +} + +func TestApplicationRelatedTrafficIsLimitedToICMPNetworkErrors(t *testing.T) { + if !reflect.DeepEqual(applicationRelatedICMPv4TypesV1, []byte{3, 11, 12}) { + t.Fatalf("IPv4 related ICMP types = %#v", applicationRelatedICMPv4TypesV1) + } + if !reflect.DeepEqual(applicationRelatedICMPv6TypesV1, []byte{1, 2, 3, 4}) { + t.Fatalf("IPv6 related ICMP types = %#v", applicationRelatedICMPv6TypesV1) + } +} + +func TestApplicationNetworkCIDRsSeparateLocalAmbiguousAndPublicExceptions(t *testing.T) { + for _, address := range []string{"192.88.99.2", "100:0:0:1::1", "2001:2::1", "3fff::1", "5f00::1"} { + if !applicationCIDRsContainAddressV1(applicationLocalIPv4CIDRsV1, applicationLocalIPv6CIDRsV1, address) { + t.Fatalf("special destination %s is not classified as local", address) + } + } + for _, address := range []string{"64:ff9b::a00:1", "64:ff9b:1::1", "2001::1", "2002:a00:1::1"} { + if !applicationCIDRsContainAddressV1(applicationAmbiguousIPv4CIDRsV1, applicationAmbiguousIPv6CIDRsV1, address) { + t.Fatalf("translation or tunneling destination %s is not classified as ambiguous", address) + } + } + if applicationCIDRsContainAddressV1(applicationAmbiguousIPv4CIDRsV1, applicationAmbiguousIPv6CIDRsV1, "::ffff:10.0.0.1") { + t.Fatal("IPv4-mapped IPv6 destination must use its embedded IPv4 class") + } + for _, address := range []string{"192.0.0.9", "192.0.0.10", "2001:20::1", "2001:30::1"} { + if !applicationCIDRsContainAddressV1(applicationPublicIPv4ExceptionsV1, applicationPublicIPv6ExceptionsV1, address) { + t.Fatalf("globally reachable exception %s is not classified as public", address) + } + } +} + +func applicationCIDRsContainAddressV1(ipv4 []string, ipv6 []string, address string) bool { + ip := net.ParseIP(address) + cidrs := ipv6 + if !strings.Contains(address, ":") { + cidrs = ipv4 + } + for _, cidr := range cidrs { + _, network, err := net.ParseCIDR(cidr) + if err != nil { + panic(err) + } + if network.Contains(ip) { + return true + } + } + return false +} diff --git a/internal/probe/sandbox_exec.go b/internal/probe/sandbox_exec.go new file mode 100644 index 00000000..f5409c67 --- /dev/null +++ b/internal/probe/sandbox_exec.go @@ -0,0 +1,139 @@ +package probe + +import ( + "flag" + "fmt" + "strconv" + "strings" +) + +type sandboxExecPlanV1 struct { + UID int + GID int + Groups []int + AllowPublic bool + AllowLocal bool + AllowAmbiguous bool + InboundTCP []uint16 + InstallRules bool + Argv []string +} + +func parseSandboxExecPlanV1(args []string, installRules bool) (sandboxExecPlanV1, error) { + separator := -1 + for index, argument := range args { + if argument == "--" { + separator = index + break + } + } + if separator < 0 || separator == len(args)-1 { + return sandboxExecPlanV1{}, fmt.Errorf("requires -- followed by an absolute application command") + } + set := flag.NewFlagSet("sandbox-exec", flag.ContinueOnError) + set.SetOutput(new(strings.Builder)) + uid := set.Int("uid", -1, "application UID") + gid := set.Int("gid", -1, "application GID") + groups := set.String("groups", "", "comma-separated supplementary GIDs") + public := set.String("public", "", "public network policy") + local := set.String("local", "", "local network policy") + ambiguous := set.String("ambiguous", "", "ambiguous destination policy") + inbound := set.String("inbound-tcp", "", "comma-separated inbound TCP ports") + if err := set.Parse(args[:separator]); err != nil { + return sandboxExecPlanV1{}, err + } + if len(set.Args()) != 0 { + return sandboxExecPlanV1{}, fmt.Errorf("unexpected positional sandbox arguments") + } + argv := args[separator+1:] + if *uid < 0 || *gid < 0 { + return sandboxExecPlanV1{}, fmt.Errorf("requires non-negative --uid and --gid") + } + parsedGroups, err := parseDecimalListV1(*groups, 0, int(^uint(0)>>1)) + if err != nil { + return sandboxExecPlanV1{}, fmt.Errorf("parse --groups: %w", err) + } + if !installRules && *inbound != "" { + return sandboxExecPlanV1{}, fmt.Errorf("--inbound-tcp is not accepted by restricted-exec") + } + parsedInbound, err := parseDecimalListV1(*inbound, 1, 65535) + if err != nil { + return sandboxExecPlanV1{}, fmt.Errorf("parse --inbound-tcp: %w", err) + } + allowPublic, err := parseNetworkAccessV1("--public", *public, installRules) + if err != nil { + return sandboxExecPlanV1{}, err + } + allowLocal, err := parseNetworkAccessV1("--local", *local, installRules) + if err != nil { + return sandboxExecPlanV1{}, err + } + allowAmbiguous, err := parseAmbiguousNetworkAccessV1(*ambiguous, allowPublic, allowLocal, installRules) + if err != nil { + return sandboxExecPlanV1{}, err + } + ports := make([]uint16, len(parsedInbound)) + for index, value := range parsedInbound { + ports[index] = uint16(value) + } + return sandboxExecPlanV1{ + UID: *uid, GID: *gid, Groups: parsedGroups, + AllowPublic: allowPublic, AllowLocal: allowLocal, AllowAmbiguous: allowAmbiguous, + InboundTCP: ports, InstallRules: installRules, + Argv: append([]string(nil), argv...), + }, nil +} + +func parseAmbiguousNetworkAccessV1(value string, allowPublic bool, allowLocal bool, required bool) (bool, error) { + if !required { + if value != "" { + return false, fmt.Errorf("--ambiguous is not accepted by restricted-exec") + } + return false, nil + } + switch value { + case "require-both": + return allowPublic && allowLocal, nil + case "allow": + return true, nil + default: + return false, fmt.Errorf("--ambiguous must be require-both or allow") + } +} + +func parseNetworkAccessV1(name string, value string, required bool) (bool, error) { + if !required { + if value != "" { + return false, fmt.Errorf("%s is not accepted by restricted-exec", name) + } + return false, nil + } + switch value { + case "allow": + return true, nil + case "deny": + return false, nil + default: + return false, fmt.Errorf("%s must be allow or deny", name) + } +} + +func parseDecimalListV1(value string, minimum int, maximum int) ([]int, error) { + if value == "" { + return []int{}, nil + } + result := []int{} + previous := -1 + for _, item := range strings.Split(value, ",") { + parsed, err := strconv.Atoi(item) + if err != nil || parsed < minimum || parsed > maximum { + return nil, fmt.Errorf("%q is outside %d..%d", item, minimum, maximum) + } + if parsed <= previous { + return nil, fmt.Errorf("values must be unique and sorted") + } + result = append(result, parsed) + previous = parsed + } + return result, nil +} diff --git a/internal/probe/sandbox_exec_linux.go b/internal/probe/sandbox_exec_linux.go new file mode 100644 index 00000000..dde1a156 --- /dev/null +++ b/internal/probe/sandbox_exec_linux.go @@ -0,0 +1,92 @@ +//go:build linux + +package probe + +import ( + "fmt" + "os" + "runtime" + "strconv" + "strings" + + "golang.org/x/sys/unix" +) + +const ( + secureNoRootV1 = 1 << 0 + secureNoRootLockedV1 = 1 << 1 + secureNoSetUIDFixupV1 = 1 << 2 + secureNoSetUIDFixupLockedV1 = 1 << 3 + secureKeepCapsLockedV1 = 1 << 5 + secureNoAmbientRaiseV1 = 1 << 6 + secureNoAmbientRaiseLockedV1 = 1 << 7 +) + +func sandboxAndExecApplicationV1(plan sandboxExecPlanV1) error { + // Linux credentials, securebits, and capability bounding sets are + // thread-scoped. Keep the trusted transition and the final exec on exactly + // one OS thread so the Go scheduler cannot move verification onto a thread + // that still carries the container's setup authority. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + if os.Geteuid() != 0 { + return fmt.Errorf("trusted sandbox setup must begin as container root") + } + if plan.InstallRules { + if err := installApplicationNetworkPolicyV1(applicationNetworkPolicyV1{ + AllowPublic: plan.AllowPublic, + AllowLocal: plan.AllowLocal, + AllowAmbiguous: plan.AllowAmbiguous, + InboundTCP: plan.InboundTCP, + }); err != nil { + return fmt.Errorf("install application network policy: %w", err) + } + } + if err := dropApplicationAuthorityV1(plan.UID, plan.GID, plan.Groups); err != nil { + return err + } + return verifyAndExecApplication(plan.Argv, readApplicationKernelStatus, execApplication) +} + +func dropApplicationAuthorityV1(uid int, gid int, groups []int) error { + lastContent, err := os.ReadFile("/proc/sys/kernel/cap_last_cap") + if err != nil { + return fmt.Errorf("read last Linux capability: %w", err) + } + lastCapability, err := strconv.Atoi(strings.TrimSpace(string(lastContent))) + if err != nil { + return fmt.Errorf("parse last Linux capability: %w", err) + } + for capability := 0; capability <= lastCapability; capability++ { + if err := unix.Prctl(unix.PR_CAPBSET_DROP, uintptr(capability), 0, 0, 0); err != nil { + return fmt.Errorf("drop capability %d from bounding set: %w", capability, err) + } + } + secureBits := uintptr( + secureNoRootV1 | secureNoRootLockedV1 | + secureNoSetUIDFixupV1 | secureNoSetUIDFixupLockedV1 | + secureKeepCapsLockedV1 | + secureNoAmbientRaiseV1 | secureNoAmbientRaiseLockedV1, + ) + if err := unix.Prctl(unix.PR_SET_SECUREBITS, secureBits, 0, 0, 0); err != nil { + return fmt.Errorf("lock application securebits: %w", err) + } + if err := unix.Setgroups(groups); err != nil { + return fmt.Errorf("set application supplementary groups: %w", err) + } + if err := unix.Setresgid(gid, gid, gid); err != nil { + return fmt.Errorf("set application GID %d: %w", gid, err) + } + if err := unix.Setresuid(uid, uid, uid); err != nil { + return fmt.Errorf("set application UID %d: %w", uid, err) + } + if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return fmt.Errorf("set application no-new-privileges: %w", err) + } + header := unix.CapUserHeader{Version: unix.LINUX_CAPABILITY_VERSION_3, Pid: 0} + data := [2]unix.CapUserData{} + if err := unix.Capset(&header, &data[0]); err != nil { + return fmt.Errorf("clear application capabilities: %w", err) + } + return nil +} diff --git a/internal/probe/sandbox_exec_other.go b/internal/probe/sandbox_exec_other.go new file mode 100644 index 00000000..5643acec --- /dev/null +++ b/internal/probe/sandbox_exec_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package probe + +import "fmt" + +func sandboxAndExecApplicationV1(sandboxExecPlanV1) error { + return fmt.Errorf("application sandbox setup is supported only in Linux containers") +} diff --git a/internal/probe/sandbox_exec_test.go b/internal/probe/sandbox_exec_test.go new file mode 100644 index 00000000..eabced64 --- /dev/null +++ b/internal/probe/sandbox_exec_test.go @@ -0,0 +1,91 @@ +package probe + +import ( + "reflect" + "strings" + "testing" +) + +func TestParseSandboxExecPlanV1PreservesPolicyIdentityAndArgv(t *testing.T) { + plan, err := parseSandboxExecPlanV1([]string{ + "--uid", "501", "--gid", "20", "--groups", "33,44", + "--public", "allow", "--local", "deny", "--ambiguous", "allow", "--inbound-tcp", "8080,8443", + "--", "/opt/app", "", "$(not-shell)", + }, true) + if err != nil { + t.Fatal(err) + } + if plan.UID != 501 || plan.GID != 20 || !reflect.DeepEqual(plan.Groups, []int{33, 44}) || + !plan.AllowPublic || plan.AllowLocal || !plan.AllowAmbiguous || !reflect.DeepEqual(plan.InboundTCP, []uint16{8080, 8443}) || + !reflect.DeepEqual(plan.Argv, []string{"/opt/app", "", "$(not-shell)"}) { + t.Fatalf("plan = %#v", plan) + } +} + +func TestParseSandboxExecPlanV1RequireBothAllowsAmbiguousOnlyWithBothGrants(t *testing.T) { + for _, test := range []struct { + name string + public string + local string + wantAllow bool + }{ + {name: "neither", public: "deny", local: "deny"}, + {name: "public only", public: "allow", local: "deny"}, + {name: "local only", public: "deny", local: "allow"}, + {name: "both", public: "allow", local: "allow", wantAllow: true}, + } { + t.Run(test.name, func(t *testing.T) { + plan, err := parseSandboxExecPlanV1([]string{ + "--uid", "501", "--gid", "20", + "--public", test.public, "--local", test.local, "--ambiguous", "require-both", + "--", "/opt/app", + }, true) + if err != nil { + t.Fatal(err) + } + if plan.AllowAmbiguous != test.wantAllow { + t.Fatalf("AllowAmbiguous = %t, want %t", plan.AllowAmbiguous, test.wantAllow) + } + }) + } +} + +func TestParseSandboxExecPlanV1FailsClosed(t *testing.T) { + base := []string{"--uid", "501", "--gid", "20", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/app"} + for _, test := range []struct { + name string + args []string + want string + }{ + {name: "separator", args: base[:len(base)-2], want: "requires --"}, + {name: "identity", args: []string{"--uid", "-1", "--gid", "20", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/app"}, want: "non-negative"}, + {name: "policy", args: []string{"--uid", "1", "--gid", "2", "--public", "yes", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/app"}, want: "--public"}, + {name: "ambiguous policy", args: []string{"--uid", "1", "--gid", "2", "--public", "deny", "--local", "deny", "--ambiguous", "yes", "--", "/opt/app"}, want: "--ambiguous"}, + {name: "groups", args: []string{"--uid", "1", "--gid", "2", "--groups", "44,33", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--", "/opt/app"}, want: "unique and sorted"}, + {name: "port", args: []string{"--uid", "1", "--gid", "2", "--public", "deny", "--local", "deny", "--ambiguous", "require-both", "--inbound-tcp", "0", "--", "/opt/app"}, want: "outside 1..65535"}, + } { + t.Run(test.name, func(t *testing.T) { + _, err := parseSandboxExecPlanV1(test.args, true) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %v, want %q", err, test.want) + } + }) + } +} + +func TestRestrictedExecDoesNotAcceptNetworkSetupArguments(t *testing.T) { + for _, argument := range []string{"--public", "--local", "--ambiguous", "--inbound-tcp"} { + t.Run(argument, func(t *testing.T) { + value := "allow" + if argument == "--inbound-tcp" { + value = "8080" + } + _, err := parseSandboxExecPlanV1([]string{ + "--uid", "501", "--gid", "20", argument, value, "--", "/bin/true", + }, false) + if err == nil || !strings.Contains(err.Error(), argument) { + t.Fatalf("restricted exec network argument error = %v", err) + } + }) + } +} diff --git a/internal/probe/startup_verifier.go b/internal/probe/startup_verifier.go index 2f3f069e..a3651a9b 100644 --- a/internal/probe/startup_verifier.go +++ b/internal/probe/startup_verifier.go @@ -16,9 +16,11 @@ var requiredApplicationKernelStatusV1 = []struct { want string hex bool }{ + {name: "CapInh", want: "0", hex: true}, {name: "CapBnd", want: "0", hex: true}, {name: "CapEff", want: "0", hex: true}, {name: "CapPrm", want: "0", hex: true}, + {name: "CapAmb", want: "0", hex: true}, {name: "NoNewPrivs", want: "1"}, {name: "Seccomp", want: "2"}, } diff --git a/internal/probe/startup_verifier_test.go b/internal/probe/startup_verifier_test.go index 9c6a169e..47526cd6 100644 --- a/internal/probe/startup_verifier_test.go +++ b/internal/probe/startup_verifier_test.go @@ -9,9 +9,11 @@ import ( ) const validApplicationKernelStatus = `Name: reploy-probe +CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 CapBnd: 0000000000000000 +CapAmb: 0000000000000000 NoNewPrivs: 1 Seccomp: 2 ` @@ -36,6 +38,8 @@ func TestVerifyApplicationKernelStatusFailsClosed(t *testing.T) { {name: "effective capabilities", content: strings.ReplaceAll(validApplicationKernelStatus, "CapEff: 0000000000000000", "CapEff: 0000000000000001"), want: "CapEff is 0000000000000001"}, {name: "permitted capabilities", content: strings.ReplaceAll(validApplicationKernelStatus, "CapPrm: 0000000000000000", "CapPrm: 0000000000000400"), want: "CapPrm is 0000000000000400"}, {name: "bounding capabilities", content: strings.ReplaceAll(validApplicationKernelStatus, "CapBnd: 0000000000000000", "CapBnd: 000001ffffffffff"), want: "CapBnd is 000001ffffffffff"}, + {name: "inheritable capabilities", content: strings.ReplaceAll(validApplicationKernelStatus, "CapInh: 0000000000000000", "CapInh: 0000000000000001"), want: "CapInh is 0000000000000001"}, + {name: "ambient capabilities", content: strings.ReplaceAll(validApplicationKernelStatus, "CapAmb: 0000000000000000", "CapAmb: 0000000000000001"), want: "CapAmb is 0000000000000001"}, {name: "invalid capability", content: strings.ReplaceAll(validApplicationKernelStatus, "CapBnd: 0000000000000000", "CapBnd: not-hex"), want: "not hexadecimal"}, } for _, test := range tests {