diff --git a/cmd/oap/internal/health/secrets.go b/cmd/oap/internal/health/secrets.go new file mode 100644 index 00000000..0f50f9b3 --- /dev/null +++ b/cmd/oap/internal/health/secrets.go @@ -0,0 +1,61 @@ +package health + +import ( + "context" + "fmt" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/authzed/openagentprimitives/cmd/oap/internal/kube" + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" +) + +func init() { + // Verifies the memory-token Secrets exist. This is the GitOps-specific check: + // `oap install` creates these imperatively, but a bundle synced by Argo/Flux + // relies on the pre-start bootstrap Job (the operator image's + // `bootstrap-secrets` entrypoint) to mint them — so their absence is the + // signal that the Job never completed (wrong image, backoff exhausted, or the + // operator ServiceAccount is missing Secret RBAC). + Register(memoryTokenSecrets{}) +} + +// memoryTokenSecrets checks that each memory-token Secret exists and carries a +// non-empty token. It covers the three memory tokens (bootstrap.MemoryTokenSecretNames) +// the bootstrap Job mints; the other app-internal secrets the Job mints (admind, +// passthrough, NATS, spicedb/postgres) are verified indirectly by the workload +// health checks that mount them. +type memoryTokenSecrets struct{} + +func (memoryTokenSecrets) Name() string { return "memory-token-secrets" } + +func (memoryTokenSecrets) Check(ctx context.Context, b *kube.Bundle) Result { + var missing []string + present := 0 + for _, name := range bootstrap.MemoryTokenSecretNames { + sec, err := b.Typed.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + missing = append(missing, name) + case err != nil: + return Result{Status: Failed, Detail: fmt.Sprintf("get %s: %v", name, err)} + case len(sec.Data[bootstrap.TokenSecretKey]) == 0: + missing = append(missing, name+" (empty token)") + default: + present++ + } + } + total := len(bootstrap.MemoryTokenSecretNames) + switch { + case len(missing) == 0: + return Result{Status: OK, Detail: fmt.Sprintf("%d/%d memory tokens present", present, total)} + case present == 0: + // None present: the bootstrap Job has not minted them. NotFound lets a + // trimmed profile skip, matching the workload checks. + return Result{Status: Failed, Detail: "no memory tokens present (the bootstrap Job has not minted them)", NotFound: true} + default: + return Result{Status: Failed, Detail: "missing memory tokens: " + strings.Join(missing, ", ")} + } +} diff --git a/cmd/oap/internal/health/secrets_test.go b/cmd/oap/internal/health/secrets_test.go new file mode 100644 index 00000000..a2faf319 --- /dev/null +++ b/cmd/oap/internal/health/secrets_test.go @@ -0,0 +1,70 @@ +package health + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + + "github.com/authzed/openagentprimitives/cmd/oap/internal/kube" + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" +) + +func tokenSecret(name string, token string) *corev1.Secret { + s := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}} + if token != "" { + s.Data = map[string][]byte{bootstrap.TokenSecretKey: []byte(token)} + } + return s +} + +func bundleWith(objs ...runtime.Object) *kube.Bundle { + return &kube.Bundle{Typed: fake.NewSimpleClientset(objs...)} +} + +func TestMemoryTokenSecrets_AllPresentOK(t *testing.T) { + var objs []runtime.Object + for _, n := range bootstrap.MemoryTokenSecretNames { + objs = append(objs, tokenSecret(n, "deadbeef")) + } + res := memoryTokenSecrets{}.Check(context.Background(), bundleWith(objs...)) + assert.Equal(t, OK, res.Status) + assert.False(t, res.NotFound) +} + +func TestMemoryTokenSecrets_MissingOneFails(t *testing.T) { + var objs []runtime.Object + for _, n := range bootstrap.MemoryTokenSecretNames[1:] { // omit the first + objs = append(objs, tokenSecret(n, "deadbeef")) + } + res := memoryTokenSecrets{}.Check(context.Background(), bundleWith(objs...)) + assert.Equal(t, Failed, res.Status) + assert.Contains(t, res.Detail, bootstrap.MemoryTokenSecretNames[0], "names the missing Secret") + assert.False(t, res.NotFound, "some present → not a clean NotFound") +} + +func TestMemoryTokenSecrets_EmptyTokenFails(t *testing.T) { + var objs []runtime.Object + objs = append(objs, tokenSecret(bootstrap.MemoryTokenSecretNames[0], "")) // present but empty + for _, n := range bootstrap.MemoryTokenSecretNames[1:] { + objs = append(objs, tokenSecret(n, "deadbeef")) + } + res := memoryTokenSecrets{}.Check(context.Background(), bundleWith(objs...)) + assert.Equal(t, Failed, res.Status) + assert.Contains(t, res.Detail, "empty token") +} + +func TestMemoryTokenSecrets_AllAbsentNotFound(t *testing.T) { + res := memoryTokenSecrets{}.Check(context.Background(), bundleWith()) + assert.Equal(t, Failed, res.Status) + assert.True(t, res.NotFound, "none present → NotFound so trimmed profiles can skip") + // The tokens are minted by the pre-start bootstrap Job, NOT the operator + // runnable (that design was abandoned). A cold-start diagnosis must point at + // the Job so an operator doesn't hunt for operator mint logs that never exist. + assert.Contains(t, res.Detail, "bootstrap Job", "attributes a cold-start miss to the Job, not the operator") + assert.NotContains(t, res.Detail, "operator bootstrap", "must not misdirect to the abandoned operator-mint design") +} diff --git a/cmd/oap/internal/installcmd/init.go b/cmd/oap/internal/installcmd/init.go index 385d814d..16352ac7 100644 --- a/cmd/oap/internal/installcmd/init.go +++ b/cmd/oap/internal/installcmd/init.go @@ -3,8 +3,6 @@ package installcmd import ( "bufio" "context" - "crypto/rand" - "encoding/hex" "fmt" "io" "os" @@ -14,9 +12,7 @@ import ( "github.com/spf13/cobra" "golang.org/x/term" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -32,6 +28,7 @@ import ( "github.com/authzed/openagentprimitives/cmd/oap/internal/settingscmd" "github.com/authzed/openagentprimitives/cmd/oap/internal/wait" spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" "github.com/authzed/openagentprimitives/pkg/platform/cloud" idpregistry "github.com/authzed/openagentprimitives/pkg/platform/identity/idp/registry" "github.com/authzed/openagentprimitives/pkg/platform/manifests" @@ -1102,27 +1099,10 @@ func ensureMemoryTokenSecret(ctx context.Context, restCfg *rest.Config, name str if err != nil { return err } - var existing corev1.Secret - if err := cli.Get(ctx, types.NamespacedName{ - Namespace: "agentprimitives-system", - Name: name, - }, &existing); err == nil { - return nil // already exists; idempotent - } else if !apierrors.IsNotFound(err) { - return fmt.Errorf("get %s secret: %w", name, err) - } - - buf := make([]byte, 32) - if _, err := rand.Read(buf); err != nil { - return fmt.Errorf("generate token: %w", err) - } - sec := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "agentprimitives-system", - }, - Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{"token": []byte(hex.EncodeToString(buf))}, - } - return cli.Create(ctx, sec) + // The mint logic lives in pkg/platform/bootstrap so the operator's leader-only + // bootstrap Runnable and this install fast path share ONE idempotent + // Create-if-absent path. A direct client.New client reads live (not a + // label-filtered cache), so it is safe as both reader and writer here. + _, err = bootstrap.EnsureTokenSecret(ctx, cli, cli, bootstrap.TokenSecretNamespace, name) + return err } diff --git a/cmd/oap/internal/installcmd/install.go b/cmd/oap/internal/installcmd/install.go index d3676f20..85853d58 100644 --- a/cmd/oap/internal/installcmd/install.go +++ b/cmd/oap/internal/installcmd/install.go @@ -1,7 +1,6 @@ package installcmd import ( - "bytes" "context" "crypto/rand" "encoding/hex" @@ -9,7 +8,6 @@ import ( "errors" "fmt" "io" - "net/url" "os" "os/exec" "os/signal" @@ -20,7 +18,6 @@ import ( "github.com/spf13/cobra" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -49,10 +46,9 @@ import ( "github.com/authzed/openagentprimitives/cmd/oap/internal/wait" "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" "github.com/authzed/openagentprimitives/pkg/authz/spicedb" - "github.com/authzed/openagentprimitives/pkg/channels/channelevents" "github.com/authzed/openagentprimitives/pkg/platform/apimage" + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" "github.com/authzed/openagentprimitives/pkg/platform/cloud" - "github.com/authzed/openagentprimitives/pkg/platform/identity/passthroughlink" "github.com/authzed/openagentprimitives/pkg/platform/manifests" apnats "github.com/authzed/openagentprimitives/pkg/platform/nats" ) @@ -2716,80 +2712,11 @@ func ensureSpiceDBToken(ctx context.Context, restCfg *rest.Config) error { return ensureSpiceDBTokenWithClient(ctx, cli) } -// mintSpiceDBToken returns a fresh pre-shared key: 32 random bytes, hex-encoded. -func mintSpiceDBToken() ([]byte, error) { - buf := make([]byte, 32) - if _, err := rand.Read(buf); err != nil { - return nil, fmt.Errorf("generate token: %w", err) - } - return []byte(hex.EncodeToString(buf)), nil -} - -// ensureSpiceDBTokenWithClient creates spicebox-spicedb-token (32 random -// bytes, hex-encoded) if it does not already exist, and guarantees -// preshared_key == token. Neither existing value is ever rotated: -// -// - `token` only: the shape oap itself provisions (and older installs). It is -// authoritative, and `preshared_key` is mirrored from it. -// - `preshared_key` only: the shape the spicedb-operator documents, and what -// an External Secrets Operator / GitOps pre-provision produces. It is -// ADOPTED — SpiceDB reads only this key, so overwriting it boots SpiceDB -// with an empty pre-shared key and wedges the install — and `token` is -// backfilled from it for oap's own components. -// - both, disagreeing: `token` wins, because that is the key every oap -// component reads. -// -// Idempotent in every one of those shapes: a converged secret is not written -// again, so repeated installs neither rotate nor churn resourceVersion. +// ensureSpiceDBTokenWithClient delegates to the shared bootstrap minter (the +// pre-start Job uses the same logic): create spicebox-spicedb-token if absent, +// reconcile the token/preshared_key dual-key shape, never rotate. func ensureSpiceDBTokenWithClient(ctx context.Context, cli client.Client) error { - var existing corev1.Secret - err := cli.Get(ctx, types.NamespacedName{ - Namespace: "agentprimitives-system", - Name: "spicebox-spicedb-token", - }, &existing) - if err == nil { - tok := existing.Data["token"] - pre := existing.Data["preshared_key"] - if len(tok) > 0 && bytes.Equal(tok, pre) { - return nil - } - if existing.Data == nil { - existing.Data = map[string][]byte{} - } - switch { - case len(tok) > 0: - existing.Data["preshared_key"] = tok - case len(pre) > 0: - existing.Data["token"] = pre - default: - minted, mintErr := mintSpiceDBToken() - if mintErr != nil { - return mintErr - } - existing.Data["token"] = minted - existing.Data["preshared_key"] = minted - } - if err := cli.Update(ctx, &existing); err != nil { - return fmt.Errorf("update spicedb token secret: %w", err) - } - return nil - } - if !apierrors.IsNotFound(err) { - return fmt.Errorf("get spicedb token secret: %w", err) - } - tok, err := mintSpiceDBToken() - if err != nil { - return err - } - sec := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "spicebox-spicedb-token", - Namespace: "agentprimitives-system", - }, - Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{"token": tok, "preshared_key": tok}, - } - return cli.Create(ctx, sec) + return bootstrap.EnsureSpiceDBToken(ctx, cli, cli) } // ensureSpiceDBDatastoreURI creates a client from restCfg and delegates to @@ -2810,40 +2737,7 @@ func ensureSpiceDBDatastoreURI(ctx context.Context, restCfg *rest.Config) error // value. Requires spicebox-spicedb-token to already exist (ensureSpiceDBToken // must run first); it hard-errors via Get otherwise. func ensureSpiceDBDatastoreURIWithClient(ctx context.Context, cli client.Client) error { - var pg corev1.Secret - if err := cli.Get(ctx, types.NamespacedName{ - Namespace: "agentprimitives-system", - Name: "spicebox-postgres-token", - }, &pg); err != nil { - return fmt.Errorf("get postgres token secret: %w", err) - } - pw := string(pg.Data["password"]) - if pw == "" { - return fmt.Errorf("postgres token secret has no password") - } - uri := (&url.URL{ - Scheme: "postgres", - User: url.UserPassword("postgres", pw), - Host: "spicebox-postgres.agentprimitives-system.svc:5432", - Path: "/spicedb", - RawQuery: "sslmode=disable", - }).String() - - var sp corev1.Secret - if err := cli.Get(ctx, types.NamespacedName{ - Namespace: "agentprimitives-system", - Name: "spicebox-spicedb-token", - }, &sp); err != nil { - return fmt.Errorf("get spicedb token secret: %w", err) - } - if string(sp.Data["datastore_uri"]) == uri { - return nil - } - if sp.Data == nil { - sp.Data = map[string][]byte{} - } - sp.Data["datastore_uri"] = []byte(uri) - return cli.Update(ctx, &sp) + return bootstrap.EnsureSpiceDBDatastoreURI(ctx, cli, cli) } // ensureSpiceDBBootstrap creates/updates the spicebox-spicedb-bootstrap @@ -3015,53 +2909,7 @@ func ensureNATSIdentity(ctx context.Context, restCfg *rest.Config) error { if err != nil { return err } - return ensureNATSIdentityWithClient(ctx, cli) -} - -func ensureNATSIdentityWithClient(ctx context.Context, c client.Client) error { - var existing corev1.Secret - err := c.Get(ctx, types.NamespacedName{ - Namespace: "agentprimitives-system", - Name: "spicebox-nats-identity", - }, &existing) - if err == nil { - return nil - } - if !apierrors.IsNotFound(err) { - return fmt.Errorf("get nats identity secret: %w", err) - } - - id, err := apnats.GenerateIdentity() - if err != nil { - return fmt.Errorf("generate nats identity: %w", err) - } - - // The interpolated values (OperatorJWT, AccountPublicKey, AccountJWT) are - // base32/base64url-encoded and contain no whitespace, quotes, or braces, so - // they are safe to inline into the NATS config format without escaping. - authConf := fmt.Sprintf(`operator: %s -resolver: MEMORY -resolver_preload: { - %s: %s -} -`, id.OperatorJWT, id.AccountPublicKey, id.AccountJWT) - - sec := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "spicebox-nats-identity", - Namespace: "agentprimitives-system", - }, - Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{ - "operator-jwt": []byte(id.OperatorJWT), - "account-jwt": []byte(id.AccountJWT), - "account-public-key": []byte(id.AccountPublicKey), - "account-signing-seed": id.AccountSigningSeed, - "account-signing-public-key": []byte(id.AccountSigningPublicKey), - "auth.conf": []byte(authConf), - }, - } - return c.Create(ctx, sec) + return bootstrap.EnsureNATSIdentity(ctx, cli) } // ensureNATSTLS creates the spicebox-nats-tls Secret if it does not already @@ -3073,45 +2921,7 @@ func ensureNATSTLS(ctx context.Context, restCfg *rest.Config) error { if err != nil { return err } - return ensureNATSTLSWithClient(ctx, cli) -} - -func ensureNATSTLSWithClient(ctx context.Context, c client.Client) error { - var existing corev1.Secret - err := c.Get(ctx, types.NamespacedName{ - Namespace: "agentprimitives-system", - Name: "spicebox-nats-tls", - }, &existing) - if err == nil { - return nil - } - if !apierrors.IsNotFound(err) { - return fmt.Errorf("get nats tls secret: %w", err) - } - - m, err := apnats.GenerateTLS([]string{ - "spicebox-nats", - "spicebox-nats.agentprimitives-system", - "spicebox-nats.agentprimitives-system.svc", - "spicebox-nats.agentprimitives-system.svc.cluster.local", - }) - if err != nil { - return fmt.Errorf("generate nats tls: %w", err) - } - - sec := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "spicebox-nats-tls", - Namespace: "agentprimitives-system", - }, - Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{ - "ca.crt": m.CACertPEM, - "tls.crt": m.ServerCertPEM, - "tls.key": m.ServerKeyPEM, - }, - } - return c.Create(ctx, sec) + return bootstrap.EnsureNATSTLS(ctx, cli) } func ensureWebhookTLSWithClient(ctx context.Context, c client.Client) ([]byte, error) { @@ -3189,177 +2999,12 @@ func patchWebhookVWCBundle(ctx context.Context, c client.Client, caPEM []byte) e // subjects), storing each as a Secret alongside the CA cert. It runs after // ensureNATSIdentity and ensureNATSTLS: both prerequisite Secrets must exist. // Each output Secret is created idempotently — a re-install does not re-mint. -// The principal name minted into each creds file. Named once because the name -// is also the input to apnats.InboxSubjectFor, and the client rederives the -// matching prefix from the same name in the JWT — writing it twice invites the -// two to drift, which breaks request/reply rather than merely widening a grant. -const ( - channelsdNATSUser = "channelsd" - cliNATSUser = "cli" - webdNATSUser = "webd" -) - -// webdNATSGrant is webd's minted NATS user grant. webd SUBSCRIBES to session -// out-subjects (chat + live-view mirror) AND does NATS request-reply: the -// builtin web-chat listener and the live-view submit view_message via -// RequestIn, so webd both PUBLISHES the request and must SUBSCRIBE to its -// reply inbox. The original "never publishes, subscribe-only" assumption -// predated those surfaces and was the bug — a missing inbox entry made every -// view-originated send time out even though the message was delivered. -// -// PubAllow is enumerated because an OMITTED one is publish-ANYWHERE, not -// deny-all (see apnats.UserGrant). This grant used to omit it, which left the -// browser-facing process holding publish rights on the entire control bus: -// ap.revocation, every session's out-subjects (forging the agent's own voice), -// and every inbound kind — including in.user_message, whose channelsd handler -// is the audit-signing inbound writer webd is deliberately kept away from -// (pkg/channels/channelkinds/clienthosted.SubmitUserMessage spells out why: webd holds -// no memory-write credential, so it must round-trip through view_message). -// -// The five subjects below are webd's complete publish surface, each a verified -// call site: view_message (chat SubmitUserMessage and the /interact -// user_message | annotation_batch | mcp_ui_action kinds, all via -// channelkinds.RequestViewMessage), interrupt_request and interaction_decision -// (clienthosted.Listener), resurface_request (channelkinds. -// PublishResurfaceRequest) and app_tool_call (webui/interact). webd links -// pkg/channels/channelsd/pipeline but only for its Authz interface type — it constructs -// no Pipeline, so none of that package's publishes can fire here. -// -// webd needs no publish grant on any inbox: it is never a NATS responder -// (nothing in its import graph calls msg.Respond), and a requester publishes to -// the target subject, not to its own reply inbox. -// -// TestWebdNATSGrantPublishSurface asserts every line of this against a real -// server; add a subject there and here together. -var webdNATSGrant = apnats.UserGrant{ - Name: webdNATSUser, - PubAllow: []string{ - "ap.session.*.*.in.view_message", - "ap.session.*.*.in.interrupt_request", - "ap.session.*.*.in.interaction_decision", - "ap.session.*.*.in.resurface_request", - "ap.session.*.*.in.app_tool_call", - // ui_data_binding is the agent-UI's own data path: pkg/web/webui/agentui's - // bindingsHandler resolves every `source: tool` binding by asking the - // session's runner, via pkg/web/uibindings/tool. - // - // Its absence is why that feature had never worked on a real install. - // The failure gave nothing away: a publish the server refuses does not - // fail the caller's request — NATS reports the violation as an ASYNC - // error on the connection, while the request itself simply waits for a - // reply that can now never come and dies on its own timeout. So every - // binding surfaced as "the runner did not answer", which reads as a - // missing or busy runner and is why the first diagnosis of it chased a - // sleeping session instead of a permission. - "ap.session.*.*.in.ui_data_binding", - // ui_presence is the liveness heartbeat webd republishes while a - // viewer has an agent-UI open and focused, so the runner answering - // that UI's bindings does not idle out underneath them. - // - // Added in the SAME change as the kind and as its row in - // TestWebdNATSGrantPublishSurface, which is the discipline the line - // above exists to teach: ui_data_binding shipped without its grant and - // the whole agent-UI data path silently never worked. - "ap.session.*.*.in.ui_presence", - // ui_action is the agent-UI's INVOKE path: pkg/web/webui/agentui's - // actions handler sends the declared action to the session's runner and - // relays the synchronous reply. It shipped without this entry, so every - // agent-UI action failed exactly the way ui_data_binding did — see above. - "ap.session.*.*.in.ui_action", - // channelevents.WebhookInboundSubject — not a *.* session template - // (there is no session yet at delivery time, see that constant's own - // doc) but the same failure shape as ui_data_binding above: without - // this entry, pkg/web/webui/channelwebhook's Publish call is silently - // refused by the server, its returned error is nil either way (NATS - // reports permission violations asynchronously on the connection, not - // to the caller), and the handler answers the webhook sender with 202 - // for a delivery that was actually dropped — the one status the - // route's whole retry contract promises will never happen. Referenced - // via the exported constant, not a copied literal, so this and the - // route's own Publish call cannot drift apart. - channelevents.WebhookInboundSubject, - }, - SubAllow: []string{"ap.session.*.*.out.>", apnats.InboxSubjectFor(webdNATSUser)}, -} - func ensureNATSClientCreds(ctx context.Context, restCfg *rest.Config, logf func(string, ...any)) error { cli, err := client.New(restCfg, client.Options{Scheme: kube.Scheme}) if err != nil { return err } - return ensureNATSClientCredsWithClient(ctx, cli, logf) -} - -// logf narrates a credential rotation. It is threaded rather than written to a -// package-level writer because a rotation is the one thing here an operator -// must be able to read afterwards: everything else this function does is a -// no-op on a healthy re-run. nil is accepted and discards. -func ensureNATSClientCredsWithClient(ctx context.Context, c client.Client, logf func(string, ...any)) error { - if logf == nil { - logf = func(string, ...any) {} - } - var identitySec corev1.Secret - if err := c.Get(ctx, types.NamespacedName{ - Namespace: "agentprimitives-system", - Name: "spicebox-nats-identity", - }, &identitySec); err != nil { - return fmt.Errorf("get nats identity secret (run ensureNATSIdentity first): %w", err) - } - - var tlsSec corev1.Secret - if err := c.Get(ctx, types.NamespacedName{ - Namespace: "agentprimitives-system", - Name: "spicebox-nats-tls", - }, &tlsSec); err != nil { - return fmt.Errorf("get nats tls secret (run ensureNATSTLS first): %w", err) - } - caCrt := tlsSec.Data["ca.crt"] - if len(caCrt) == 0 { - return fmt.Errorf("nats tls secret is missing ca.crt") - } - - // MintUser only consults AccountPublicKey + AccountSigningSeed. - id := &apnats.Identity{ - AccountPublicKey: string(identitySec.Data["account-public-key"]), - AccountSigningSeed: identitySec.Data["account-signing-seed"], - } - if id.AccountPublicKey == "" || len(id.AccountSigningSeed) == 0 { - return fmt.Errorf("nats identity secret is missing account-public-key or account-signing-seed") - } - - // Creds are minted lazily, per Secret, by ensureNATSCredsSecret: a Secret - // already carrying the shipped grant is left byte-identical (minting is - // non-deterministic — a fresh user nkey every call — so writing - // unconditionally would rotate every principal's credential on every - // re-run), and one carrying a STALE grant is re-minted. See that - // function's doc for why "already exists" is not a sufficient answer. - // - // Both the channelsd broker and the `oap` CLI get broad `ap.>`. channelsd IS - // the broker (it relays every session's in/out traffic and serves - // read_thread_history via NATS request/reply). The `oap chat` command embeds - // a full channelsd pipeline + outbound relay in-process, so the CLI - // legitimately publishes the same `ap.session.*.*.out.*` and `ap.channel.*` - // subjects. The runner's per-session JWT (minted in the AgentSession - // controller) stays narrowly scoped to its own session. - // - // Both keep `_INBOX.>` on PUBLISH and narrow only SUBSCRIBE. Both are - // responders — `m.Respond` publishes to whichever principal asked, on a - // top-level inbox subject NOT under `ap.>` — and a responder cannot know - // its requesters' names, so its publish side must span the whole inbox - // root. Subscribing is the direction that leaks (any `_INBOX.>` subscriber - // reads every reply on the bus), and that is the direction scoped here. - // - // The channelsd creds Secret is mounted by three binaries — channelsd, - // authzd and the operator — which is exactly why the inbox prefix is - // derived from the JWT's name rather than from each process's connection - // name: all three resolve to the same "channelsd" principal and the same - // inbox root, with their own nuids underneath. - for _, cred := range natsClientCreds() { - if err := ensureNATSCredsSecret(ctx, c, cred, id, caCrt, logf); err != nil { - return err - } - } - return nil + return bootstrap.EnsureNATSClientCreds(ctx, cli, logf) } // ensureWebdExternalURLConfigMap creates the spicebox-webd-external-url @@ -3467,63 +3112,8 @@ func ensureWebdExternalURLConfigMap(ctx context.Context, logf func(string, ...an // ensureWebdExternalURLConfigMap for why these do not write straight to a // writer while the checklist's live region is up. func ensurePassthroughLinkSigningKey(ctx context.Context, logf func(string, ...any), c client.Client) error { - var existing corev1.Secret - err := c.Get(ctx, types.NamespacedName{ - Namespace: "agentprimitives-system", - Name: v1alpha1.PassthroughLinkSigningKeySecret, - }, &existing) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("get %s secret: %w", v1alpha1.PassthroughLinkSigningKeySecret, err) - } - exists := err == nil - // unusable is why the existing key is being replaced — the readers' own - // verdict, phrased in their words ("is empty", "is not valid hex", "is too - // short"). It doubles as the text of the repair message below, so an - // operator whose provisioner owns this Secret sees that their material was - // REJECTED rather than a bare "regenerated". DecodeHexKey deliberately never - // echoes key bytes into its error (see its doc), so this is log-safe. - var unusable error - if exists { - if _, unusable = passthroughlink.DecodeHexKey(existing.Data["key"]); unusable == nil { - logf(" %s Secret exists with a usable key, skipping", v1alpha1.PassthroughLinkSigningKeySecret) - return nil - } - } - buf := make([]byte, 32) - if _, err := rand.Read(buf); err != nil { - return fmt.Errorf("generate passthrough link signing key: %w", err) - } - encoded := []byte(hex.EncodeToString(buf)) - - // Update in place when the Secret is already there, so we fill in the key - // without disturbing labels, annotations, or other data keys its - // provisioner owns. - if exists { - if existing.Data == nil { - existing.Data = map[string][]byte{} - } - existing.Data["key"] = encoded - if err := c.Update(ctx, &existing); err != nil { - return fmt.Errorf("update %s secret: %w", v1alpha1.PassthroughLinkSigningKeySecret, err) - } - logf(" regenerated the key in the existing %s Secret (%v)", - v1alpha1.PassthroughLinkSigningKeySecret, unusable) - return nil - } - - sec := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: v1alpha1.PassthroughLinkSigningKeySecret, - Namespace: "agentprimitives-system", - }, - Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{"key": encoded}, - } - if err := c.Create(ctx, sec); err != nil { - return fmt.Errorf("create %s secret: %w", v1alpha1.PassthroughLinkSigningKeySecret, err) - } - logf(" generated %s Secret", v1alpha1.PassthroughLinkSigningKeySecret) - return nil + _, err := bootstrap.EnsurePassthroughLinkKey(ctx, c, c, logf) + return err } // ensureSlackOAuthSecret creates the spicebox-slack-oauth Secret with empty @@ -3573,200 +3163,6 @@ func ensureSlackOAuthSecret(ctx context.Context, logf func(string, ...any), c cl return nil } -// createNATSCredsSecret creates a per-client NATS creds Secret if it does not -// already exist. Idempotent per-secret so a re-install does not re-mint creds. -// natsClientCred pairs one creds Secret with the grant its user JWT must -// carry. Pairing them in one value is what lets the mint and the staleness -// check read the SAME grant: a subject added to a grant reaches both by -// construction, instead of by remembering to change a second place. -type natsClientCred struct { - secretName string - grant apnats.UserGrant -} - -// natsClientCreds is every principal `oap install` mints a bus credential for. -// -// A function rather than a var because two of the three grants call -// apnats.InboxSubjectFor, and building them at package-init time would put -// their evaluation order before the constants they read in a way nothing -// enforces; there is one caller and it runs once per install. -func natsClientCreds() []natsClientCred { - return []natsClientCred{ - { - secretName: "spicebox-channelsd-nats-creds", - grant: apnats.UserGrant{ - Name: channelsdNATSUser, - PubAllow: []string{"ap.>", "_INBOX.>"}, - SubAllow: []string{"ap.>", apnats.InboxSubjectFor(channelsdNATSUser)}, - }, - }, - { - secretName: "spicebox-cli-nats-creds", - grant: apnats.UserGrant{ - Name: cliNATSUser, - PubAllow: []string{"ap.>", "_INBOX.>"}, - SubAllow: []string{"ap.>", apnats.InboxSubjectFor(cliNATSUser)}, - }, - }, - {secretName: "spicebox-webd-nats-creds", grant: webdNATSGrant}, - } -} - -// ensureNATSCredsSecret reconciles one creds Secret against the grant this -// binary ships, minting a user JWT only when the stored one does not already -// carry it. -// -// "The Secret already exists" is NOT a sufficient answer, and treating it as -// one was a real, silent defect: a user JWT is a frozen copy of the grant it -// was minted from, so a cluster installed before a subject was added to a -// grant kept a JWT without it forever. Every upgrade re-ran this, saw the -// Secret, and skipped. The webd grant gaining ap.channel.webhook_inbound is -// the case that exposed it — nats-server reports a permission violation -// asynchronously, so the webhook handler's Publish returned nil and it -// answered 202 Accepted for a delivery the bus had dropped. -// -// A re-mint rotates the user nkey, so it MUST be rare: minting is -// non-deterministic and an unconditional write would rotate every principal on -// every re-run, disconnecting live clients for nothing. apnats.UserGrant's -// CredsDrift is the gate, and it compares allow-lists as sets so ordering -// alone never triggers a rotation. -// -// ca.crt is refreshed on the same write, because a creds Secret carrying a CA -// that no longer matches the server's is the same class of stale. -func ensureNATSCredsSecret( - ctx context.Context, c client.Client, cred natsClientCred, id *apnats.Identity, caCrt []byte, - logf func(string, ...any), -) error { - key := types.NamespacedName{Namespace: "agentprimitives-system", Name: cred.secretName} - - var existing corev1.Secret - err := c.Get(ctx, key, &existing) - switch { - case err == nil: - drift := cred.grant.CredsDrift(string(existing.Data["nats.creds"])) - if drift == "" && bytes.Equal(existing.Data["ca.crt"], caCrt) { - return nil - } - if drift == "" { - drift = "the stored CA no longer matches the NATS server's" - } - // Say why out loud. The whole defect class here is a permission the - // bus refuses without telling anyone, so an operator who later sees a - // component reconnect deserves a line explaining what rotated and why. - logf("re-minting %s: %s", cred.secretName, drift) - - creds, mintErr := apnats.MintUser(id, cred.grant) - if mintErr != nil { - return fmt.Errorf("re-mint %s nats creds: %w", cred.grant.Name, mintErr) - } - if existing.Data == nil { - existing.Data = map[string][]byte{} - } - existing.Data["nats.creds"] = []byte(creds) - existing.Data["ca.crt"] = caCrt - if err := c.Update(ctx, &existing); err != nil { - return fmt.Errorf("update %s secret: %w", cred.secretName, err) - } - // A running process read its creds file once, at connect time; the - // kubelet refreshing the mounted file changes nothing for it. Without - // this the re-mint lands in the cluster and the stale JWT stays on the - // wire — the exact half-fix this function exists to close. - return restartWorkloadsMountingSecret(ctx, c, key, logf) - - case apierrors.IsNotFound(err): - creds, mintErr := apnats.MintUser(id, cred.grant) - if mintErr != nil { - return fmt.Errorf("mint %s nats creds: %w", cred.grant.Name, mintErr) - } - sec := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: cred.secretName, - Namespace: "agentprimitives-system", - }, - Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{ - "nats.creds": []byte(creds), - "ca.crt": caCrt, - }, - } - return c.Create(ctx, sec) - - default: - return fmt.Errorf("get %s secret: %w", cred.secretName, err) - } -} - -// restartWorkloadsMountingSecret rolls every Deployment in the Secret's -// namespace whose pod template mounts it, so a rotated credential actually -// reaches the processes using it. -// -// Which Deployments those are is DERIVED from the live pod specs rather than -// transcribed into a table here: the channelsd creds Secret alone is mounted -// by three different binaries, and a table would go stale the first time a -// fourth mounted it — silently, in the direction of "the rotation did not -// reach someone". -// -// Nothing to roll is the normal case on a first install: the component -// manifests have not been applied yet, so whatever starts later mounts the -// current Secret and needs no restart. -func restartWorkloadsMountingSecret(ctx context.Context, c client.Client, secret types.NamespacedName, logf func(string, ...any)) error { - var deps appsv1.DeploymentList - if err := c.List(ctx, &deps, client.InNamespace(secret.Namespace)); err != nil { - return fmt.Errorf("list deployments in %s to roll after rotating %s: %w", - secret.Namespace, secret.Name, err) - } - stamp := time.Now().UTC().Format(time.RFC3339Nano) - for i := range deps.Items { - d := &deps.Items[i] - if !podSpecMountsSecret(&d.Spec.Template.Spec, secret.Name) { - continue - } - patch := []byte(fmt.Sprintf( - `{"spec":{"template":{"metadata":{"annotations":{"ap.authzed.com/restartedAt":%q}}}}}`, stamp)) - if err := c.Patch(ctx, d, client.RawPatch(types.StrategicMergePatchType, patch)); err != nil { - return fmt.Errorf("rollout-restart %s/%s after rotating %s: %w", - d.Namespace, d.Name, secret.Name, err) - } - logf("restarting %s to pick up the re-minted %s", d.Name, secret.Name) - } - return nil -} - -// podSpecMountsSecret reports whether the pod template reads the named Secret -// — as a volume (the creds-file shape) or through env/envFrom, so a future -// consumer that reads it a different way is still rolled. -func podSpecMountsSecret(spec *corev1.PodSpec, name string) bool { - for _, v := range spec.Volumes { - if v.Secret != nil && v.Secret.SecretName == name { - return true - } - if v.Projected == nil { - continue - } - for _, src := range v.Projected.Sources { - if src.Secret != nil && src.Secret.Name == name { - return true - } - } - } - containers := make([]corev1.Container, 0, len(spec.Containers)+len(spec.InitContainers)) - containers = append(containers, spec.Containers...) - containers = append(containers, spec.InitContainers...) - for _, ctr := range containers { - for _, ef := range ctr.EnvFrom { - if ef.SecretRef != nil && ef.SecretRef.Name == name { - return true - } - } - for _, e := range ctr.Env { - if e.ValueFrom != nil && e.ValueFrom.SecretKeyRef != nil && e.ValueFrom.SecretKeyRef.Name == name { - return true - } - } - } - return false -} - // ensurePostgresToken creates the spicebox-postgres-token Secret if it does // not already exist. The Secret holds the Postgres password and a full // connection URI for the in-cluster Postgres instance backing the memory @@ -3776,42 +3172,7 @@ func ensurePostgresToken(ctx context.Context, restCfg *rest.Config) error { if err != nil { return err } - var existing corev1.Secret - err = cli.Get(ctx, types.NamespacedName{ - Namespace: "agentprimitives-system", - Name: "spicebox-postgres-token", - }, &existing) - if err == nil { - return nil - } - if !apierrors.IsNotFound(err) { - return fmt.Errorf("get postgres token secret: %w", err) - } - buf := make([]byte, 16) - if _, err := rand.Read(buf); err != nil { - return fmt.Errorf("generate postgres password: %w", err) - } - password := hex.EncodeToString(buf) - pgURL := &url.URL{ - Scheme: "postgres", - User: url.UserPassword("postgres", password), - Host: "spicebox-postgres.agentprimitives-system.svc:5432", - Path: "/memory", - RawQuery: "sslmode=disable", - } - uri := pgURL.String() - sec := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "spicebox-postgres-token", - Namespace: "agentprimitives-system", - }, - Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{ - "password": []byte(password), - "uri": []byte(uri), - }, - } - return cli.Create(ctx, sec) + return bootstrap.EnsurePostgresToken(ctx, cli, cli) } // ensureNeo4jToken creates the spicebox-neo4j-token Secret if it does not diff --git a/cmd/oap/internal/installcmd/render.go b/cmd/oap/internal/installcmd/render.go new file mode 100644 index 00000000..cff95e32 --- /dev/null +++ b/cmd/oap/internal/installcmd/render.go @@ -0,0 +1,187 @@ +package installcmd + +import ( + "context" + "fmt" + "io" + + "github.com/spf13/cobra" + + "github.com/authzed/openagentprimitives/cmd/oap/internal/apcmd" + "github.com/authzed/openagentprimitives/cmd/oap/internal/cliout" + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" + "github.com/authzed/openagentprimitives/pkg/platform/cloud" + "github.com/authzed/openagentprimitives/pkg/platform/manifests" +) + +// MemoryTokenLinks names existing Secrets to link per component instead of +// operator-minting. An empty string means "operator mints this one" (the +// default). The three are kept separate on purpose: webd must not hold a +// write-capable memory credential, so linking is per-component. +type MemoryTokenLinks struct { + Channelsd string + Webd string + Authzd string +} + +// RenderConfig is the full input to `oap render`. It is populated from flags and +// is the sole input to WriteOverlay (which must be a pure function of it). +type RenderConfig struct { + OutputDir string + ClusterKind string + Tags manifests.Tags + StorageClass string + Routing WebdRoutingOpts + WithPostgres bool + MemoryToken MemoryTokenLinks + NoDigestPin bool + // ExternalSpiceDB/ExternalPostgres: the user points at an EXTERNAL instance and + // provides its token Secret themselves, so the bootstrap Job must NOT mint that + // token (it would fight the external instance). + ExternalSpiceDB bool + ExternalPostgres bool + // ExternalWebhookTLS: do not emit the cert-manager Issuer/Certificate for the + // operator webhook serving cert; you provide spicebox-webhook-tls + the VWC + // caBundle yourself (for clusters without cert-manager). + ExternalWebhookTLS bool + Strat cloud.Strategy +} + +func NewRenderCmd(g *apcmd.Globals) *cobra.Command { + cfg := RenderConfig{OutputDir: "deploy/oap"} + var validate bool + cmd := &cobra.Command{ + Use: "render", + Short: "Render the install as a GitOps-ready kustomize overlay (no cluster mutation)", + Long: "Render the Agent Primitives install as a self-contained kustomize overlay for Argo CD / Flux.\n\n" + + "Unlike `oap install`, render never mutates a cluster (its only remote call is read-only image-digest " + + "resolution, skipped with --no-digest-pin). Commit the output directory and point your GitOps controller " + + "at it. The overlay provisions every internal Secret: a pre-start Job mints the tokens/NATS material " + + "before the operator starts, and a cert-manager Certificate populates the webhook cert (so cert-manager " + + "is a dependency). Use --external-spicedb/--external-postgres/--external-webhook-tls to provide any of " + + "those yourself, or link an existing memory token with the per-component --*-memory-token-secret flags. " + + "Verify a synced cluster with `oap check`.", + RunE: func(cmd *cobra.Command, _ []string) error { + return runRender(cmd.Context(), cmd.OutOrStdout(), g, cfg, validate) + }, + } + cmd.Flags().StringVar(&cfg.OutputDir, "output-dir", cfg.OutputDir, "Directory to write the kustomize overlay into") + cmd.Flags().StringVar(&cfg.ClusterKind, "cluster-kind", "", clusterKindFlagUsage) + cmd.Flags().StringVar(&cfg.Tags.Registry, "image-registry", "", "Rewrite first-party images to /") + cmd.Flags().StringVar(&cfg.StorageClass, "storage-class", "", "StorageClass for stateful PVCs/StatefulSets") + cmd.Flags().BoolVar(&cfg.WithPostgres, "with-postgres", false, "Include the Postgres dependency component") + cmd.Flags().BoolVar(&cfg.NoDigestPin, "no-digest-pin", false, "Emit mutable tags instead of pinning digests (skips registry digest resolution)") + cmd.Flags().BoolVar(&cfg.ExternalSpiceDB, "external-spicedb", false, "You run an external SpiceDB and provide spicebox-spicedb-token yourself; the bootstrap Job will not mint it") + cmd.Flags().BoolVar(&cfg.ExternalPostgres, "external-postgres", false, "You run an external Postgres and provide spicebox-postgres-token yourself; the bootstrap Job will not mint it") + cmd.Flags().BoolVar(&cfg.ExternalWebhookTLS, "external-webhook-tls", false, "Do not emit cert-manager resources for the operator webhook cert; you provide spicebox-webhook-tls + the VWC caBundle yourself (clusters without cert-manager)") + // Per-component only, by design: there is deliberately no "one Secret for all + // three" flag — webd's token must differ from channelsd/authzd (it is + // read-only; they are write-capable), and a single shared value would break + // that (see validateRenderFlags). + cmd.Flags().StringVar(&cfg.MemoryToken.Channelsd, "channelsd-memory-token-secret", "", "Link channelsd memory token to an existing Secret (default: operator mints)") + cmd.Flags().StringVar(&cfg.MemoryToken.Webd, "webd-memory-token-secret", "", "Link webd memory token to an existing Secret; must differ from the channelsd/authzd Secrets (webd is read-only)") + cmd.Flags().StringVar(&cfg.MemoryToken.Authzd, "authzd-memory-token-secret", "", "Link authzd memory token to an existing Secret (default: operator mints)") + cmd.Flags().StringVar(&cfg.Routing.trustedHostname, "trusted-hostname", "", "Bare hostname for webd's trusted origin; emits a Gateway + HTTPRoute (opt-in). Requires --gateway-class. Omit to bring your own ingress.") + cmd.Flags().StringVar(&cfg.Routing.sandboxHostname, "sandbox-hostname", "", "Bare hostname for webd's artifact-viewer sandbox origin (adds a second HTTPS listener + HTTPRoute). Requires --trusted-hostname.") + cmd.Flags().StringVar(&cfg.Routing.gatewayClass, "gateway-class", "", "GatewayClass name for the emitted Gateway (required with --trusted-hostname; render cannot auto-detect it).") + cmd.Flags().BoolVar(&validate, "validate", false, "After rendering, lint the overlay for completeness and consistency") + return cmd +} + +// runRender resolves the cluster strategy (without mutating the cluster), writes +// the overlay, and self-checks it. g is used only for lazy, read-only registry +// digest resolution (added by a later stage), never for mutation. +func runRender(ctx context.Context, out io.Writer, g *apcmd.Globals, cfg RenderConfig, validate bool) error { + // Resolve the strategy WITHOUT a cluster call: cloud.For on an explicit kind, + // cloud.Default when unset. A render targeting a specific GitOps cluster kind + // passes --cluster-kind, so the empty-kind path is the rare "just give me the + // default shape" case. + if cfg.Strat == nil { + if cfg.ClusterKind == "" { + strat, err := cloud.Default() + if err != nil { + return err + } + cfg.Strat = strat + } else { + strat, err := cloud.For(cfg.ClusterKind) + if err != nil { + return err + } + cfg.Strat = strat + } + } + + if err := validateRenderFlags(cfg); err != nil { + return err + } + + // Registry digest resolution is the ONLY remote call render makes, and it is + // read-only. Skipped entirely under --no-digest-pin or a local (no-registry) + // install, so an offline render never reaches out. It does not touch the + // target cluster (it queries the image registry), so g is not consulted here. + if cfg.Tags.Registry != "" && !cfg.NoDigestPin { + digests, err := resolveMissingRegistryDigests(ctx, cfg.Tags.Registry, cfg.Tags.Digests) + if err != nil { + return err + } + cfg.Tags.Digests = digests + } + + if err := WriteOverlay(cfg); err != nil { + return err + } + cliout.Step(out, "rendered kustomize overlay to %s", cfg.OutputDir) + + if validate { + return runRenderValidate(ctx, out, cfg, cfg.OutputDir) + } + if _, err := buildOverlay(cfg.OutputDir); err != nil { + return fmt.Errorf("rendered overlay failed to build: %w", err) + } + return nil +} + +// validateRenderFlags rejects configurations that cannot produce a safe GitOps +// bundle. +// +// The memory-datastore refusal is the load-bearing one: local/desktop run an +// in-memory SpiceDB whose base schema is seeded at startup via +// --datastore-bootstrap-files (spicedbModeFor(...).bootstrapsSchemaAtStartup()). +// That is safe ONLY for an ephemeral, single-replica datastore — against the +// durable, multi-replica cluster a GitOps render targets it is fatal once the +// datastore is non-empty (one replica wins, the rest crashloop forever). The +// operator-applied schema path (pkg/platform/bootstrap) is what persistent +// datastores use instead, so a memory-datastore kind simply is not a GitOps +// target. Refuse it rather than emit an unsafe bundle. +func validateRenderFlags(cfg RenderConfig) error { + if cfg.Strat != nil && spicedbModeFor(cfg.Strat.InstallProfile().SpiceDBDatastore()).bootstrapsSchemaAtStartup() { + return fmt.Errorf("cluster kind %q uses an in-memory SpiceDB datastore whose schema is bootstrapped at startup, which is fatal against the durable, multi-replica cluster a GitOps render targets; render a persistent-datastore kind (the operator applies the schema there)", cfg.Strat.Key()) + } + if cfg.Routing.trustedHostname == "" && cfg.Routing.sandboxHostname != "" { + return fmt.Errorf("--sandbox-hostname requires --trusted-hostname") + } + if cfg.Routing.trustedHostname != "" && cfg.Routing.gatewayClass == "" { + return fmt.Errorf("--trusted-hostname requires --gateway-class (render cannot auto-detect the cluster's GatewayClass)") + } + // webd's memory token is READ-ONLY by value (pkg/memory/tokens): the channelsd + // and authzd tokens are accepted for writes (append/forge the audit log), the + // webd one is not, because webd is browser-facing. Token capability is a plain + // value compare, so webd mounting the SAME Secret as channelsd or authzd + // silently hands webd that write capability. Compare EFFECTIVE names — the + // link if set, else the default name the bootstrap Job mints — because the Job + // mints the defaults regardless of links: `--webd-memory-token-secret` pointed + // at channelsd's default name (channelsd left to mint) is the same leak as + // linking both to one Secret, and must be refused just the same. + eff := func(link string, i int) string { + if link != "" { + return link + } + return bootstrap.MemoryTokenSecretNames[i] + } + webd := eff(cfg.MemoryToken.Webd, 1) + if webd == eff(cfg.MemoryToken.Channelsd, 0) || webd == eff(cfg.MemoryToken.Authzd, 2) { + return fmt.Errorf("webd's memory token (--webd-memory-token-secret, or its operator-minted default %q) must resolve to a different Secret than the channelsd/authzd memory tokens: webd is browser-facing and must not hold a write-capable memory credential", bootstrap.MemoryTokenSecretNames[1]) + } + return nil +} diff --git a/cmd/oap/internal/installcmd/render_bootstrap_job.go b/cmd/oap/internal/installcmd/render_bootstrap_job.go new file mode 100644 index 00000000..e16070a6 --- /dev/null +++ b/cmd/oap/internal/installcmd/render_bootstrap_job.go @@ -0,0 +1,105 @@ +package installcmd + +import ( + "crypto/sha256" + "encoding/hex" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/authzed/openagentprimitives/pkg/platform/apimage" + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" + "github.com/authzed/openagentprimitives/pkg/platform/manifests" +) + +// bootstrapJobFile is the overlay file holding the pre-start secret-minting Job. +const bootstrapJobFile = "bootstrap-job.yaml" + +// bootstrapJobBaseName is the stable prefix every bootstrap Job name carries; the +// content hash is appended so a changed pod template is a new object. +const bootstrapJobBaseName = "spicebox-bootstrap-secrets" + +// bootstrapJobArgs is the single source for the Job's entrypoint args, consumed +// by BOTH buildBootstrapJob and the name hash so the two cannot disagree about +// what the Job runs. +func bootstrapJobArgs(cfg RenderConfig) []string { + args := []string{"bootstrap-secrets"} + if cfg.ExternalSpiceDB { + args = append(args, "--skip-spicedb-token") + } + if cfg.ExternalPostgres { + args = append(args, "--skip-postgres-token") + } + return args +} + +// bootstrapJobName names the Job by a hash of the two inputs that make its pod +// template vary: the resolved operator image ref and the entrypoint args. A Job's +// spec.template is immutable once applied, so a GitOps re-sync after an operator +// image bump would try to patch it and be rejected (Argo/Flux SyncFailed). A +// content-addressed name instead makes a changed template a NEW object the +// controller prunes-and-creates, while a byte-identical re-render hashes the same +// → same name → a no-op apply (preserving SSA idempotency). The image ref is +// substituted the same way the emitted manifest's is, so the hash tracks exactly +// what will be written. +func bootstrapJobName(cfg RenderConfig) (string, error) { + img, err := manifests.Substitute([]byte(apimage.Operator.LocalRef()), cfg.Tags) + if err != nil { + return "", err + } + h := sha256.New() + h.Write(img) + h.Write([]byte{0}) + for _, a := range bootstrapJobArgs(cfg) { + h.Write([]byte(a)) + h.Write([]byte{0}) + } + return bootstrapJobBaseName + "-" + hex.EncodeToString(h.Sum(nil))[:10], nil +} + +// renderBootstrapJob builds the Job that mints the internal Secrets a +// GitOps-applied install needs, as marshaled YAML with the operator image +// rewritten by the render's substitution. It reuses the operator's +// ServiceAccount (which already holds Secret create/get/list RBAC) and runs the +// operator image's `bootstrap-secrets` entrypoint. +// +// Ordering is by eventual convergence, not sync-waves: the Job, the namespace, +// and the operator SA it depends on are all in the same bundle, so a negative +// wave would run the Job before its own prerequisites. Applied together, the Job +// retries (backoffLimit) until the namespace/SA exist and the operator +// crash-loops until the tokens appear — both designed-for, self-healing states. +func renderBootstrapJob(cfg RenderConfig) ([]byte, error) { + name, err := bootstrapJobName(cfg) + if err != nil { + return nil, err + } + y, err := marshalClean(buildBootstrapJob(cfg, name)) + if err != nil { + return nil, err + } + // Rewrite the operator image ref (and apply any token links — a no-op here). + return processBaseBytes(y, cfg) +} + +func buildBootstrapJob(cfg RenderConfig, name string) *batchv1.Job { + backoff := int32(6) + return &batchv1.Job{ + TypeMeta: metav1.TypeMeta{APIVersion: "batch/v1", Kind: "Job"}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: bootstrap.TokenSecretNamespace}, + Spec: batchv1.JobSpec{ + BackoffLimit: &backoff, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + ServiceAccountName: "spicebox-operator", + RestartPolicy: corev1.RestartPolicyOnFailure, + Containers: []corev1.Container{{ + Name: "bootstrap-secrets", + Image: apimage.Operator.LocalRef(), + Args: bootstrapJobArgs(cfg), + }}, + }, + }, + }, + } +} diff --git a/cmd/oap/internal/installcmd/render_components.go b/cmd/oap/internal/installcmd/render_components.go new file mode 100644 index 00000000..2b884306 --- /dev/null +++ b/cmd/oap/internal/installcmd/render_components.go @@ -0,0 +1,84 @@ +package installcmd + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + + "github.com/authzed/openagentprimitives/pkg/platform/manifests" +) + +// dependencyComponent describes one optional, opt-in dependency bundle the render +// can include as a kustomize Component. +type dependencyComponent struct { + name string + enabled func(RenderConfig) bool + load func() ([][]byte, error) +} + +// dependencyComponents is the fixed, ordered set of optional dependencies. Order +// is fixed so the emitted `components:` list (and the on-disk layout) is +// deterministic across renders. +// +// Postgres only, by design: its token is Job-minted, so --with-postgres renders +// a bundle that works end to end. Graphiti/Neo4j are omitted until a follow-up +// adds the external-LLM-key story — graphiti needs a user-supplied +// spicebox-graphiti-config (OpenAI key) the overlay cannot mint, which render's +// "nothing external by default" contract has no place for yet. (The install path +// still supports them; see install.go / manifests.Graphiti/Neo4j.) +var dependencyComponents = []dependencyComponent{ + {"postgres", func(c RenderConfig) bool { return c.WithPostgres }, manifests.Postgres}, +} + +// enabledComponentNames returns the enabled dependency names in fixed order. Used +// by both the writer and the kustomization emitter so they cannot disagree. +func enabledComponentNames(cfg RenderConfig) []string { + var names []string + for _, d := range dependencyComponents { + if d.enabled(cfg) { + names = append(names, d.name) + } + } + return names +} + +// writeDependencyComponents writes a kustomize Component directory per enabled +// dependency, with the storage class injected into each stateful doc (a no-op +// for non-PVC/StatefulSet kinds). Creates components/ only when at least one +// dependency is enabled. +func writeDependencyComponents(cfg RenderConfig) error { + for _, d := range dependencyComponents { + if !d.enabled(cfg) { + continue + } + docs, err := d.load() + if err != nil { + return fmt.Errorf("load %s component: %w", d.name, err) + } + if cfg.StorageClass != "" { + for i := range docs { + m, err := manifests.InjectStorageClass(docs[i], cfg.StorageClass) + if err != nil { + return fmt.Errorf("inject storageClassName=%q into %s: %w", cfg.StorageClass, d.name, err) + } + docs[i] = m + } + } + compDir := filepath.Join(cfg.OutputDir, "components", d.name) + if err := os.MkdirAll(compDir, 0o755); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(compDir, d.name+".yaml"), bytes.Join(docs, []byte("---\n")), 0o644); err != nil { + return err + } + comp := "apiVersion: kustomize.config.k8s.io/v1alpha1\n" + + "kind: Component\n" + + "resources:\n" + + " - " + d.name + ".yaml\n" + if err := os.WriteFile(filepath.Join(compDir, "kustomization.yaml"), []byte(comp), 0o644); err != nil { + return err + } + } + return nil +} diff --git a/cmd/oap/internal/installcmd/render_golden_test.go b/cmd/oap/internal/installcmd/render_golden_test.go new file mode 100644 index 00000000..094d77ad --- /dev/null +++ b/cmd/oap/internal/installcmd/render_golden_test.go @@ -0,0 +1,107 @@ +package installcmd + +import ( + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/authzed/openagentprimitives/pkg/platform/cloud" +) + +// goldenOverlayFiles returns the SMALL, config-driven overlay files (the ones +// render actually composes) keyed by relative path. It deliberately excludes +// base/*.yaml and components//.yaml — those are the vendored bundles +// (1.5 MB), drift-tested by TestInstallYAMLMatchesKustomize, and would bloat the +// repo if goldened here. The byte-identical re-render property of the base is +// covered by TestWriteOverlay_DeterministicAndBuildable. +func goldenOverlayFiles(t *testing.T, dir string) map[string]string { + t.Helper() + out := map[string]string{} + require.NoError(t, filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + keep := rel == "kustomization.yaml" || + rel == "routing.yaml" || + rel == "bootstrap-job.yaml" || + rel == "webhook-cert.yaml" || + strings.HasPrefix(rel, "patches/") || + (strings.HasPrefix(rel, "components/") && filepath.Base(rel) == "kustomization.yaml") + if !keep { + return nil + } + b, err := os.ReadFile(p) + if err != nil { + return err + } + out[rel] = string(b) + return nil + })) + return out +} + +// TestRenderGoldens freezes the composed overlay files for representative +// configs. Regenerate with RENDER_UPDATE_GOLDEN=1 and READ THE DIFF — an +// unexplained change is the regression this guards. +func TestRenderGoldens(t *testing.T) { + cases := []struct { + name string + cfg RenderConfig + }{ + {"local", RenderConfig{ClusterKind: cloud.KeyLocal, Strat: cloud.MustFor(cloud.KeyLocal)}}, + {"gke-postgres-routing", func() RenderConfig { + c := RenderConfig{ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE), WithPostgres: true} + c.Routing.trustedHostname = "webd.example.com" + c.Routing.sandboxHostname = "sandbox.example.com" + c.Routing.gatewayClass = "eg" + return c + }()}, + } + update := os.Getenv("RENDER_UPDATE_GOLDEN") != "" + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + tc.cfg.OutputDir = dir + require.NoError(t, WriteOverlay(tc.cfg)) + got := goldenOverlayFiles(t, dir) + + goldenDir := filepath.Join("testdata", "render", tc.name) + if update { + require.NoError(t, os.RemoveAll(goldenDir)) + for rel, content := range got { + p := filepath.Join(goldenDir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(t, os.WriteFile(p, []byte(content), 0o644)) + } + return + } + + want := goldenOverlayFiles(t, goldenDir) + gotKeys, wantKeys := keysOf(got), keysOf(want) + require.Equal(t, wantKeys, gotKeys, "overlay file set changed (run RENDER_UPDATE_GOLDEN=1 and read the diff)") + for rel, content := range got { + assert.Equal(t, want[rel], content, "%s changed (run RENDER_UPDATE_GOLDEN=1 and read the diff)", rel) + } + }) + } +} + +func keysOf(m map[string]string) []string { + ks := make([]string, 0, len(m)) + for k := range m { + ks = append(ks, k) + } + sort.Strings(ks) + return ks +} diff --git a/cmd/oap/internal/installcmd/render_overlay.go b/cmd/oap/internal/installcmd/render_overlay.go new file mode 100644 index 00000000..5ccb7f5b --- /dev/null +++ b/cmd/oap/internal/installcmd/render_overlay.go @@ -0,0 +1,238 @@ +package installcmd + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + + "sigs.k8s.io/kustomize/api/krusty" + "sigs.k8s.io/kustomize/kyaml/filesys" + + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" + "github.com/authzed/openagentprimitives/pkg/platform/cloud" + "github.com/authzed/openagentprimitives/pkg/platform/manifests" +) + +// processBaseBytes applies the render's image substitution and memory-token +// linking to a raw bundle. Image refs are rewritten by the SAME textual +// substitution oap install uses (reaching container AND non-container refs, +// no-op for a local install); memory-token links rewrite every reference to a +// linked component's default Secret name. Both are byte replacements, so the +// output stays well-formed and diffable. +func processBaseBytes(raw []byte, cfg RenderConfig) ([]byte, error) { + out, err := manifests.Substitute(raw, cfg.Tags) + if err != nil { + return nil, err + } + return applyMemoryTokenLinks(out, cfg.MemoryToken), nil +} + +// processBundle loads a separate embedded bundle (channelsd/nats), injects the +// storage class into any stateful doc (a no-op for the rest), joins, and applies +// the same image + memory-token processing as the base. +func processBundle(load func() ([][]byte, error), cfg RenderConfig, storageClass string) ([]byte, error) { + docs, err := load() + if err != nil { + return nil, err + } + if storageClass != "" { + for i := range docs { + m, err := manifests.InjectStorageClass(docs[i], storageClass) + if err != nil { + return nil, fmt.Errorf("inject storageClassName=%q: %w", storageClass, err) + } + docs[i] = m + } + } + return processBaseBytes(bytes.Join(docs, []byte("\n---\n")), cfg) +} + +// applyMemoryTokenLinks rewrites each linked component's default memory-token +// Secret name to the user-supplied existing name, across the whole base (every +// reference). An unset link leaves the default name (the operator mints it). +// Index-aligned with bootstrap.MemoryTokenSecretNames: [channelsd, webd, authzd]. +func applyMemoryTokenLinks(base []byte, links MemoryTokenLinks) []byte { + repl := func(b []byte, def, linked string) []byte { + if linked == "" { + return b + } + return bytes.ReplaceAll(b, []byte(def), []byte(linked)) + } + base = repl(base, bootstrap.MemoryTokenSecretNames[0], links.Channelsd) + base = repl(base, bootstrap.MemoryTokenSecretNames[1], links.Webd) + base = repl(base, bootstrap.MemoryTokenSecretNames[2], links.Authzd) + return base +} + +// clusterKindPatchFile is the overlay patch that stamps AP_CLUSTER_KIND onto the +// operator and webd Deployments. Kept as a relative path constant so the writer +// and the kustomization emitter cannot disagree. +const clusterKindPatchFile = "patches/cluster-kind.yaml" + +// WriteOverlay renders a self-contained kustomize overlay into cfg.OutputDir. It +// is a pure function of cfg (and the digests already resolved into cfg.Tags): +// no wall-clock, no randomness, deterministic ordering — so re-rendering +// identical inputs is byte-identical and GitOps controllers see no drift. +func WriteOverlay(cfg RenderConfig) error { + baseDir := filepath.Join(cfg.OutputDir, "base") + if err := os.MkdirAll(baseDir, 0o755); err != nil { + return err + } + // The base is the embedded bundle with first-party image refs rewritten by + // the SAME textual substitution oap install uses (manifests.Substitute). This + // reaches every ref — container AND non-container (toolchain overlays, AgentClass + // runner image) — which a kustomize `images:` transformer cannot, and is a no-op + // for a local install (empty Tags). The result is still well-formed YAML (a byte + // replace preserves formatting), so the base stays diffable. + // The core workload set is base/install.yaml (= the embedded `kubectl + // kustomize config/` bundle) PLUS channelsd and nats, which `oap install` + // applies from separate embedded bundles (they are not in config/). All three + // are always-on; without channelsd + nats the install does not function. + // Each is image-substituted and memory-token-linked identically; nats, a + // StatefulSet, also takes the storage class. + install, err := processBaseBytes(manifests.Install, cfg) + if err != nil { + return fmt.Errorf("process base: %w", err) + } + if err := os.WriteFile(filepath.Join(baseDir, "install.yaml"), install, 0o644); err != nil { + return err + } + channelsd, err := processBundle(manifests.ChannelsD, cfg, "") + if err != nil { + return fmt.Errorf("process channelsd: %w", err) + } + if err := os.WriteFile(filepath.Join(baseDir, "channelsd.yaml"), channelsd, 0o644); err != nil { + return err + } + nats, err := processBundle(manifests.NATS, cfg, cfg.StorageClass) + if err != nil { + return fmt.Errorf("process nats: %w", err) + } + if err := os.WriteFile(filepath.Join(baseDir, "nats.yaml"), nats, 0o644); err != nil { + return err + } + + // AP_CLUSTER_KIND is stamped as an overlay patch (not baked into the base) so + // the one value a GitOps reader most wants to see is visible in the overlay, + // not buried in a 1.5 MB base. The operator and webd both fail closed when it + // is unset, so it is emitted on every render. + patchDir := filepath.Join(cfg.OutputDir, "patches") + if err := os.MkdirAll(patchDir, 0o755); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(cfg.OutputDir, clusterKindPatchFile), clusterKindPatch(cfg.Strat.Key()), 0o644); err != nil { + return err + } + + // Routing is opt-in: emit the Gateway + HTTPRoute(s) only when a trusted + // hostname is configured. renderKustomization includes routing.yaml in + // resources under the same condition, so the two cannot disagree. + routing, emitted, err := renderRoutingResources(cfg.Routing) + if err != nil { + return err + } + if emitted { + if err := os.WriteFile(filepath.Join(cfg.OutputDir, routingResourceFile), routing, 0o644); err != nil { + return err + } + } + + // Pre-start Job that mints the internal Secrets (always emitted — a GitOps + // apply has no `oap install` to mint them). + job, err := renderBootstrapJob(cfg) + if err != nil { + return fmt.Errorf("render bootstrap job: %w", err) + } + if err := os.WriteFile(filepath.Join(cfg.OutputDir, bootstrapJobFile), job, 0o644); err != nil { + return err + } + + // Webhook serving cert via cert-manager (unless the user provides it). The + // Issuer+Certificate populate spicebox-webhook-tls; the VWC patch below adds + // the ca-injector annotation. + if !cfg.ExternalWebhookTLS { + if err := os.WriteFile(filepath.Join(cfg.OutputDir, webhookCertResourceFile), renderWebhookCertResources(), 0o644); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(cfg.OutputDir, webhookCAInjectPatchFile), webhookCAInjectPatch(), 0o644); err != nil { + return err + } + } + + // Optional dependencies (postgres/graphiti/neo4j) become kustomize Components, + // pulled in via `components:` only when their --with-* flag is set. + if err := writeDependencyComponents(cfg); err != nil { + return err + } + + k, err := renderKustomization(cfg) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(cfg.OutputDir, "kustomization.yaml"), k, 0o644) +} + +// clusterKindPatch is a two-document strategic-merge patch setting the +// AP_CLUSTER_KIND env on the operator and webd app containers (both single- +// container Deployments; the container names `operator`/`webd` are the strategic- +// merge keys). A pure function of key. +func clusterKindPatch(key string) []byte { + one := func(deployment, container string) string { + return "apiVersion: apps/v1\n" + + "kind: Deployment\n" + + "metadata:\n" + + " name: " + deployment + "\n" + + " namespace: " + cloud.WebdServiceNamespace + "\n" + + "spec:\n" + + " template:\n" + + " spec:\n" + + " containers:\n" + + " - name: " + container + "\n" + + " env:\n" + + " - name: " + cloud.ClusterKindEnvVar + "\n" + + " value: " + key + "\n" + } + return []byte(one(cloud.OperatorDeploymentName, "operator") + "---\n" + one(webdDeploymentName, "webd")) +} + +// renderKustomization builds the overlay kustomization.yaml bytes +// deterministically. Components and additional patches are appended by later +// render stages; order is fixed so re-renders are byte-identical. +func renderKustomization(cfg RenderConfig) ([]byte, error) { + resources := "resources:\n - base/install.yaml\n - base/channelsd.yaml\n - base/nats.yaml\n - " + bootstrapJobFile + "\n" + if cfg.Routing.trustedHostname != "" { + resources += " - " + routingResourceFile + "\n" + } + if !cfg.ExternalWebhookTLS { + resources += " - " + webhookCertResourceFile + "\n" + } + patches := "patches:\n - path: " + clusterKindPatchFile + "\n" + if !cfg.ExternalWebhookTLS { + patches += " - path: " + webhookCAInjectPatchFile + "\n" + } + k := "apiVersion: kustomize.config.k8s.io/v1beta1\n" + + "kind: Kustomization\n" + + resources + + patches + if names := enabledComponentNames(cfg); len(names) > 0 { + k += "components:\n" + for _, n := range names { + k += " - components/" + n + "\n" + } + } + return []byte(k), nil +} + +// buildOverlay runs kustomize build in-process (krusty, no binary dependency) +// over dir and returns the rendered YAML. Used both as a render-time self-check +// and by `oap render --validate`. +func buildOverlay(dir string) ([]byte, error) { + fsys := filesys.MakeFsOnDisk() + k := krusty.MakeKustomizer(krusty.MakeDefaultOptions()) + res, err := k.Run(fsys, dir) + if err != nil { + return nil, fmt.Errorf("kustomize build %s: %w", dir, err) + } + return res.AsYaml() +} diff --git a/cmd/oap/internal/installcmd/render_routing.go b/cmd/oap/internal/installcmd/render_routing.go new file mode 100644 index 00000000..803c6921 --- /dev/null +++ b/cmd/oap/internal/installcmd/render_routing.go @@ -0,0 +1,150 @@ +package installcmd + +import ( + "bytes" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/yaml" + + "github.com/authzed/openagentprimitives/pkg/platform/cloud" +) + +// routingResourceFile is the overlay file holding the emitted Gateway + HTTPRoutes. +const routingResourceFile = "routing.yaml" + +// renderRoutingResources builds webd's external-access Gateway + HTTPRoute(s) as +// marshaled YAML, but ONLY when a trusted hostname is configured — routing is +// opt-in, so a render with no --trusted-hostname emits nothing and the user +// brings their own ingress. +// +// cert-manager objects (ClusterIssuer / Certificate) are deliberately NOT +// emitted: the HTTPS listeners reference the standard cert Secret names, and the +// platform provides the issuer and certificates (the GitOps norm — ingress and +// cert-manager are cluster-wide concerns, not per-app bundle state). +func renderRoutingResources(r WebdRoutingOpts) (out []byte, emitted bool, err error) { + if r.trustedHostname == "" { + return nil, false, nil + } + docs := make([][]byte, 0, 3) + + gw, err := marshalClean(buildWebdGateway(r)) + if err != nil { + return nil, false, err + } + docs = append(docs, gw) + + tr, err := marshalClean(buildWebdHTTPRoute(webdTrustedRouteName, r.trustedHostname)) + if err != nil { + return nil, false, err + } + docs = append(docs, tr) + + if r.sandboxHostname != "" { + sb, err := marshalClean(buildWebdHTTPRoute(webdSandboxRouteName, r.sandboxHostname)) + if err != nil { + return nil, false, err + } + docs = append(docs, sb) + } + + return bytes.Join(docs, []byte("---\n")), true, nil +} + +// buildWebdGateway mirrors cloud.ApplyWebdGateway's listener shape (trusted-https +// 443, optional sandbox-https 443, acme-http 80) but with cert-manager +// Secret-backed TLS (Terminate + certificateRefs), the portable shape. TypeMeta +// is set explicitly so kustomize can parse the marshaled object. +func buildWebdGateway(r WebdRoutingOpts) *gatewayv1.Gateway { + terminate := gatewayv1.TLSModeTerminate + httpsListener := func(name, host, certSecret string) gatewayv1.Listener { + h := gatewayv1.Hostname(host) + return gatewayv1.Listener{ + Name: gatewayv1.SectionName(name), + Hostname: &h, + Port: 443, + Protocol: gatewayv1.HTTPSProtocolType, + TLS: &gatewayv1.GatewayTLSConfig{ + Mode: &terminate, + CertificateRefs: []gatewayv1.SecretObjectReference{{Name: gatewayv1.ObjectName(certSecret)}}, + }, + } + } + listeners := []gatewayv1.Listener{httpsListener("trusted-https", r.trustedHostname, cloud.WebdTrustedCertSecretName)} + if r.sandboxHostname != "" { + listeners = append(listeners, httpsListener("sandbox-https", r.sandboxHostname, cloud.WebdSandboxCertSecretName)) + } + listeners = append(listeners, gatewayv1.Listener{Name: "acme-http", Port: 80, Protocol: gatewayv1.HTTPProtocolType}) + return &gatewayv1.Gateway{ + TypeMeta: metav1.TypeMeta{APIVersion: gatewayv1.GroupVersion.String(), Kind: "Gateway"}, + ObjectMeta: metav1.ObjectMeta{Name: cloud.WebdGatewayName, Namespace: cloud.WebdServiceNamespace}, + Spec: gatewayv1.GatewaySpec{ + GatewayClassName: gatewayv1.ObjectName(r.gatewayClass), + Listeners: listeners, + }, + } +} + +// marshalClean marshals a typed object to YAML with the noise a GitOps reader +// does not want: the empty `status` stanza (controller-owned, ignored on apply) +// and null-valued optional fields (e.g. a SecretObjectReference's defaulted +// group/kind). It round-trips through a map and prunes, so the committed +// resource reads cleanly. +func marshalClean(obj any) ([]byte, error) { + raw, err := yaml.Marshal(obj) + if err != nil { + return nil, err + } + var m map[string]any + if err := yaml.Unmarshal(raw, &m); err != nil { + return nil, err + } + delete(m, "status") + pruneNulls(m) + return yaml.Marshal(m) +} + +// pruneNulls recursively deletes nil map values (an unset optional field marshals +// to null; dropping it is equivalent and cleaner). +func pruneNulls(v any) { + switch t := v.(type) { + case map[string]any: + for k, val := range t { + if val == nil { + delete(t, k) + continue + } + pruneNulls(val) + } + case []any: + for _, e := range t { + pruneNulls(e) + } + } +} + +// buildWebdHTTPRoute mirrors applyWebdHTTPRoute's spec (parentRef → the Gateway, +// one hostname, backendRef → webd:8080). TypeMeta set for kustomize parsing. +func buildWebdHTTPRoute(name, host string) *gatewayv1.HTTPRoute { + h := gatewayv1.Hostname(host) + return &gatewayv1.HTTPRoute{ + TypeMeta: metav1.TypeMeta{APIVersion: gatewayv1.GroupVersion.String(), Kind: "HTTPRoute"}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: cloud.WebdServiceNamespace}, + Spec: gatewayv1.HTTPRouteSpec{ + CommonRouteSpec: gatewayv1.CommonRouteSpec{ + ParentRefs: []gatewayv1.ParentReference{{Name: gatewayv1.ObjectName(cloud.WebdGatewayName)}}, + }, + Hostnames: []gatewayv1.Hostname{h}, + Rules: []gatewayv1.HTTPRouteRule{{ + BackendRefs: []gatewayv1.HTTPBackendRef{{ + BackendRef: gatewayv1.BackendRef{ + BackendObjectReference: gatewayv1.BackendObjectReference{ + Name: gatewayv1.ObjectName(cloud.WebdServiceName), + Port: ptr(gatewayv1.PortNumber(cloud.WebdServicePort)), + }, + }, + }}, + }}, + }, + } +} diff --git a/cmd/oap/internal/installcmd/render_test.go b/cmd/oap/internal/installcmd/render_test.go new file mode 100644 index 00000000..b495066a --- /dev/null +++ b/cmd/oap/internal/installcmd/render_test.go @@ -0,0 +1,337 @@ +package installcmd + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/authzed/openagentprimitives/cmd/oap/internal/apcmd" + "github.com/authzed/openagentprimitives/cmd/oap/internal/kube" + "github.com/authzed/openagentprimitives/pkg/platform/apimage" + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" + "github.com/authzed/openagentprimitives/pkg/platform/cloud" + "github.com/authzed/openagentprimitives/pkg/platform/manifests" +) + +func TestWriteOverlay_DeterministicAndBuildable(t *testing.T) { + dir1, dir2 := t.TempDir(), t.TempDir() + base := RenderConfig{ClusterKind: cloud.KeyLocal, Strat: cloud.MustFor(cloud.KeyLocal)} + + cfg := base + cfg.OutputDir = dir1 + require.NoError(t, WriteOverlay(cfg)) + cfg.OutputDir = dir2 + require.NoError(t, WriteOverlay(cfg)) + + assert.FileExists(t, filepath.Join(dir1, "base", "install.yaml")) + assert.FileExists(t, filepath.Join(dir1, "kustomization.yaml")) + + for _, rel := range []string{"kustomization.yaml", filepath.Join("base", "install.yaml")} { + b1, err := os.ReadFile(filepath.Join(dir1, rel)) + require.NoError(t, err) + b2, err := os.ReadFile(filepath.Join(dir2, rel)) + require.NoError(t, err) + assert.Equal(t, string(b1), string(b2), "re-render of %s is byte-identical", rel) + } + + out, err := buildOverlay(dir1) + require.NoError(t, err, "emitted overlay must kustomize build") + assert.Contains(t, string(out), "kind: Deployment") +} + +func TestRunRender_OfflineSkipsDigestResolution(t *testing.T) { + g := &apcmd.Globals{BundleFn: func() (*kube.Bundle, error) { + panic("render must not touch the cluster offline") + }} + // A GitOps-valid (persistent-datastore) kind with --no-digest-pin: render must + // succeed without any remote call. gke, not local — local is a memory-datastore + // kind and validateRenderFlags refuses it (see TestValidateRenderFlags_*). + cfg := RenderConfig{OutputDir: t.TempDir(), ClusterKind: cloud.KeyGKE, NoDigestPin: true} + require.NoError(t, runRender(context.Background(), io.Discard, g, cfg, false)) +} + +func TestRender_StampsClusterKindAndImages(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{ + OutputDir: dir, + ClusterKind: cloud.KeyGKE, + Strat: cloud.MustFor(cloud.KeyGKE), + Tags: manifests.Tags{ + Registry: "reg.example.com", + Digests: map[string]string{ + "spicebox-operator": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "agentprimitives-runner": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + }, + }, + } + require.NoError(t, WriteOverlay(cfg)) + out, err := buildOverlay(dir) + require.NoError(t, err) + s := string(out) + assert.GreaterOrEqual(t, strings.Count(s, "AP_CLUSTER_KIND"), 2, "AP_CLUSTER_KIND stamped on operator + webd") + assert.Contains(t, s, cloud.KeyGKE, "cluster-kind value present") + assert.Contains(t, s, "reg.example.com/spicebox-operator", "operator image rewritten to the registry") + assert.Contains(t, s, "sha256:1111111111111111111111111111111111111111111111111111111111111111", "operator image pinned by digest") +} + +func TestRender_EmitsRoutingWhenHostnameGiven(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE)} + cfg.Routing.trustedHostname = "webd.example.com" + cfg.Routing.sandboxHostname = "sandbox.example.com" + cfg.Routing.gatewayClass = "eg" + require.NoError(t, WriteOverlay(cfg)) + + out, err := buildOverlay(dir) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, "kind: Gateway") + assert.Contains(t, s, "kind: HTTPRoute") + assert.Contains(t, s, "webd.example.com") + assert.Contains(t, s, "sandbox.example.com") + assert.Contains(t, s, cloud.WebdTrustedCertSecretName, "trusted listener references the cert Secret") + assert.Contains(t, s, cloud.WebdSandboxCertSecretName, "sandbox listener references the cert Secret") +} + +func TestRender_NoRoutingWithoutHostname(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE)} + require.NoError(t, WriteOverlay(cfg)) + + out, err := buildOverlay(dir) + require.NoError(t, err) + assert.NotContains(t, string(out), "kind: HTTPRoute", "no --trusted-hostname → bring-your-own ingress, no routing emitted") +} + +func TestValidateRenderFlags_RoutingRequiresGatewayClass(t *testing.T) { + cfg := RenderConfig{ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE)} + cfg.Routing.trustedHostname = "webd.example.com" + err := validateRenderFlags(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "gateway-class") +} + +func TestRender_WithPostgresComponentInjectsStorageClass(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{ + OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE), + WithPostgres: true, StorageClass: "fast-ssd", + } + require.NoError(t, WriteOverlay(cfg)) + assert.FileExists(t, filepath.Join(dir, "components", "postgres", "kustomization.yaml")) + + out, err := buildOverlay(dir) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, "kind: PersistentVolumeClaim", "postgres PVC pulled in by the component") + assert.Contains(t, s, "storageClassName: fast-ssd", "--storage-class injected into the postgres PVC") +} + +func TestRender_IncludesChannelsdAndNats(t *testing.T) { + // channelsd + nats are applied by `oap install` from separate embedded + // bundles (not config/), so the render must add them explicitly or the + // bundle deploys an install that cannot function. + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE)} + require.NoError(t, WriteOverlay(cfg)) + assert.FileExists(t, filepath.Join(dir, "base", "channelsd.yaml")) + assert.FileExists(t, filepath.Join(dir, "base", "nats.yaml")) + + out, err := buildOverlay(dir) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, "spicebox-channelsd", "channelsd workload present") + assert.Contains(t, s, "spicebox-nats", "nats workload present") +} + +func TestRender_EmitsBootstrapJob(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE)} + require.NoError(t, WriteOverlay(cfg)) + assert.FileExists(t, filepath.Join(dir, "bootstrap-job.yaml")) + + out, err := buildOverlay(dir) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, "kind: Job") + assert.Contains(t, s, "spicebox-bootstrap-secrets") + assert.Contains(t, s, "bootstrap-secrets", "runs the operator bootstrap-secrets entrypoint") + assert.Contains(t, s, "serviceAccountName: spicebox-operator", "reuses the operator SA's RBAC") + assert.NotContains(t, s, "--skip-spicedb-token", "mints the bundled spicedb token by default") +} + +func TestRender_BootstrapJobSkipsExternalTokens(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE), ExternalSpiceDB: true, ExternalPostgres: true} + require.NoError(t, WriteOverlay(cfg)) + + out, err := buildOverlay(dir) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, "--skip-spicedb-token", "external SpiceDB: Job does not mint the token") + assert.Contains(t, s, "--skip-postgres-token", "external Postgres: Job does not mint the token") +} + +func TestRenderValidate_ExternalSpiceDBStillValidates(t *testing.T) { + // With --external-spicedb the Job does not mint spicedb-token, but the token's + // k8s refs are optional (the app requires the value at runtime, not the kubelet + // at mount), so --validate must still pass — the externalized instance does not + // make the overlay structurally incomplete. + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE), NoDigestPin: true, ExternalSpiceDB: true} + require.NoError(t, WriteOverlay(cfg)) + + var buf bytes.Buffer + require.NoError(t, runRenderValidate(context.Background(), &buf, cfg, dir), "external spicedb does not break validation") +} + +func TestRender_EmitsWebhookCertManagerByDefault(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE)} + require.NoError(t, WriteOverlay(cfg)) + assert.FileExists(t, filepath.Join(dir, "webhook-cert.yaml")) + + out, err := buildOverlay(dir) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, "kind: Certificate") + assert.Contains(t, s, "kind: Issuer") + assert.Contains(t, s, "spicebox-webhook.agentprimitives-system.svc", "cert SANs cover the webhook Service") + assert.Contains(t, s, "cert-manager.io/inject-ca-from: agentprimitives-system/spicebox-webhook-tls", "VWC gets the CA-injector annotation") +} + +func TestRender_ExternalWebhookTLSSkipsCertManager(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE), ExternalWebhookTLS: true} + require.NoError(t, WriteOverlay(cfg)) + assert.NoFileExists(t, filepath.Join(dir, "webhook-cert.yaml")) + + out, err := buildOverlay(dir) + require.NoError(t, err) + assert.NotContains(t, string(out), "cert-manager.io/inject-ca-from", "no CA-injector annotation when the user provides webhook-tls") +} + +func TestRender_NoComponentsByDefault(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE)} + require.NoError(t, WriteOverlay(cfg)) + assert.NoDirExists(t, filepath.Join(dir, "components"), "no --with-* flag → no components dir") +} + +func TestRender_LinksExistingMemoryTokenSecret(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE)} + cfg.MemoryToken.Channelsd = "my-channelsd-secret" + require.NoError(t, WriteOverlay(cfg)) + + out, err := buildOverlay(dir) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, "my-channelsd-secret", "channelsd memory-token references rewritten to the linked Secret") + assert.NotContains(t, s, "spicebox-channelsd-memory-token", "default channelsd token name fully replaced (operator mount + channelsd volume)") + assert.Contains(t, s, "spicebox-webd-memory-token", "unlinked webd token still uses the operator-minted default") +} + +func TestValidateRenderFlags_RefusesWebdLinkEqualToOthers(t *testing.T) { + // webd is browser-facing and its memory token is READ-ONLY by value; linking + // it to the same Secret as channelsd/authzd (whose tokens are write-capable) + // would hand webd write access to the audit log. Refuse it. + base := RenderConfig{ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE)} + + c := base + c.MemoryToken = MemoryTokenLinks{Channelsd: "shared", Webd: "shared"} + require.Error(t, validateRenderFlags(c), "webd == channelsd link must be refused") + + c = base + c.MemoryToken = MemoryTokenLinks{Authzd: "shared", Webd: "shared"} + require.Error(t, validateRenderFlags(c), "webd == authzd link must be refused") + + // The bypass: linking webd to a DEFAULT-minted name while the matching + // component is left to operator-mint (its link == ""). The Job mints the + // default names regardless of links, so webd would mount the write-capable + // channelsd/authzd token. The guard must compare EFFECTIVE names (link if + // set, else the default), not just the raw link values. + c = base + c.MemoryToken = MemoryTokenLinks{Webd: bootstrap.MemoryTokenSecretNames[0]} // channelsd default, channelsd unlinked + require.Error(t, validateRenderFlags(c), "webd linked to channelsd's default-minted token must be refused") + + c = base + c.MemoryToken = MemoryTokenLinks{Webd: bootstrap.MemoryTokenSecretNames[2]} // authzd default, authzd unlinked + require.Error(t, validateRenderFlags(c), "webd linked to authzd's default-minted token must be refused") + + // Distinct links are fine. + c = base + c.MemoryToken = MemoryTokenLinks{Channelsd: "ch", Webd: "wd", Authzd: "az"} + require.NoError(t, validateRenderFlags(c)) + + // The all-default (no-link) case must stay valid: the operator mints three + // distinct tokens, so webd never collides. + c = base + c.MemoryToken = MemoryTokenLinks{} + require.NoError(t, validateRenderFlags(c), "all-default links must remain valid") +} + +func TestRenderCmd_NoGraphitiOrNeo4jFlags(t *testing.T) { + // graphiti/neo4j require a user-supplied LLM API key (spicebox-graphiti-config) + // that the overlay cannot mint — fundamentally incompatible with render's + // "every internal Secret is provisioned, nothing external by default" + // property. Rather than ship flags that emit a non-functional bundle (and a + // --validate that hard-fails with no remedy), they are omitted until a + // follow-up adds the external-secret story. Keep --with-postgres (its token IS + // Job-minted, so it works end to end). + flags := NewRenderCmd(&apcmd.Globals{}).Flags() + assert.NotNil(t, flags.Lookup("with-postgres"), "postgres is Job-provisionable and stays") + assert.Nil(t, flags.Lookup("with-graphiti"), "with-graphiti must not be offered (non-functional bundle)") + assert.Nil(t, flags.Lookup("with-neo4j"), "with-neo4j must not be offered (non-functional bundle)") +} + +func TestBootstrapJobName_TracksImageAndArgs(t *testing.T) { + // The Job's pod template is immutable once applied. A GitOps re-sync after an + // operator-image bump (new digest) would otherwise try to patch spec.template + // and be rejected (SyncFailed). Naming the Job by a hash of its + // image+args means a changed template is a NEW object (prune-and-create), + // while a byte-identical re-render keeps the same name (no-op apply). + // A real remote GitOps render pins by registry+digest, which is exactly the + // ref that changes on an operator upgrade. + base := RenderConfig{ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE)} + base.Tags.Registry = "reg.example.com" + base.Tags.Digests = map[string]string{apimage.Operator.Name: "sha256:" + strings.Repeat("a", 64)} + + n1, err := bootstrapJobName(base) + require.NoError(t, err) + n1b, err := bootstrapJobName(base) + require.NoError(t, err) + assert.Equal(t, n1, n1b, "deterministic: identical config → identical Job name") + assert.True(t, strings.HasPrefix(n1, "spicebox-bootstrap-secrets-"), "keeps the recognizable prefix, got %q", n1) + + bumped := base + bumped.Tags.Digests = map[string]string{apimage.Operator.Name: "sha256:" + strings.Repeat("b", 64)} + nb, err := bootstrapJobName(bumped) + require.NoError(t, err) + assert.NotEqual(t, n1, nb, "a bumped operator image digest must yield a new Job object") + + skipped := base + skipped.ExternalSpiceDB = true + ns, err := bootstrapJobName(skipped) + require.NoError(t, err) + assert.NotEqual(t, n1, ns, "changed args (--skip-spicedb-token) must yield a new Job object") +} + +func TestValidateRenderFlags_RefusesMemoryDatastoreKind(t *testing.T) { + // local/desktop use an in-memory SpiceDB datastore whose schema is bootstrapped + // at startup (bootstrapsSchemaAtStartup) — fatal against a durable, multi-replica + // GitOps target. Render must refuse it rather than emit an unsafe bundle. + err := validateRenderFlags(RenderConfig{ClusterKind: cloud.KeyLocal, Strat: cloud.MustFor(cloud.KeyLocal)}) + require.Error(t, err) + assert.Contains(t, err.Error(), "memory", "error explains the memory datastore is the problem") + + // A persistent-datastore kind is accepted. + require.NoError(t, validateRenderFlags(RenderConfig{ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE)})) +} diff --git a/cmd/oap/internal/installcmd/render_validate.go b/cmd/oap/internal/installcmd/render_validate.go new file mode 100644 index 00000000..519b8bd5 --- /dev/null +++ b/cmd/oap/internal/installcmd/render_validate.go @@ -0,0 +1,266 @@ +package installcmd + +import ( + "context" + "fmt" + "io" + "sort" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/authzed/openagentprimitives/cmd/oap/internal/cliout" + "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/platform/apimage" + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" + "github.com/authzed/openagentprimitives/pkg/platform/cloud" + "github.com/authzed/openagentprimitives/pkg/platform/manifests" +) + +// runRenderValidate lints an already-rendered overlay at dir. It is the runtime +// twin of pkg/platform/manifests/secret_bootstrap_test.go, run against the built +// output rather than the embedded bundle, plus the GitOps-specific checks a +// synced cluster depends on. cfg carries the render's own decisions (linked +// token Secrets, digest-pinning) so the lint does not false-positive on them. +// +// What it CANNOT check (no cluster): whether a LINKED Secret actually exists — +// that is a cluster-state fact `oap check` verifies post-sync. The lint confirms +// the overlay is structurally complete and will sync, not that the external +// dependencies it references are present. +func runRenderValidate(ctx context.Context, out io.Writer, cfg RenderConfig, dir string) error { + built, err := buildOverlay(dir) + if err != nil { + return err // a non-buildable overlay is the first thing that would fail at sync + } + docs, err := manifests.Split(built) + if err != nil { + return fmt.Errorf("parse built overlay: %w", err) + } + + var problems []string + + // 1. Every non-optional Secret reference has a creator: shipped in the bundle, + // minted by the bootstrap Job / operator, or linked by this render. A + // reference to an app-internal secret nothing here yet provisions (the NATS + // stack, webhook-tls, or a token the user externalized) is not a hard failure + // — it is listed as "provide this". + satisfied := shippedSecretNames(docs) + for _, n := range jobMintedSecretNames(cfg) { + satisfied[n] = true + } + // webhook-tls is populated by the emitted cert-manager Certificate (unless + // the user provides it themselves). + if !cfg.ExternalWebhookTLS { + satisfied["spicebox-webhook-tls"] = true + } + for _, n := range []string{cfg.MemoryToken.Channelsd, cfg.MemoryToken.Webd, cfg.MemoryToken.Authzd} { + if n != "" { + satisfied[n] = true + } + } + external := externalSecretNames(cfg) + provideExternally := map[string]bool{} + for _, ref := range nonOptionalSecretRefs(docs) { + switch { + case satisfied[ref.secret]: + // satisfied + case external[ref.secret]: + provideExternally[ref.secret] = true + default: + problems = append(problems, fmt.Sprintf("%s/%s references Secret %q with no creator (not shipped, not minted by the bootstrap Job, not linked)", ref.workloadKind, ref.workload, ref.secret)) + } + } + + // 2. AP_CLUSTER_KIND is stamped on the operator and webd (both fail closed + // without it). + for _, dep := range []string{cloud.OperatorDeploymentName, webdDeploymentName} { + if !deploymentHasEnv(docs, dep, cloud.ClusterKindEnvVar) { + problems = append(problems, fmt.Sprintf("Deployment %s is missing the %s env var", dep, cloud.ClusterKindEnvVar)) + } + } + + // 3. First-party images are pinned (no mutable :dev), unless --no-digest-pin. + if !cfg.NoDigestPin { + for _, img := range firstPartyMutableImages(built) { + problems = append(problems, fmt.Sprintf("image %q uses a mutable tag; GitOps wants an immutable digest (render with a registry, or pass --no-digest-pin to accept this)", img)) + } + } + + if len(problems) > 0 { + return fmt.Errorf("overlay validation found %d problem(s):\n - %s", len(problems), strings.Join(problems, "\n - ")) + } + cliout.Step(out, "overlay validated: %d resources, all Secret references satisfied, cluster-kind stamped", len(docs)) + if len(provideExternally) > 0 { + names := make([]string, 0, len(provideExternally)) + for n := range provideExternally { + names = append(names, n) + } + sort.Strings(names) + cliout.Warn(out, "provide these app-internal Secret(s) yourself until a later release has the operator mint them (e.g. via External Secrets Operator or a sealed secret):\n - %s", strings.Join(names, "\n - ")) + } + return nil +} + +// jobMintedSecretNames is the set of internal Secrets the pre-start bootstrap Job +// mints for this config — so --validate treats their references as satisfied. +// The memory tokens, admind token and passthrough key are always minted; the +// SpiceDB/Postgres tokens are minted unless the user externalized that instance. +func jobMintedSecretNames(cfg RenderConfig) []string { + names := append([]string{}, bootstrap.MemoryTokenSecretNames...) + names = append(names, bootstrap.AdmindTokenSecretName, v1alpha1.PassthroughLinkSigningKeySecret, + "spicebox-nats-identity", "spicebox-nats-tls", "spicebox-channelsd-nats-creds", "spicebox-webd-nats-creds", + "spicebox-postgres-token", "spicebox-spicedb-token") + out := names[:0:len(names)] + for _, n := range names { + if n == "spicebox-spicedb-token" && cfg.ExternalSpiceDB { + continue + } + if n == "spicebox-postgres-token" && cfg.ExternalPostgres { + continue + } + out = append(out, n) + } + return out +} + +// externalSecretNames are the Secrets the USER chose to provide themselves via +// the --external-* flags (an external SpiceDB/Postgres instance, or webhook-tls +// on a cluster without cert-manager). --validate lists these as "provide these" +// rather than failing. By default the overlay provisions every internal Secret +// (the bootstrap Job + the cert-manager webhook Certificate), so this set is +// empty and --validate reports nothing to provide. +func externalSecretNames(cfg RenderConfig) map[string]bool { + out := map[string]bool{} + if cfg.ExternalWebhookTLS { + out["spicebox-webhook-tls"] = true + } + if cfg.ExternalSpiceDB { + out["spicebox-spicedb-token"] = true + } + if cfg.ExternalPostgres { + out["spicebox-postgres-token"] = true + } + return out +} + +// shippedSecretNames returns the set of Secret object names present in docs. +func shippedSecretNames(docs []*unstructured.Unstructured) map[string]bool { + out := map[string]bool{} + for _, d := range docs { + if d.GetKind() == "Secret" { + out[d.GetName()] = true + } + } + return out +} + +type secretRef struct { + workloadKind string + workload string + secret string +} + +// nonOptionalSecretRefs collects every required (optional!=true) Secret reference +// from pod templates in docs — env valueFrom.secretKeyRef and volume secrets. +// A missing optional field defaults to required, matching the kubelet. +func nonOptionalSecretRefs(docs []*unstructured.Unstructured) []secretRef { + var refs []secretRef + for _, d := range docs { + spec, found, _ := unstructured.NestedMap(d.Object, "spec", "template", "spec") + if !found { + continue + } + for _, cf := range []string{"containers", "initContainers"} { + containers, _, _ := unstructured.NestedSlice(spec, cf) + for _, c := range containers { + cm, ok := c.(map[string]any) + if !ok { + continue + } + env, _, _ := unstructured.NestedSlice(cm, "env") + for _, e := range env { + em, ok := e.(map[string]any) + if !ok { + continue + } + skr, found, _ := unstructured.NestedMap(em, "valueFrom", "secretKeyRef") + if !found { + continue + } + if optionalTrue(skr) { + continue + } + if name, ok, _ := unstructured.NestedString(skr, "name"); ok { + refs = append(refs, secretRef{d.GetKind(), d.GetName(), name}) + } + } + } + } + volumes, _, _ := unstructured.NestedSlice(spec, "volumes") + for _, v := range volumes { + vm, ok := v.(map[string]any) + if !ok { + continue + } + sec, found, _ := unstructured.NestedMap(vm, "secret") + if !found { + continue + } + if optionalTrue(sec) { + continue + } + if name, ok, _ := unstructured.NestedString(sec, "secretName"); ok { + refs = append(refs, secretRef{d.GetKind(), d.GetName(), name}) + } + } + } + return refs +} + +func optionalTrue(m map[string]any) bool { + b, ok, _ := unstructured.NestedBool(m, "optional") + return ok && b +} + +// deploymentHasEnv reports whether the named Deployment has an env var of the +// given name on any of its containers. +func deploymentHasEnv(docs []*unstructured.Unstructured, name, envName string) bool { + for _, d := range docs { + if d.GetKind() != "Deployment" || d.GetName() != name { + continue + } + containers, _, _ := unstructured.NestedSlice(d.Object, "spec", "template", "spec", "containers") + for _, c := range containers { + cm, ok := c.(map[string]any) + if !ok { + continue + } + env, _, _ := unstructured.NestedSlice(cm, "env") + for _, e := range env { + if em, ok := e.(map[string]any); ok { + if n, ok, _ := unstructured.NestedString(em, "name"); ok && n == envName { + return true + } + } + } + } + } + return false +} + +// firstPartyMutableImages returns first-party image refs in the built YAML that +// are NOT digest-pinned (a mutable ":tag" with no @sha256). Scans textually +// since image refs appear in container specs and a few CR fields alike. +func firstPartyMutableImages(built []byte) []string { + var mutable []string + text := string(built) + for _, img := range apimage.All { + // A pinned ref contains "...@sha256:"; a mutable one is ":dev" + // (or any tag) with no digest. Flag the bare ":dev" refs that survive. + needle := img.Name + ":dev" + if strings.Contains(text, needle) && !strings.Contains(text, img.Name+":dev@sha256:") { + mutable = append(mutable, needle) + } + } + return mutable +} diff --git a/cmd/oap/internal/installcmd/render_validate_test.go b/cmd/oap/internal/installcmd/render_validate_test.go new file mode 100644 index 00000000..9309bbe1 --- /dev/null +++ b/cmd/oap/internal/installcmd/render_validate_test.go @@ -0,0 +1,56 @@ +package installcmd + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/authzed/openagentprimitives/pkg/platform/cloud" +) + +func TestRenderValidate_PassesWithNothingExternalByDefault(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE), NoDigestPin: true} + require.NoError(t, WriteOverlay(cfg)) + + var buf bytes.Buffer + require.NoError(t, runRenderValidate(context.Background(), &buf, cfg, dir), "a complete default render passes validation") + // Everything internal is now provisioned by the overlay (Job + cert-manager), + // so there is nothing to provide yourself. + assert.NotContains(t, buf.String(), "provide these", "no provide-yourself secrets by default") + assert.NotContains(t, buf.String(), "spicebox-webhook-tls", "webhook-tls is cert-manager-provisioned, not external") +} + +func TestRenderValidate_ExternalWebhookTLSListsIt(t *testing.T) { + dir := t.TempDir() + cfg := RenderConfig{OutputDir: dir, ClusterKind: cloud.KeyGKE, Strat: cloud.MustFor(cloud.KeyGKE), NoDigestPin: true, ExternalWebhookTLS: true} + require.NoError(t, WriteOverlay(cfg)) + + var buf bytes.Buffer + require.NoError(t, runRenderValidate(context.Background(), &buf, cfg, dir), "externalized webhook-tls is listed, not a failure") + assert.Contains(t, buf.String(), "spicebox-webhook-tls", "with --external-webhook-tls it is surfaced as provide-yourself") +} + +func TestRenderValidate_FailsOnUnsatisfiedSecret(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "kustomization.yaml"), []byte( + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nresources:\n - dep.yaml\n"), 0o644)) + // A Deployment referencing a Secret nothing creates and that is not a known + // app-internal secret → a hard validation failure. + require.NoError(t, os.WriteFile(filepath.Join(dir, "dep.yaml"), []byte( + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: widget\n namespace: agentprimitives-system\n"+ + "spec:\n selector:\n matchLabels:\n app: widget\n template:\n metadata:\n labels:\n app: widget\n"+ + " spec:\n containers:\n - name: c\n image: busybox\n env:\n - name: T\n"+ + " valueFrom:\n secretKeyRef:\n name: nobody-makes-this\n key: token\n"), 0o644)) + + cfg := RenderConfig{OutputDir: dir, NoDigestPin: true} + err := runRenderValidate(context.Background(), new(bytes.Buffer), cfg, dir) + require.Error(t, err) + assert.Contains(t, err.Error(), "nobody-makes-this") + assert.Contains(t, err.Error(), "no creator") +} diff --git a/cmd/oap/internal/installcmd/render_webhook.go b/cmd/oap/internal/installcmd/render_webhook.go new file mode 100644 index 00000000..e610a12b --- /dev/null +++ b/cmd/oap/internal/installcmd/render_webhook.go @@ -0,0 +1,64 @@ +package installcmd + +// Webhook serving-cert provisioning for the GitOps path, via cert-manager. +// +// The operator's webhook server mounts spicebox-webhook-tls as a non-optional +// volume, so the Secret must exist before the operator pod can start — it cannot +// be minted by the pre-start Job (a same-cluster Job can't get the cert into the +// operator pod's mount before the pod is created) nor by the operator itself. +// cert-manager solves both halves declaratively: a self-signed Issuer + a +// Certificate populate the Secret, and the ca-injector copies the CA into the +// ValidatingWebhookConfiguration's caBundle via the inject-ca-from annotation — +// no ordering races, continuously reconciled. +// +// This makes cert-manager a hard dependency of the GitOps path. A user who does +// not run it passes --external-webhook-tls and provides spicebox-webhook-tls + +// the VWC caBundle themselves. +const ( + webhookCertResourceFile = "webhook-cert.yaml" + webhookCAInjectPatchFile = "patches/webhook-ca-inject.yaml" + + // webhookCertName is the cert-manager Certificate name; the VWC's + // inject-ca-from annotation references it as /. + webhookCertName = "spicebox-webhook-tls" + webhookVWCName = "spicebox-settings" +) + +// renderWebhookCertResources returns the self-signed Issuer + Certificate that +// populate the spicebox-webhook-tls Secret with a cert whose SANs cover the +// spicebox-webhook Service. A pure constant — no cfg dependence. +func renderWebhookCertResources() []byte { + return []byte("apiVersion: cert-manager.io/v1\n" + + "kind: Issuer\n" + + "metadata:\n" + + " name: spicebox-webhook-selfsign\n" + + " namespace: agentprimitives-system\n" + + "spec:\n" + + " selfSigned: {}\n" + + "---\n" + + "apiVersion: cert-manager.io/v1\n" + + "kind: Certificate\n" + + "metadata:\n" + + " name: " + webhookCertName + "\n" + + " namespace: agentprimitives-system\n" + + "spec:\n" + + " secretName: " + webhookCertName + "\n" + + " dnsNames:\n" + + " - spicebox-webhook.agentprimitives-system.svc\n" + + " - spicebox-webhook.agentprimitives-system.svc.cluster.local\n" + + " issuerRef:\n" + + " name: spicebox-webhook-selfsign\n" + + " kind: Issuer\n") +} + +// webhookCAInjectPatch annotates the ValidatingWebhookConfiguration so +// cert-manager's ca-injector writes the Certificate's CA into every webhook's +// caBundle. +func webhookCAInjectPatch() []byte { + return []byte("apiVersion: admissionregistration.k8s.io/v1\n" + + "kind: ValidatingWebhookConfiguration\n" + + "metadata:\n" + + " name: " + webhookVWCName + "\n" + + " annotations:\n" + + " cert-manager.io/inject-ca-from: agentprimitives-system/" + webhookCertName + "\n") +} diff --git a/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/bootstrap-job.yaml b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/bootstrap-job.yaml new file mode 100644 index 00000000..7ab7f450 --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/bootstrap-job.yaml @@ -0,0 +1,18 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: spicebox-bootstrap-secrets-703e5c2463 + namespace: agentprimitives-system +spec: + backoffLimit: 6 + template: + metadata: {} + spec: + containers: + - args: + - bootstrap-secrets + image: spicebox-operator:dev + name: bootstrap-secrets + resources: {} + restartPolicy: OnFailure + serviceAccountName: spicebox-operator diff --git a/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/components/postgres/kustomization.yaml b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/components/postgres/kustomization.yaml new file mode 100644 index 00000000..d89144aa --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/components/postgres/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - postgres.yaml diff --git a/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/kustomization.yaml b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/kustomization.yaml new file mode 100644 index 00000000..e2b00690 --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - base/install.yaml + - base/channelsd.yaml + - base/nats.yaml + - bootstrap-job.yaml + - routing.yaml + - webhook-cert.yaml +patches: + - path: patches/cluster-kind.yaml + - path: patches/webhook-ca-inject.yaml +components: + - components/postgres diff --git a/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/patches/cluster-kind.yaml b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/patches/cluster-kind.yaml new file mode 100644 index 00000000..a88a28f5 --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/patches/cluster-kind.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spicebox-operator + namespace: agentprimitives-system +spec: + template: + spec: + containers: + - name: operator + env: + - name: AP_CLUSTER_KIND + value: gke +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spicebox-webd + namespace: agentprimitives-system +spec: + template: + spec: + containers: + - name: webd + env: + - name: AP_CLUSTER_KIND + value: gke diff --git a/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/patches/webhook-ca-inject.yaml b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/patches/webhook-ca-inject.yaml new file mode 100644 index 00000000..702ca6be --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/patches/webhook-ca-inject.yaml @@ -0,0 +1,6 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: spicebox-settings + annotations: + cert-manager.io/inject-ca-from: agentprimitives-system/spicebox-webhook-tls diff --git a/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/routing.yaml b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/routing.yaml new file mode 100644 index 00000000..b848d10b --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/routing.yaml @@ -0,0 +1,57 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: spicebox-webd + namespace: agentprimitives-system +spec: + gatewayClassName: eg + listeners: + - hostname: webd.example.com + name: trusted-https + port: 443 + protocol: HTTPS + tls: + certificateRefs: + - name: spicebox-webd-trusted-tls + mode: Terminate + - hostname: sandbox.example.com + name: sandbox-https + port: 443 + protocol: HTTPS + tls: + certificateRefs: + - name: spicebox-webd-sandbox-tls + mode: Terminate + - name: acme-http + port: 80 + protocol: HTTP +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: spicebox-webd-trusted + namespace: agentprimitives-system +spec: + hostnames: + - webd.example.com + parentRefs: + - name: spicebox-webd + rules: + - backendRefs: + - name: spicebox-webd + port: 8080 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: spicebox-webd-sandbox + namespace: agentprimitives-system +spec: + hostnames: + - sandbox.example.com + parentRefs: + - name: spicebox-webd + rules: + - backendRefs: + - name: spicebox-webd + port: 8080 diff --git a/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/webhook-cert.yaml b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/webhook-cert.yaml new file mode 100644 index 00000000..005c8d01 --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/gke-postgres-routing/webhook-cert.yaml @@ -0,0 +1,21 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: spicebox-webhook-selfsign + namespace: agentprimitives-system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: spicebox-webhook-tls + namespace: agentprimitives-system +spec: + secretName: spicebox-webhook-tls + dnsNames: + - spicebox-webhook.agentprimitives-system.svc + - spicebox-webhook.agentprimitives-system.svc.cluster.local + issuerRef: + name: spicebox-webhook-selfsign + kind: Issuer diff --git a/cmd/oap/internal/installcmd/testdata/render/local/bootstrap-job.yaml b/cmd/oap/internal/installcmd/testdata/render/local/bootstrap-job.yaml new file mode 100644 index 00000000..7ab7f450 --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/local/bootstrap-job.yaml @@ -0,0 +1,18 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: spicebox-bootstrap-secrets-703e5c2463 + namespace: agentprimitives-system +spec: + backoffLimit: 6 + template: + metadata: {} + spec: + containers: + - args: + - bootstrap-secrets + image: spicebox-operator:dev + name: bootstrap-secrets + resources: {} + restartPolicy: OnFailure + serviceAccountName: spicebox-operator diff --git a/cmd/oap/internal/installcmd/testdata/render/local/kustomization.yaml b/cmd/oap/internal/installcmd/testdata/render/local/kustomization.yaml new file mode 100644 index 00000000..295c62c0 --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/local/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - base/install.yaml + - base/channelsd.yaml + - base/nats.yaml + - bootstrap-job.yaml + - webhook-cert.yaml +patches: + - path: patches/cluster-kind.yaml + - path: patches/webhook-ca-inject.yaml diff --git a/cmd/oap/internal/installcmd/testdata/render/local/patches/cluster-kind.yaml b/cmd/oap/internal/installcmd/testdata/render/local/patches/cluster-kind.yaml new file mode 100644 index 00000000..9a75e5b5 --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/local/patches/cluster-kind.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spicebox-operator + namespace: agentprimitives-system +spec: + template: + spec: + containers: + - name: operator + env: + - name: AP_CLUSTER_KIND + value: local +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spicebox-webd + namespace: agentprimitives-system +spec: + template: + spec: + containers: + - name: webd + env: + - name: AP_CLUSTER_KIND + value: local diff --git a/cmd/oap/internal/installcmd/testdata/render/local/patches/webhook-ca-inject.yaml b/cmd/oap/internal/installcmd/testdata/render/local/patches/webhook-ca-inject.yaml new file mode 100644 index 00000000..702ca6be --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/local/patches/webhook-ca-inject.yaml @@ -0,0 +1,6 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: spicebox-settings + annotations: + cert-manager.io/inject-ca-from: agentprimitives-system/spicebox-webhook-tls diff --git a/cmd/oap/internal/installcmd/testdata/render/local/webhook-cert.yaml b/cmd/oap/internal/installcmd/testdata/render/local/webhook-cert.yaml new file mode 100644 index 00000000..005c8d01 --- /dev/null +++ b/cmd/oap/internal/installcmd/testdata/render/local/webhook-cert.yaml @@ -0,0 +1,21 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: spicebox-webhook-selfsign + namespace: agentprimitives-system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: spicebox-webhook-tls + namespace: agentprimitives-system +spec: + secretName: spicebox-webhook-tls + dnsNames: + - spicebox-webhook.agentprimitives-system.svc + - spicebox-webhook.agentprimitives-system.svc.cluster.local + issuerRef: + name: spicebox-webhook-selfsign + kind: Issuer diff --git a/cmd/oap/root.go b/cmd/oap/root.go index 7c93da15..351197e5 100644 --- a/cmd/oap/root.go +++ b/cmd/oap/root.go @@ -59,6 +59,7 @@ func NewRootCmdWithGlobals(g *Globals) *cobra.Command { root.AddCommand(installcmd.NewImageCmd(g)) root.AddCommand(installcmd.NewInitCmd(g)) root.AddCommand(installcmd.NewCleanCmd(g)) + root.AddCommand(installcmd.NewRenderCmd(g)) root.AddCommand(toolscmd.NewCmd(g)) root.AddCommand(identitycmd.NewCmd(g)) root.AddCommand(identitycmd.NewUserIdentityCmd(g)) diff --git a/config/manager/role.yaml b/config/manager/role.yaml index 33f5773f..383c7f54 100644 --- a/config/manager/role.yaml +++ b/config/manager/role.yaml @@ -254,6 +254,13 @@ rules: - apps resources: - deployments + verbs: + - get + - list + - patch +- apiGroups: + - apps + resources: - statefulsets verbs: - get diff --git a/go.mod b/go.mod index 030cb501..57265df0 100644 --- a/go.mod +++ b/go.mod @@ -365,8 +365,8 @@ require ( sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20260424122448-c8b4b9d61fbd // indirect sigs.k8s.io/controller-tools v0.20.1 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect - sigs.k8s.io/kustomize/api v0.21.1 // indirect - sigs.k8s.io/kustomize/kyaml v0.21.1 // indirect + sigs.k8s.io/kustomize/api v0.21.1 + sigs.k8s.io/kustomize/kyaml v0.21.1 sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) diff --git a/internal/cmd/operator/bootstrapcmd.go b/internal/cmd/operator/bootstrapcmd.go new file mode 100644 index 00000000..0ea88d7d --- /dev/null +++ b/internal/cmd/operator/bootstrapcmd.go @@ -0,0 +1,48 @@ +package main + +import ( + "fmt" + + "github.com/spf13/cobra" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" +) + +// newBootstrapSecretsCmd is the pre-start bootstrap Job's entrypoint: it mints +// every internal Secret a GitOps-applied install needs (idempotently, never +// rotating) and exits, WITHOUT starting the manager. It runs before the operator +// (and before SpiceDB), so the operator and the data-plane pods mount their +// Secrets cleanly at first boot — no mint-after-start, no pod rolls. +// +// It uses a direct (uncached) client, which reads live and so is both a correct +// reader and writer for the create-if-absent mint logic. The default client-go +// scheme already knows corev1.Secret. +func newBootstrapSecretsCmd() *cobra.Command { + var skipSpiceDBToken, skipPostgresToken bool + cmd := &cobra.Command{ + Use: "bootstrap-secrets", + Short: "Mint the internal Secrets a GitOps-applied install needs, then exit (run as a pre-start Job).", + SilenceUsage: true, + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, err := ctrl.GetConfig() + if err != nil { + return fmt.Errorf("load kube config: %w", err) + } + cl, err := client.New(cfg, client.Options{}) + if err != nil { + return fmt.Errorf("build client: %w", err) + } + return bootstrap.MintAllInternalSecrets(cmd.Context(), cl, cl, bootstrap.MintOptions{ + SkipSpiceDBToken: skipSpiceDBToken, + SkipPostgresToken: skipPostgresToken, + }) + }, + } + cmd.Flags().BoolVar(&skipSpiceDBToken, "skip-spicedb-token", false, + "Do not mint spicebox-spicedb-token (external SpiceDB: the pre-shared key is yours to provide)") + cmd.Flags().BoolVar(&skipPostgresToken, "skip-postgres-token", false, + "Do not mint spicebox-postgres-token (external Postgres: the URI is yours to provide)") + return cmd +} diff --git a/internal/cmd/operator/main.go b/internal/cmd/operator/main.go index c37be848..94588356 100644 --- a/internal/cmd/operator/main.go +++ b/internal/cmd/operator/main.go @@ -58,6 +58,7 @@ import ( "github.com/authzed/openagentprimitives/pkg/platform/apimage" "github.com/authzed/openagentprimitives/pkg/platform/artifactstore" blobstore "github.com/authzed/openagentprimitives/pkg/platform/artifactstore/blob" + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" "github.com/authzed/openagentprimitives/pkg/web/admind" // mcpui is registered ONLY here (the artifactrender controller's @@ -86,6 +87,7 @@ import ( "github.com/authzed/openagentprimitives/pkg/authz/revocation" "github.com/authzed/openagentprimitives/pkg/authz/spicedb" _ "github.com/authzed/openagentprimitives/pkg/authz/spicedb/relsource/imports" // completes the relsource claim table before any guarded write (BootstrapSource, pttagmint, ...) is checked + spicedbschema "github.com/authzed/openagentprimitives/pkg/authz/spicedb/schema" _ "github.com/authzed/openagentprimitives/pkg/channels/channelassets/mcpui" _ "github.com/authzed/openagentprimitives/pkg/channels/channelassets/svg" "github.com/authzed/openagentprimitives/pkg/channels/channelevents" @@ -840,6 +842,9 @@ func newCommand() *cobra.Command { return nil }, } + // Pre-start bootstrap Job entrypoint: `operator bootstrap-secrets` mints the + // internal Secrets and exits, without starting the manager. + cmd.AddCommand(newBootstrapSecretsCmd()) fs := cmd.Flags() fs.StringVar(&cfg.metricsAddr, "metrics-bind-address", "127.0.0.1:8080", "Address for the metrics endpoint. The endpoint is unauthenticated, so the default binds loopback-only; widen it (e.g. \":8080\") only behind an authenticating or network-policy-enforced scrape path.") @@ -3057,6 +3062,21 @@ func run(cfg *config) { log.Info("admind token not present; admin API disabled", "path", admindTokenFile) } + // Leader-only bootstrap: apply the SpiceDB base schema so a bare GitOps apply + // (Argo/Flux, no `oap install`) has its schema seeded. The internal Secrets + // are minted earlier, by the pre-start bootstrap Job (MintAllInternalSecrets), + // not here — schema is the one step that needs SpiceDB already running. A plain + // RunnableFunc does not implement LeaderElectionRunnable, so controller-runtime + // runs it only on the leader. spiceDBClient is a real, non-nil *spicedb.Client + // by this point (construction failure exits above), so SchemaIOFor cannot + // produce a typed-nil interface. + if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + return bootstrap.RunBootstrap(ctx, spicedb.SchemaIOFor(spiceDBClient), spicedbschema.Schema) + })); err != nil { + log.Error(err, "registering bootstrap runnable") + os.Exit(1) + } + if adm != nil { // AgentSession informer feeds the live aggregator. Registered as // a Runnable so it runs once the manager's cache has started. diff --git a/pkg/controllers/agentsession/controller.go b/pkg/controllers/agentsession/controller.go index f7d83082..b2761e1c 100644 --- a/pkg/controllers/agentsession/controller.go +++ b/pkg/controllers/agentsession/controller.go @@ -144,6 +144,10 @@ import ( // NATS and Neo4j ship as StatefulSets. // +kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list // +kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list +// The bootstrap Job's NATS-creds re-mint (pkg/platform/bootstrap, run via the +// operator SA) rolls consumer Deployments when a grant gains a subject on +// upgrade, so a stale JWT does not stay on the wire — that patch needs deployments/patch. +// +kubebuilder:rbac:groups="apps",resources=deployments,verbs=patch // metrics.k8s.io pods (get;list) backs the same snapshot's CPU/Memory rollup, // summing live PodMetrics for the namespace. Best-effort: absent // metrics-server the List degrades to "n/a" rather than failing. diff --git a/pkg/platform/bootstrap/appsecrets.go b/pkg/platform/bootstrap/appsecrets.go new file mode 100644 index 00000000..9ed1fd92 --- /dev/null +++ b/pkg/platform/bootstrap/appsecrets.go @@ -0,0 +1,94 @@ +package bootstrap + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/authz/adoptguard" + "github.com/authzed/openagentprimitives/pkg/platform/identity/passthroughlink" +) + +// AdmindTokenSecretName is the admin API bearer token Secret. It is a plain +// random token like the memory tokens (minted via EnsureTokenSecret), so it has +// no dedicated minter — RunBootstrap mints it by name. +const AdmindTokenSecretName = "spicebox-admind-token" + +// EnsurePassthroughLinkKey creates the passthrough-link HMAC signing-key Secret +// if absent, or repairs it if the existing `key` is unusable (not hex, too +// short) — the ESO/GitOps-provisioned-empty case. It reports minted=true when it +// wrote a new key (create OR repair), so the caller can roll the consumers +// (channelsd/webd) that read the key once at startup. An existing, usable key is +// left untouched (minted=false), so it is never rotated out from under a running +// signer, and a repair preserves the caller's other metadata (labels/annotations). +// +// logf narrates the outcome (a rotation is the one thing here an operator must be +// able to read afterwards); nil discards. reader must be a live/uncached reader. +func EnsurePassthroughLinkKey(ctx context.Context, reader client.Reader, writer client.Client, logf func(string, ...any)) (minted bool, err error) { + if logf == nil { + logf = func(string, ...any) {} + } + name := v1alpha1.PassthroughLinkSigningKeySecret + var existing corev1.Secret + getErr := reader.Get(ctx, types.NamespacedName{Namespace: TokenSecretNamespace, Name: name}, &existing) + var unusable error + switch { + case getErr == nil: + if _, unusable = passthroughlink.DecodeHexKey(existing.Data["key"]); unusable == nil { + logf(" %s Secret exists with a usable key, skipping", name) + return false, nil // usable key present — never rotate + } + case !apierrors.IsNotFound(getErr): + return false, fmt.Errorf("get %s secret: %w", name, getErr) + } + + encoded, err := randomHexKey() + if err != nil { + return false, err + } + + if getErr == nil { // exists but unusable → repair in place, preserving metadata + if existing.Data == nil { + existing.Data = map[string][]byte{} + } + existing.Data["key"] = encoded + if err := writer.Update(ctx, &existing); err != nil { + return false, fmt.Errorf("update %s secret: %w", name, err) + } + logf(" regenerated the key in the existing %s Secret (%v)", name, unusable) + return true, nil + } + + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: TokenSecretNamespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{"key": encoded}, + } + adoptguard.WithAdoptedLabel(sec) + if err := writer.Create(ctx, sec); err != nil { + if apierrors.IsAlreadyExists(err) { + return false, nil + } + return false, fmt.Errorf("create %s secret: %w", name, err) + } + logf(" generated %s Secret", name) + return true, nil +} + +// randomHexKey returns 32 random bytes hex-encoded (64 chars) — the HMAC key +// shape passthroughlink.DecodeHexKey accepts. +func randomHexKey() ([]byte, error) { + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return nil, fmt.Errorf("generate signing key: %w", err) + } + return []byte(hex.EncodeToString(buf)), nil +} diff --git a/pkg/platform/bootstrap/appsecrets_test.go b/pkg/platform/bootstrap/appsecrets_test.go new file mode 100644 index 00000000..345f05fe --- /dev/null +++ b/pkg/platform/bootstrap/appsecrets_test.go @@ -0,0 +1,61 @@ +package bootstrap + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/platform/identity/passthroughlink" +) + +func TestEnsurePassthroughLinkKey_CreatesUsableWhenAbsent(t *testing.T) { + c := newFakeClient() + minted, err := EnsurePassthroughLinkKey(context.Background(), c, c, nil) + require.NoError(t, err) + assert.True(t, minted, "a fresh key is minted") + + var got corev1.Secret + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: TokenSecretNamespace, Name: v1alpha1.PassthroughLinkSigningKeySecret}, &got)) + _, usable := passthroughlink.DecodeHexKey(got.Data["key"]) + assert.NoError(t, usable, "minted key passes DecodeHexKey") +} + +func TestEnsurePassthroughLinkKey_LeavesUsableUntouched(t *testing.T) { + good := []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") // 64 hex = 32 bytes + _, err := passthroughlink.DecodeHexKey(good) + require.NoError(t, err, "fixture key must be usable") + existing := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: v1alpha1.PassthroughLinkSigningKeySecret, Namespace: TokenSecretNamespace}, + Data: map[string][]byte{"key": good}, + } + c := newFakeClient(existing) + minted, err := EnsurePassthroughLinkKey(context.Background(), c, c, nil) + require.NoError(t, err) + assert.False(t, minted, "a usable key is never rotated") + + var got corev1.Secret + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: TokenSecretNamespace, Name: v1alpha1.PassthroughLinkSigningKeySecret}, &got)) + assert.Equal(t, string(good), string(got.Data["key"])) +} + +func TestEnsurePassthroughLinkKey_RepairsUnusable(t *testing.T) { + existing := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: v1alpha1.PassthroughLinkSigningKeySecret, Namespace: TokenSecretNamespace}, + Data: map[string][]byte{"key": []byte("too-short")}, // not hex / too short + } + c := newFakeClient(existing) + minted, err := EnsurePassthroughLinkKey(context.Background(), c, c, nil) + require.NoError(t, err) + assert.True(t, minted, "an unusable key is repaired (reported as minted so consumers roll)") + + var got corev1.Secret + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: TokenSecretNamespace, Name: v1alpha1.PassthroughLinkSigningKeySecret}, &got)) + _, usable := passthroughlink.DecodeHexKey(got.Data["key"]) + assert.NoError(t, usable, "repaired key is usable") +} diff --git a/pkg/platform/bootstrap/mint.go b/pkg/platform/bootstrap/mint.go new file mode 100644 index 00000000..793bafb2 --- /dev/null +++ b/pkg/platform/bootstrap/mint.go @@ -0,0 +1,201 @@ +package bootstrap + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "net/url" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/authzed/openagentprimitives/pkg/authz/adoptguard" +) + +const ( + spiceDBTokenSecretName = "spicebox-spicedb-token" + postgresTokenSecretName = "spicebox-postgres-token" +) + +// MintOptions tunes which optional, instance-coupled Secrets MintAllInternalSecrets +// creates. Both default to false (mint); set true to skip when the user points at +// an EXTERNAL instance and provides the Secret themselves. +type MintOptions struct { + SkipSpiceDBToken bool // external SpiceDB: token is the user's pre-shared key + SkipPostgresToken bool // external Postgres: URI is the user's +} + +// MintAllInternalSecrets creates every app-internal Secret the bundle needs +// (except webhook-tls, which is provisioned separately), idempotently and +// without rotating anything that already exists. It is the entry point the +// pre-start bootstrap Job runs, so every consumer mounts its Secret cleanly at +// first boot. reader must be a live/uncached reader; writer the controller client. +// +// NOTE: the NATS Secrets (identity/tls/creds) are minted by MintNATSSecrets, +// added in a follow-up commit; this covers the token-shaped ones. +func MintAllInternalSecrets(ctx context.Context, reader client.Reader, writer client.Client, opts MintOptions) error { + for _, name := range append(append([]string{}, MemoryTokenSecretNames...), AdmindTokenSecretName) { + if _, err := EnsureTokenSecret(ctx, reader, writer, TokenSecretNamespace, name); err != nil { + return err + } + } + if _, err := EnsurePassthroughLinkKey(ctx, reader, writer, nil); err != nil { + return err + } + // NATS identity → TLS → client creds (creds require the first two). Always + // bundled; never gated by the external flags. + if err := EnsureNATSIdentity(ctx, writer); err != nil { + return err + } + if err := EnsureNATSTLS(ctx, writer); err != nil { + return err + } + if err := EnsureNATSClientCreds(ctx, writer, nil); err != nil { + return err + } + if !opts.SkipPostgresToken { + if err := EnsurePostgresToken(ctx, reader, writer); err != nil { + return err + } + } + if !opts.SkipSpiceDBToken { + if err := EnsureSpiceDBToken(ctx, reader, writer); err != nil { + return err + } + // The /spicedb datastore URI is derived from the postgres password, so it + // can only be set once BOTH tokens exist. Skipped when either is external. + if !opts.SkipPostgresToken { + if err := EnsureSpiceDBDatastoreURI(ctx, reader, writer); err != nil { + return err + } + } + } + return nil +} + +// EnsureSpiceDBToken creates spicebox-spicedb-token (32 random bytes hex) with +// the dual `token`/`preshared_key` shape, reconciling an ESO/GitOps-provisioned +// preshared_key-only Secret. Create-if-absent; never rotates an existing token. +func EnsureSpiceDBToken(ctx context.Context, reader client.Reader, writer client.Client) error { + var existing corev1.Secret + err := reader.Get(ctx, types.NamespacedName{Namespace: TokenSecretNamespace, Name: spiceDBTokenSecretName}, &existing) + if err == nil { + tok, pre := existing.Data["token"], existing.Data["preshared_key"] + if len(tok) > 0 && bytes.Equal(tok, pre) { + return nil + } + if existing.Data == nil { + existing.Data = map[string][]byte{} + } + switch { + case len(tok) > 0: + existing.Data["preshared_key"] = tok + case len(pre) > 0: + existing.Data["token"] = pre + default: + minted, mErr := randomHexKey() + if mErr != nil { + return mErr + } + existing.Data["token"], existing.Data["preshared_key"] = minted, minted + } + if err := writer.Update(ctx, &existing); err != nil { + return fmt.Errorf("update spicedb token secret: %w", err) + } + return nil + } + if !apierrors.IsNotFound(err) { + return fmt.Errorf("get spicedb token secret: %w", err) + } + tok, err := randomHexKey() + if err != nil { + return err + } + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: spiceDBTokenSecretName, Namespace: TokenSecretNamespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{"token": tok, "preshared_key": tok}, + } + adoptguard.WithAdoptedLabel(sec) + if err := writer.Create(ctx, sec); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create spicedb token secret: %w", err) + } + return nil +} + +// EnsurePostgresToken creates spicebox-postgres-token (16 random bytes hex +// password + a `uri` to the bundled Postgres `/memory` DB). Create-if-absent; +// never rotates. +func EnsurePostgresToken(ctx context.Context, reader client.Reader, writer client.Client) error { + var existing corev1.Secret + err := reader.Get(ctx, types.NamespacedName{Namespace: TokenSecretNamespace, Name: postgresTokenSecretName}, &existing) + if err == nil { + return nil + } + if !apierrors.IsNotFound(err) { + return fmt.Errorf("get postgres token secret: %w", err) + } + buf := make([]byte, 16) + if _, err := rand.Read(buf); err != nil { + return fmt.Errorf("generate postgres password: %w", err) + } + password := hex.EncodeToString(buf) + uri := (&url.URL{ + Scheme: "postgres", + User: url.UserPassword("postgres", password), + Host: "spicebox-postgres.agentprimitives-system.svc:5432", + Path: "/memory", + RawQuery: "sslmode=disable", + }).String() + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: postgresTokenSecretName, Namespace: TokenSecretNamespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{"password": []byte(password), "uri": []byte(uri)}, + } + adoptguard.WithAdoptedLabel(sec) + if err := writer.Create(ctx, sec); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create postgres token secret: %w", err) + } + return nil +} + +// EnsureSpiceDBDatastoreURI writes a `datastore_uri` (the bundled Postgres +// `/spicedb` DB, password read from spicebox-postgres-token) onto +// spicebox-spicedb-token. Both Secrets must already exist. Idempotent. +func EnsureSpiceDBDatastoreURI(ctx context.Context, reader client.Reader, writer client.Client) error { + var pg corev1.Secret + if err := reader.Get(ctx, types.NamespacedName{Namespace: TokenSecretNamespace, Name: postgresTokenSecretName}, &pg); err != nil { + return fmt.Errorf("get postgres token secret: %w", err) + } + pw := string(pg.Data["password"]) + if pw == "" { + return fmt.Errorf("postgres token secret has no password") + } + uri := (&url.URL{ + Scheme: "postgres", + User: url.UserPassword("postgres", pw), + Host: "spicebox-postgres.agentprimitives-system.svc:5432", + Path: "/spicedb", + RawQuery: "sslmode=disable", + }).String() + var sp corev1.Secret + if err := reader.Get(ctx, types.NamespacedName{Namespace: TokenSecretNamespace, Name: spiceDBTokenSecretName}, &sp); err != nil { + return fmt.Errorf("get spicedb token secret: %w", err) + } + if string(sp.Data["datastore_uri"]) == uri { + return nil + } + if sp.Data == nil { + sp.Data = map[string][]byte{} + } + sp.Data["datastore_uri"] = []byte(uri) + if err := writer.Update(ctx, &sp); err != nil { + return fmt.Errorf("update spicedb datastore_uri: %w", err) + } + return nil +} diff --git a/pkg/platform/bootstrap/mint_test.go b/pkg/platform/bootstrap/mint_test.go new file mode 100644 index 00000000..305169ea --- /dev/null +++ b/pkg/platform/bootstrap/mint_test.go @@ -0,0 +1,79 @@ +package bootstrap + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/platform/identity/passthroughlink" +) + +func getSecret(t *testing.T, c client.Client, name string) *corev1.Secret { + t.Helper() + var s corev1.Secret + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: TokenSecretNamespace, Name: name}, &s), "get %s", name) + return &s +} + +func TestMintAllInternalSecrets_MintsTheTokenSet(t *testing.T) { + c := newFakeClient() + require.NoError(t, MintAllInternalSecrets(context.Background(), c, c, MintOptions{})) + + for _, n := range MemoryTokenSecretNames { + s := getSecret(t, c, n) + assert.Len(t, string(s.Data[TokenSecretKey]), 64, "%s", n) + } + assert.Len(t, string(getSecret(t, c, AdmindTokenSecretName).Data[TokenSecretKey]), 64) + + _, usableErr := passthroughlink.DecodeHexKey(getSecret(t, c, v1alpha1.PassthroughLinkSigningKeySecret).Data["key"]) + assert.NoError(t, usableErr, "passthrough key usable") + + sp := getSecret(t, c, spiceDBTokenSecretName) + assert.Equal(t, string(sp.Data["token"]), string(sp.Data["preshared_key"]), "spicedb token == preshared_key") + assert.Contains(t, string(sp.Data["datastore_uri"]), "/spicedb", "datastore URI set from postgres password") + + pg := getSecret(t, c, postgresTokenSecretName) + assert.NotEmpty(t, pg.Data["password"]) + assert.True(t, strings.Contains(string(pg.Data["uri"]), "/memory")) +} + +func TestMintAllInternalSecrets_SkipsExternalTokens(t *testing.T) { + c := newFakeClient() + require.NoError(t, MintAllInternalSecrets(context.Background(), c, c, MintOptions{SkipSpiceDBToken: true, SkipPostgresToken: true})) + + // External instances: those two tokens are the user's to provide. + var sp corev1.Secret + assert.Error(t, c.Get(context.Background(), types.NamespacedName{Namespace: TokenSecretNamespace, Name: spiceDBTokenSecretName}, &sp), "spicedb token NOT minted when external") + var pg corev1.Secret + assert.Error(t, c.Get(context.Background(), types.NamespacedName{Namespace: TokenSecretNamespace, Name: postgresTokenSecretName}, &pg), "postgres token NOT minted when external") + + // But the always-internal ones are still minted. + assert.Len(t, string(getSecret(t, c, "spicebox-webd-memory-token").Data[TokenSecretKey]), 64) +} + +func TestMintAllInternalSecrets_Idempotent(t *testing.T) { + c := newFakeClient() + require.NoError(t, MintAllInternalSecrets(context.Background(), c, c, MintOptions{})) + before := getSecret(t, c, spiceDBTokenSecretName).Data["token"] + require.NoError(t, MintAllInternalSecrets(context.Background(), c, c, MintOptions{})) + after := getSecret(t, c, spiceDBTokenSecretName).Data["token"] + assert.Equal(t, string(before), string(after), "second run does not rotate the spicedb token") +} + +func TestEnsurePostgresToken_IdempotentNoRotate(t *testing.T) { + existing := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: postgresTokenSecretName, Namespace: TokenSecretNamespace}, + Data: map[string][]byte{"password": []byte("keepme"), "uri": []byte("postgres://x/memory")}, + } + c := newFakeClient(existing) + require.NoError(t, EnsurePostgresToken(context.Background(), c, c)) + assert.Equal(t, "keepme", string(getSecret(t, c, postgresTokenSecretName).Data["password"]), "existing password untouched") +} diff --git a/pkg/platform/bootstrap/nats.go b/pkg/platform/bootstrap/nats.go new file mode 100644 index 00000000..144c409a --- /dev/null +++ b/pkg/platform/bootstrap/nats.go @@ -0,0 +1,375 @@ +package bootstrap + +import ( + "bytes" + "context" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/authzed/openagentprimitives/pkg/channels/channelevents" + apnats "github.com/authzed/openagentprimitives/pkg/platform/nats" +) + +// The principal name minted into each creds file. Named once because the name +// is also the input to apnats.InboxSubjectFor, and the client rederives the +// matching prefix from the same name in the JWT — writing it twice invites the +// two to drift, which breaks request/reply rather than merely widening a grant. +const ( + channelsdNATSUser = "channelsd" + cliNATSUser = "cli" + webdNATSUser = "webd" +) + +// WebdNATSGrant is webd's minted NATS user grant. webd SUBSCRIBES to session +// out-subjects (chat + live-view mirror) AND does NATS request-reply: the +// builtin web-chat listener and the live-view submit view_message via +// RequestIn, so webd both PUBLISHES the request and must SUBSCRIBE to its +// reply inbox. The original "never publishes, subscribe-only" assumption +// predated those surfaces and was the bug — a missing inbox entry made every +// view-originated send time out even though the message was delivered. +// +// PubAllow is enumerated because an OMITTED one is publish-ANYWHERE, not +// deny-all (see apnats.UserGrant). This grant used to omit it, which left the +// browser-facing process holding publish rights on the entire control bus: +// ap.revocation, every session's out-subjects (forging the agent's own voice), +// and every inbound kind — including in.user_message, whose channelsd handler +// is the audit-signing inbound writer webd is deliberately kept away from +// (pkg/channels/channelkinds/clienthosted.SubmitUserMessage spells out why: webd holds +// no memory-write credential, so it must round-trip through view_message). +// +// The five subjects below are webd's complete publish surface, each a verified +// call site: view_message (chat SubmitUserMessage and the /interact +// user_message | annotation_batch | mcp_ui_action kinds, all via +// channelkinds.RequestViewMessage), interrupt_request and interaction_decision +// (clienthosted.Listener), resurface_request (channelkinds. +// PublishResurfaceRequest) and app_tool_call (webui/interact). webd links +// pkg/channels/channelsd/pipeline but only for its Authz interface type — it constructs +// no Pipeline, so none of that package's publishes can fire here. +// +// webd needs no publish grant on any inbox: it is never a NATS responder +// (nothing in its import graph calls msg.Respond), and a requester publishes to +// the target subject, not to its own reply inbox. +// +// TestWebdNATSGrantPublishSurface asserts every line of this against a real +// server; add a subject there and here together. +var WebdNATSGrant = apnats.UserGrant{ + Name: webdNATSUser, + PubAllow: []string{ + "ap.session.*.*.in.view_message", + "ap.session.*.*.in.interrupt_request", + "ap.session.*.*.in.interaction_decision", + "ap.session.*.*.in.resurface_request", + "ap.session.*.*.in.app_tool_call", + // ui_data_binding is the agent-UI's own data path: pkg/web/webui/agentui's + // bindingsHandler resolves every `source: tool` binding by asking the + // session's runner, via pkg/web/uibindings/tool. + // + // Its absence is why that feature had never worked on a real install. + // The failure gave nothing away: a publish the server refuses does not + // fail the caller's request — NATS reports the violation as an ASYNC + // error on the connection, while the request itself simply waits for a + // reply that can now never come and dies on its own timeout. So every + // binding surfaced as "the runner did not answer", which reads as a + // missing or busy runner and is why the first diagnosis of it chased a + // sleeping session instead of a permission. + "ap.session.*.*.in.ui_data_binding", + // ui_presence is the liveness heartbeat webd republishes while a + // viewer has an agent-UI open and focused, so the runner answering + // that UI's bindings does not idle out underneath them. + // + // Added in the SAME change as the kind and as its row in + // TestWebdNATSGrantPublishSurface, which is the discipline the line + // above exists to teach: ui_data_binding shipped without its grant and + // the whole agent-UI data path silently never worked. + "ap.session.*.*.in.ui_presence", + // ui_action is the agent-UI's INVOKE path: pkg/web/webui/agentui's + // actions handler sends the declared action to the session's runner and + // relays the synchronous reply. It shipped without this entry, so every + // agent-UI action failed exactly the way ui_data_binding did — see above. + "ap.session.*.*.in.ui_action", + // channelevents.WebhookInboundSubject — not a *.* session template + // (there is no session yet at delivery time, see that constant's own + // doc) but the same failure shape as ui_data_binding above: without + // this entry, pkg/web/webui/channelwebhook's Publish call is silently + // refused by the server, its returned error is nil either way (NATS + // reports permission violations asynchronously on the connection, not + // to the caller), and the handler answers the webhook sender with 202 + // for a delivery that was actually dropped — the one status the + // route's whole retry contract promises will never happen. Referenced + // via the exported constant, not a copied literal, so this and the + // route's own Publish call cannot drift apart. + channelevents.WebhookInboundSubject, + }, + SubAllow: []string{"ap.session.*.*.out.>", apnats.InboxSubjectFor(webdNATSUser)}, +} + +// EnsureNATSIdentity creates the spicebox-nats-identity Secret if it does not +// already exist. Idempotent: a re-run must not rotate the identity. +func EnsureNATSIdentity(ctx context.Context, c client.Client) error { + var existing corev1.Secret + err := c.Get(ctx, types.NamespacedName{Namespace: TokenSecretNamespace, Name: "spicebox-nats-identity"}, &existing) + if err == nil { + return nil + } + if !apierrors.IsNotFound(err) { + return fmt.Errorf("get nats identity secret: %w", err) + } + + id, err := apnats.GenerateIdentity() + if err != nil { + return fmt.Errorf("generate nats identity: %w", err) + } + + // The interpolated values (OperatorJWT, AccountPublicKey, AccountJWT) are + // base32/base64url-encoded and contain no whitespace, quotes, or braces, so + // they are safe to inline into the NATS config format without escaping. + authConf := fmt.Sprintf(`operator: %s +resolver: MEMORY +resolver_preload: { + %s: %s +} +`, id.OperatorJWT, id.AccountPublicKey, id.AccountJWT) + + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "spicebox-nats-identity", Namespace: TokenSecretNamespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + "operator-jwt": []byte(id.OperatorJWT), + "account-jwt": []byte(id.AccountJWT), + "account-public-key": []byte(id.AccountPublicKey), + "account-signing-seed": id.AccountSigningSeed, + "account-signing-public-key": []byte(id.AccountSigningPublicKey), + "auth.conf": []byte(authConf), + }, + } + if err := c.Create(ctx, sec); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create nats identity secret: %w", err) + } + return nil +} + +// EnsureNATSTLS creates the spicebox-nats-tls Secret if it does not already +// exist. Idempotent: a re-run must not rotate the TLS material. +func EnsureNATSTLS(ctx context.Context, c client.Client) error { + var existing corev1.Secret + err := c.Get(ctx, types.NamespacedName{Namespace: TokenSecretNamespace, Name: "spicebox-nats-tls"}, &existing) + if err == nil { + return nil + } + if !apierrors.IsNotFound(err) { + return fmt.Errorf("get nats tls secret: %w", err) + } + + m, err := apnats.GenerateTLS([]string{ + "spicebox-nats", + "spicebox-nats.agentprimitives-system", + "spicebox-nats.agentprimitives-system.svc", + "spicebox-nats.agentprimitives-system.svc.cluster.local", + }) + if err != nil { + return fmt.Errorf("generate nats tls: %w", err) + } + + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "spicebox-nats-tls", Namespace: TokenSecretNamespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{"ca.crt": m.CACertPEM, "tls.crt": m.ServerCertPEM, "tls.key": m.ServerKeyPEM}, + } + if err := c.Create(ctx, sec); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create nats tls secret: %w", err) + } + return nil +} + +// natsClientCred pairs one creds Secret with the grant its user JWT must carry. +type natsClientCred struct { + secretName string + grant apnats.UserGrant +} + +// natsClientCreds is every principal a bus credential is minted for. A function +// rather than a var because two of the three grants call apnats.InboxSubjectFor. +func natsClientCreds() []natsClientCred { + return []natsClientCred{ + { + secretName: "spicebox-channelsd-nats-creds", + grant: apnats.UserGrant{ + Name: channelsdNATSUser, + PubAllow: []string{"ap.>", "_INBOX.>"}, + SubAllow: []string{"ap.>", apnats.InboxSubjectFor(channelsdNATSUser)}, + }, + }, + { + secretName: "spicebox-cli-nats-creds", + grant: apnats.UserGrant{ + Name: cliNATSUser, + PubAllow: []string{"ap.>", "_INBOX.>"}, + SubAllow: []string{"ap.>", apnats.InboxSubjectFor(cliNATSUser)}, + }, + }, + {secretName: "spicebox-webd-nats-creds", grant: WebdNATSGrant}, + } +} + +// EnsureNATSClientCreds mints the per-principal NATS creds Secrets. It requires +// spicebox-nats-identity + spicebox-nats-tls to already exist (EnsureNATSIdentity +// + EnsureNATSTLS run first). Each creds Secret is minted only when absent or +// when the stored JWT no longer carries the shipped grant (see ensureNATSCredsSecret). +func EnsureNATSClientCreds(ctx context.Context, c client.Client, logf func(string, ...any)) error { + if logf == nil { + logf = func(string, ...any) {} + } + var identitySec corev1.Secret + if err := c.Get(ctx, types.NamespacedName{Namespace: TokenSecretNamespace, Name: "spicebox-nats-identity"}, &identitySec); err != nil { + return fmt.Errorf("get nats identity secret (run EnsureNATSIdentity first): %w", err) + } + var tlsSec corev1.Secret + if err := c.Get(ctx, types.NamespacedName{Namespace: TokenSecretNamespace, Name: "spicebox-nats-tls"}, &tlsSec); err != nil { + return fmt.Errorf("get nats tls secret (run EnsureNATSTLS first): %w", err) + } + caCrt := tlsSec.Data["ca.crt"] + if len(caCrt) == 0 { + return fmt.Errorf("nats tls secret is missing ca.crt") + } + + // MintUser only consults AccountPublicKey + AccountSigningSeed. + id := &apnats.Identity{ + AccountPublicKey: string(identitySec.Data["account-public-key"]), + AccountSigningSeed: identitySec.Data["account-signing-seed"], + } + if id.AccountPublicKey == "" || len(id.AccountSigningSeed) == 0 { + return fmt.Errorf("nats identity secret is missing account-public-key or account-signing-seed") + } + + for _, cred := range natsClientCreds() { + if err := ensureNATSCredsSecret(ctx, c, cred, id, caCrt, logf); err != nil { + return err + } + } + return nil +} + +// ensureNATSCredsSecret reconciles one creds Secret against the shipped grant, +// minting a user JWT only when the stored one does not already carry it. A +// user JWT is a frozen copy of the grant it was minted from, so a cluster +// installed before a subject was added to a grant kept a JWT without it forever +// until this drift check (grant.CredsDrift) forced a re-mint; a re-mint rotates +// the user nkey, so it must be rare (set-compare, order-insensitive). ca.crt is +// refreshed on the same write. +func ensureNATSCredsSecret(ctx context.Context, c client.Client, cred natsClientCred, id *apnats.Identity, caCrt []byte, logf func(string, ...any)) error { + key := types.NamespacedName{Namespace: TokenSecretNamespace, Name: cred.secretName} + + var existing corev1.Secret + err := c.Get(ctx, key, &existing) + switch { + case err == nil: + drift := cred.grant.CredsDrift(string(existing.Data["nats.creds"])) + if drift == "" && bytes.Equal(existing.Data["ca.crt"], caCrt) { + return nil + } + if drift == "" { + drift = "the stored CA no longer matches the NATS server's" + } + logf("re-minting %s: %s", cred.secretName, drift) + + creds, mintErr := apnats.MintUser(id, cred.grant) + if mintErr != nil { + return fmt.Errorf("re-mint %s nats creds: %w", cred.grant.Name, mintErr) + } + if existing.Data == nil { + existing.Data = map[string][]byte{} + } + existing.Data["nats.creds"] = []byte(creds) + existing.Data["ca.crt"] = caCrt + if err := c.Update(ctx, &existing); err != nil { + return fmt.Errorf("update %s secret: %w", cred.secretName, err) + } + // A running process read its creds file once, at connect time; the + // kubelet refreshing the mounted file changes nothing for it, so a + // re-mint must roll every consumer or the stale JWT stays on the wire. + return restartWorkloadsMountingSecret(ctx, c, key, logf) + + case apierrors.IsNotFound(err): + creds, mintErr := apnats.MintUser(id, cred.grant) + if mintErr != nil { + return fmt.Errorf("mint %s nats creds: %w", cred.grant.Name, mintErr) + } + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: cred.secretName, Namespace: TokenSecretNamespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{"nats.creds": []byte(creds), "ca.crt": caCrt}, + } + if err := c.Create(ctx, sec); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create %s secret: %w", cred.secretName, err) + } + return nil + + default: + return fmt.Errorf("get %s secret: %w", cred.secretName, err) + } +} + +// restartWorkloadsMountingSecret rolls every Deployment in the Secret's +// namespace whose pod template mounts it, so a rotated credential actually +// reaches the processes using it. Nothing to roll is the normal case on a first +// mint: nobody is running yet. +func restartWorkloadsMountingSecret(ctx context.Context, c client.Client, secret types.NamespacedName, logf func(string, ...any)) error { + var deps appsv1.DeploymentList + if err := c.List(ctx, &deps, client.InNamespace(secret.Namespace)); err != nil { + return fmt.Errorf("list deployments in %s to roll after rotating %s: %w", secret.Namespace, secret.Name, err) + } + stamp := time.Now().UTC().Format(time.RFC3339Nano) + for i := range deps.Items { + d := &deps.Items[i] + if !podSpecMountsSecret(&d.Spec.Template.Spec, secret.Name) { + continue + } + patch := []byte(fmt.Sprintf(`{"spec":{"template":{"metadata":{"annotations":{"ap.authzed.com/restartedAt":%q}}}}}`, stamp)) + if err := c.Patch(ctx, d, client.RawPatch(types.StrategicMergePatchType, patch)); err != nil { + return fmt.Errorf("rollout-restart %s/%s after rotating %s: %w", d.Namespace, d.Name, secret.Name, err) + } + logf("restarting %s to pick up the re-minted %s", d.Name, secret.Name) + } + return nil +} + +// podSpecMountsSecret reports whether the pod template reads the named Secret — +// as a volume (the creds-file shape) or through env/envFrom. +func podSpecMountsSecret(spec *corev1.PodSpec, name string) bool { + for _, v := range spec.Volumes { + if v.Secret != nil && v.Secret.SecretName == name { + return true + } + if v.Projected == nil { + continue + } + for _, src := range v.Projected.Sources { + if src.Secret != nil && src.Secret.Name == name { + return true + } + } + } + containers := make([]corev1.Container, 0, len(spec.Containers)+len(spec.InitContainers)) + containers = append(containers, spec.Containers...) + containers = append(containers, spec.InitContainers...) + for _, ctr := range containers { + for _, ef := range ctr.EnvFrom { + if ef.SecretRef != nil && ef.SecretRef.Name == name { + return true + } + } + for _, e := range ctr.Env { + if e.ValueFrom != nil && e.ValueFrom.SecretKeyRef != nil && e.ValueFrom.SecretKeyRef.Name == name { + return true + } + } + } + return false +} diff --git a/cmd/oap/internal/installcmd/nats_grant_test.go b/pkg/platform/bootstrap/nats_grant_test.go similarity index 94% rename from cmd/oap/internal/installcmd/nats_grant_test.go rename to pkg/platform/bootstrap/nats_grant_test.go index b8bb9d99..c59a94c7 100644 --- a/cmd/oap/internal/installcmd/nats_grant_test.go +++ b/pkg/platform/bootstrap/nats_grant_test.go @@ -1,4 +1,4 @@ -package installcmd +package bootstrap import ( "testing" @@ -26,11 +26,11 @@ import ( func TestWebdNATSGrantAllowsInboxReply(t *testing.T) { h := natstest.New(t) - assert.True(t, h.SubscribeAllowed(t, webdNATSGrant, apnats.InboxPrefixFor(webdNATSUser)+".abc.def"), + assert.True(t, h.SubscribeAllowed(t, WebdNATSGrant, apnats.InboxPrefixFor(webdNATSUser)+".abc.def"), "webd does NATS request-reply (view_message); its reply inbox must be subscribable or replies time out") - assert.True(t, h.SubscribeAllowed(t, webdNATSGrant, "ap.session.ns1.sess1.out.user_message"), + assert.True(t, h.SubscribeAllowed(t, WebdNATSGrant, "ap.session.ns1.sess1.out.user_message"), "webd must still subscribe to session out-subjects for the chat/live-view mirror") - assert.False(t, h.SubscribeAllowed(t, webdNATSGrant, "_INBOX.>"), + assert.False(t, h.SubscribeAllowed(t, WebdNATSGrant, "_INBOX.>"), "webd must NOT be able to read every principal's replies off the shared inbox root") } @@ -49,7 +49,7 @@ func TestWebdNATSGrantAllowsInboxReply(t *testing.T) { // // Each permitted subject below is a verified call site; each denied one is a // subject no webd code path publishes. If a new webd surface needs one, add it -// here and to webdNATSGrant together — a missing entry surfaces as a logged +// here and to WebdNATSGrant together — a missing entry surfaces as a logged // publish error, not silence. func TestWebdNATSGrantPublishSurface(t *testing.T) { h := natstest.New(t) @@ -79,7 +79,7 @@ func TestWebdNATSGrantPublishSurface(t *testing.T) { channelevents.WebhookInboundSubject: "pkg/web/webui/channelwebhook's verified-delivery handoff to channelsd", } for subject, why := range permitted { - assert.True(t, h.PublishAllowed(t, webdNATSGrant, subject), + assert.True(t, h.PublishAllowed(t, WebdNATSGrant, subject), "webd must be able to publish %s — %s", subject, why) } @@ -97,7 +97,7 @@ func TestWebdNATSGrantPublishSurface(t *testing.T) { "ap.channel.session_attached": "the only publisher is the outbound relay's patchOutputChannel, unreachable in webd (every builtin sender returns a zero SubChannelSendResult, so captured is empty and it returns early)", } for subject, why := range denied { - assert.False(t, h.PublishAllowed(t, webdNATSGrant, subject), + assert.False(t, h.PublishAllowed(t, WebdNATSGrant, subject), "webd must NOT be able to publish %s — %s", subject, why) } } diff --git a/cmd/oap/internal/installcmd/nats_test.go b/pkg/platform/bootstrap/nats_test.go similarity index 78% rename from cmd/oap/internal/installcmd/nats_test.go rename to pkg/platform/bootstrap/nats_test.go index 402e4bfb..7009bbc7 100644 --- a/cmd/oap/internal/installcmd/nats_test.go +++ b/pkg/platform/bootstrap/nats_test.go @@ -1,4 +1,4 @@ -package installcmd +package bootstrap import ( "bytes" @@ -12,15 +12,13 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/authzed/openagentprimitives/cmd/oap/internal/kube" ) func TestEnsureNATSIdentityIsIdempotent(t *testing.T) { ctx := context.Background() - c := fake.NewClientBuilder().WithScheme(kube.Scheme).Build() + c := fake.NewClientBuilder().Build() - require.NoError(t, ensureNATSIdentityWithClient(ctx, c), "first call must succeed") + require.NoError(t, EnsureNATSIdentity(ctx, c), "first call must succeed") var first corev1.Secret require.NoError(t, c.Get(ctx, types.NamespacedName{ @@ -36,7 +34,7 @@ func TestEnsureNATSIdentityIsIdempotent(t *testing.T) { assert.Contains(t, string(authConf), "resolver: MEMORY") assert.Contains(t, string(authConf), "resolver_preload") - require.NoError(t, ensureNATSIdentityWithClient(ctx, c), "second call must succeed") + require.NoError(t, EnsureNATSIdentity(ctx, c), "second call must succeed") var second corev1.Secret require.NoError(t, c.Get(ctx, types.NamespacedName{ @@ -49,9 +47,9 @@ func TestEnsureNATSIdentityIsIdempotent(t *testing.T) { func TestEnsureNATSTLSIsIdempotent(t *testing.T) { ctx := context.Background() - c := fake.NewClientBuilder().WithScheme(kube.Scheme).Build() + c := fake.NewClientBuilder().Build() - require.NoError(t, ensureNATSTLSWithClient(ctx, c), "first call must succeed") + require.NoError(t, EnsureNATSTLS(ctx, c), "first call must succeed") var first corev1.Secret require.NoError(t, c.Get(ctx, types.NamespacedName{ @@ -63,7 +61,7 @@ func TestEnsureNATSTLSIsIdempotent(t *testing.T) { } firstKey := first.Data["tls.key"] - require.NoError(t, ensureNATSTLSWithClient(ctx, c), "second call must succeed") + require.NoError(t, EnsureNATSTLS(ctx, c), "second call must succeed") var second corev1.Secret require.NoError(t, c.Get(ctx, types.NamespacedName{ @@ -76,12 +74,12 @@ func TestEnsureNATSTLSIsIdempotent(t *testing.T) { func TestEnsureNATSClientCredsMintsBothSecrets(t *testing.T) { ctx := context.Background() - c := fake.NewClientBuilder().WithScheme(kube.Scheme).Build() + c := fake.NewClientBuilder().Build() - require.NoError(t, ensureNATSIdentityWithClient(ctx, c), "seed identity") - require.NoError(t, ensureNATSTLSWithClient(ctx, c), "seed tls") + require.NoError(t, EnsureNATSIdentity(ctx, c), "seed identity") + require.NoError(t, EnsureNATSTLS(ctx, c), "seed tls") - require.NoError(t, ensureNATSClientCredsWithClient(ctx, c, nil), "mint client creds") + require.NoError(t, EnsureNATSClientCreds(ctx, c, nil), "mint client creds") for _, name := range []string{"spicebox-channelsd-nats-creds", "spicebox-cli-nats-creds", "spicebox-webd-nats-creds"} { var sec corev1.Secret @@ -108,16 +106,16 @@ func TestEnsureNATSClientCredsFailsWithoutPrerequisites(t *testing.T) { { name: "only identity seeded: TLS missing", setup: func(ctx context.Context, c client.Client) { - require.NoError(t, ensureNATSIdentityWithClient(ctx, c), "seed identity") + require.NoError(t, EnsureNATSIdentity(ctx, c), "seed identity") }, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() - c := fake.NewClientBuilder().WithScheme(kube.Scheme).Build() + c := fake.NewClientBuilder().Build() tc.setup(ctx, c) - err := ensureNATSClientCredsWithClient(ctx, c, nil) + err := EnsureNATSClientCreds(ctx, c, nil) require.Error(t, err, "must fail when prerequisites are not fully seeded") }) } @@ -125,12 +123,12 @@ func TestEnsureNATSClientCredsFailsWithoutPrerequisites(t *testing.T) { func TestEnsureNATSClientCredsIsIdempotent(t *testing.T) { ctx := context.Background() - c := fake.NewClientBuilder().WithScheme(kube.Scheme).Build() + c := fake.NewClientBuilder().Build() - require.NoError(t, ensureNATSIdentityWithClient(ctx, c), "seed identity") - require.NoError(t, ensureNATSTLSWithClient(ctx, c), "seed tls") + require.NoError(t, EnsureNATSIdentity(ctx, c), "seed identity") + require.NoError(t, EnsureNATSTLS(ctx, c), "seed tls") - require.NoError(t, ensureNATSClientCredsWithClient(ctx, c, nil), "first call must succeed") + require.NoError(t, EnsureNATSClientCreds(ctx, c, nil), "first call must succeed") // Capture creds bytes from both Secrets after the first call. var firstChannelsd, firstCLI corev1.Secret @@ -146,7 +144,7 @@ func TestEnsureNATSClientCredsIsIdempotent(t *testing.T) { require.NotEmpty(t, firstChannelsdCreds, "channelsd nats.creds must be populated") require.NotEmpty(t, firstCLICreds, "cli nats.creds must be populated") - require.NoError(t, ensureNATSClientCredsWithClient(ctx, c, nil), "second call must succeed") + require.NoError(t, EnsureNATSClientCreds(ctx, c, nil), "second call must succeed") // Assert that neither Secret was modified — stored creds are byte-identical. var secondChannelsd, secondCLI corev1.Secret diff --git a/cmd/oap/internal/installcmd/nats_upgrade_test.go b/pkg/platform/bootstrap/nats_upgrade_test.go similarity index 90% rename from cmd/oap/internal/installcmd/nats_upgrade_test.go rename to pkg/platform/bootstrap/nats_upgrade_test.go index 578a2cec..c88eca83 100644 --- a/cmd/oap/internal/installcmd/nats_upgrade_test.go +++ b/pkg/platform/bootstrap/nats_upgrade_test.go @@ -1,4 +1,4 @@ -package installcmd +package bootstrap import ( "context" @@ -15,7 +15,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "github.com/authzed/openagentprimitives/cmd/oap/internal/kube" "github.com/authzed/openagentprimitives/pkg/channels/channelevents" apnats "github.com/authzed/openagentprimitives/pkg/platform/nats" "github.com/authzed/openagentprimitives/pkg/platform/nats/natstest" @@ -24,17 +23,17 @@ import ( const natsSystemNS = "agentprimitives-system" // seededCluster returns a fake cluster with the NATS identity and TLS Secrets -// already written — the state ensureNATSClientCredsWithClient requires. +// already written — the state EnsureNATSClientCreds requires. func seededCluster(t *testing.T, objs ...client.Object) client.Client { t.Helper() - c := fake.NewClientBuilder().WithScheme(kube.Scheme).WithObjects(objs...).Build() + c := fake.NewClientBuilder().WithObjects(objs...).Build() ctx := context.Background() - require.NoError(t, ensureNATSIdentityWithClient(ctx, c), "seed identity") - require.NoError(t, ensureNATSTLSWithClient(ctx, c), "seed tls") + require.NoError(t, EnsureNATSIdentity(ctx, c), "seed identity") + require.NoError(t, EnsureNATSTLS(ctx, c), "seed tls") return c } -// storedIdentity reads back the account keys ensureNATSIdentityWithClient +// storedIdentity reads back the account keys EnsureNATSIdentity // wrote, so a test can mint an "older release's" credential signed by the same // account the installer will compare against. func storedIdentity(t *testing.T, c client.Client) *apnats.Identity { @@ -49,7 +48,7 @@ func storedIdentity(t *testing.T, c client.Client) *apnats.Identity { } } -// currentNATSCA is the CA ensureNATSTLSWithClient just wrote — what a healthy +// currentNATSCA is the CA EnsureNATSTLS just wrote — what a healthy // creds Secret carries, so a test can isolate grant drift from CA drift. func currentNATSCA(t *testing.T, c client.Client) []byte { t.Helper() @@ -75,8 +74,8 @@ func natsCreds(t *testing.T, c client.Client, name string) string { // Derived by subtraction from the live grant rather than transcribed, so it // stays a faithful "one release behind" as the real grant grows. func webdGrantBeforeWebhookInbound() apnats.UserGrant { - old := apnats.UserGrant{Name: webdNATSGrant.Name, SubAllow: webdNATSGrant.SubAllow} - for _, s := range webdNATSGrant.PubAllow { + old := apnats.UserGrant{Name: WebdNATSGrant.Name, SubAllow: WebdNATSGrant.SubAllow} + for _, s := range WebdNATSGrant.PubAllow { if s == channelevents.WebhookInboundSubject { continue } @@ -128,7 +127,7 @@ func restartedAt(t *testing.T, c client.Client, name string) string { // TestEnsureNATSClientCreds_ReMintsAGrantThatGainedASubject is the upgrade // this blocker is about. // -// A cluster installed before ap.channel.webhook_inbound joined webdNATSGrant +// A cluster installed before ap.channel.webhook_inbound joined WebdNATSGrant // carries a webd JWT without it. `oap install` used to see the Secret, return // early, and leave the stale JWT in place forever — and the resulting failure // is invisible: nats-server reports a publish permission violation @@ -166,7 +165,7 @@ func TestEnsureNATSClientCreds_ReMintsAGrantThatGainedASubject(t *testing.T) { "precondition: the pre-upgrade credential must NOT permit the webhook-inbound publish, or this test proves nothing") var narrated []string - require.NoError(t, ensureNATSClientCredsWithClient(ctx, c, func(f string, a ...any) { + require.NoError(t, EnsureNATSClientCreds(ctx, c, func(f string, a ...any) { narrated = append(narrated, fmt.Sprintf(f, a...)) }), "the upgrade run must succeed") @@ -201,12 +200,12 @@ func TestEnsureNATSClientCreds_LeavesAnUpToDateCredentialAlone(t *testing.T) { ctx := context.Background() c := seededCluster(t, webdDeployment("spicebox-webd-nats-creds")) - require.NoError(t, ensureNATSClientCredsWithClient(ctx, c, nil), "first install") + require.NoError(t, EnsureNATSClientCreds(ctx, c, nil), "first install") before := natsCreds(t, c, "spicebox-webd-nats-creds") require.NotEmpty(t, before) var narrated []string - require.NoError(t, ensureNATSClientCredsWithClient(ctx, c, func(f string, a ...any) { + require.NoError(t, EnsureNATSClientCreds(ctx, c, func(f string, a ...any) { narrated = append(narrated, fmt.Sprintf(f, a...)) }), "second install") @@ -224,7 +223,7 @@ func TestEnsureNATSClientCreds_RefreshesAStaleCA(t *testing.T) { ctx := context.Background() c := seededCluster(t, webdDeployment("spicebox-cli-nats-creds")) - require.NoError(t, ensureNATSClientCredsWithClient(ctx, c, nil), "first install") + require.NoError(t, EnsureNATSClientCreds(ctx, c, nil), "first install") creds := natsCreds(t, c, "spicebox-cli-nats-creds") var sec corev1.Secret @@ -233,7 +232,7 @@ func TestEnsureNATSClientCreds_RefreshesAStaleCA(t *testing.T) { sec.Data["ca.crt"] = []byte("a CA from a previous NATS install") require.NoError(t, c.Update(ctx, &sec)) - require.NoError(t, ensureNATSClientCredsWithClient(ctx, c, nil), "second install") + require.NoError(t, EnsureNATSClientCreds(ctx, c, nil), "second install") require.NoError(t, c.Get(ctx, key, &sec)) var tls corev1.Secret diff --git a/pkg/platform/bootstrap/runnable.go b/pkg/platform/bootstrap/runnable.go new file mode 100644 index 00000000..6cf39f61 --- /dev/null +++ b/pkg/platform/bootstrap/runnable.go @@ -0,0 +1,28 @@ +package bootstrap + +import ( + "context" + "fmt" +) + +// SchemaWriter is the subset of spicedb.SchemaIOAdapter the bootstrap needs. +type SchemaWriter interface { + WriteSchema(ctx context.Context, text string) error +} + +// RunBootstrap is the operator's leader-only bootstrap step. It applies the +// SpiceDB base schema idempotently (WriteSchema is a full-schema replace). +// +// It deliberately does NOT mint Secrets: the internal Secrets are minted by the +// pre-start bootstrap Job (MintAllInternalSecrets), which runs BEFORE the +// operator so every consumer mounts its Secret cleanly at first boot — no +// mint-after-start, no pod rolls. Schema stays here because it is the one step +// that needs SpiceDB already running (the Job runs before SpiceDB is up). +// +// Safe to run on every leader acquisition: WriteSchema is idempotent. +func RunBootstrap(ctx context.Context, schemaIO SchemaWriter, schemaText string) error { + if err := schemaIO.WriteSchema(ctx, schemaText); err != nil { + return fmt.Errorf("apply spicedb schema: %w", err) + } + return nil +} diff --git a/pkg/platform/bootstrap/runnable_envtest_test.go b/pkg/platform/bootstrap/runnable_envtest_test.go new file mode 100644 index 00000000..cbb13c1a --- /dev/null +++ b/pkg/platform/bootstrap/runnable_envtest_test.go @@ -0,0 +1,62 @@ +//go:build integration + +package bootstrap_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" + "github.com/authzed/openagentprimitives/pkg/controllers/testenv" + "github.com/authzed/openagentprimitives/pkg/platform/bootstrap" +) + +func TestMintAllInternalSecretsEnvtest_MintsOnceIdempotent(t *testing.T) { + env := testenv.Shared(t) + ctx := context.Background() + + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: bootstrap.TokenSecretNamespace}} + if err := env.Client.Create(ctx, ns); err != nil && !apierrors.IsAlreadyExists(err) { + require.NoError(t, err, "create namespace") + } + + require.NoError(t, bootstrap.MintAllInternalSecrets(ctx, env.Client, env.Client, bootstrap.MintOptions{})) + + want := append(append([]string{}, bootstrap.MemoryTokenSecretNames...), + bootstrap.AdmindTokenSecretName, v1alpha1.PassthroughLinkSigningKeySecret, + "spicebox-spicedb-token", "spicebox-postgres-token") + for _, n := range want { + var s corev1.Secret + require.NoError(t, env.Client.Get(ctx, types.NamespacedName{Namespace: bootstrap.TokenSecretNamespace, Name: n}, &s), "minted %s", n) + } + + // Second run is idempotent — no rotation of the spicedb token. + var sp1 corev1.Secret + require.NoError(t, env.Client.Get(ctx, types.NamespacedName{Namespace: bootstrap.TokenSecretNamespace, Name: "spicebox-spicedb-token"}, &sp1)) + require.NoError(t, bootstrap.MintAllInternalSecrets(ctx, env.Client, env.Client, bootstrap.MintOptions{})) + var sp2 corev1.Secret + require.NoError(t, env.Client.Get(ctx, types.NamespacedName{Namespace: bootstrap.TokenSecretNamespace, Name: "spicebox-spicedb-token"}, &sp2)) + assert.Equal(t, string(sp1.Data["token"]), string(sp2.Data["token"]), "spicedb token not rotated on re-run") +} + +func TestMintAllInternalSecretsEnvtest_SkipsExternal(t *testing.T) { + env := testenv.Shared(t) + ctx := context.Background() + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: bootstrap.TokenSecretNamespace}} + if err := env.Client.Create(ctx, ns); err != nil && !apierrors.IsAlreadyExists(err) { + require.NoError(t, err) + } + + require.NoError(t, bootstrap.MintAllInternalSecrets(ctx, env.Client, env.Client, bootstrap.MintOptions{SkipSpiceDBToken: true, SkipPostgresToken: true})) + + var sp corev1.Secret + err := env.Client.Get(ctx, types.NamespacedName{Namespace: bootstrap.TokenSecretNamespace, Name: "spicebox-spicedb-token"}, &sp) + assert.True(t, apierrors.IsNotFound(err), "spicedb token not minted when external") +} diff --git a/pkg/platform/bootstrap/runnable_test.go b/pkg/platform/bootstrap/runnable_test.go new file mode 100644 index 00000000..945cac80 --- /dev/null +++ b/pkg/platform/bootstrap/runnable_test.go @@ -0,0 +1,28 @@ +package bootstrap + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type recordingSchema struct { + calls int + last string +} + +func (r *recordingSchema) WriteSchema(_ context.Context, text string) error { + r.calls++ + r.last = text + return nil +} + +func TestRunBootstrap_AppliesSchemaIdempotently(t *testing.T) { + sch := &recordingSchema{} + require.NoError(t, RunBootstrap(context.Background(), sch, "definition user {}")) + require.NoError(t, RunBootstrap(context.Background(), sch, "definition user {}")) + assert.Equal(t, 2, sch.calls, "WriteSchema runs each invocation (it is idempotent)") + assert.Equal(t, "definition user {}", sch.last) +} diff --git a/pkg/platform/bootstrap/sharedenv_test.go b/pkg/platform/bootstrap/sharedenv_test.go new file mode 100644 index 00000000..9cce35ec --- /dev/null +++ b/pkg/platform/bootstrap/sharedenv_test.go @@ -0,0 +1,12 @@ +//go:build integration + +package bootstrap_test + +import ( + "os" + "testing" + + "github.com/authzed/openagentprimitives/pkg/controllers/testenv" +) + +func TestMain(m *testing.M) { os.Exit(testenv.RunPackage(m)) } diff --git a/pkg/platform/bootstrap/token.go b/pkg/platform/bootstrap/token.go new file mode 100644 index 00000000..f9e3efe4 --- /dev/null +++ b/pkg/platform/bootstrap/token.go @@ -0,0 +1,71 @@ +// Package bootstrap holds the once-only, against-empty-state install steps that +// must run in exactly one place. Minting the memory-token Secrets lives here so +// both oap install's fast path and the operator's leader-only bootstrap Runnable +// invoke the SAME idempotent logic — the token is write-once and never rotated, +// so two Create-if-absent callers cannot drift. +package bootstrap + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/authzed/openagentprimitives/pkg/authz/adoptguard" +) + +const ( + // TokenSecretNamespace is the namespace every bootstrap token Secret lives in. + TokenSecretNamespace = "agentprimitives-system" + // TokenSecretKey is the single data key each bootstrap token Secret carries. + TokenSecretKey = "token" +) + +// MemoryTokenSecretNames is the canonical set of per-component memory-token +// Secrets the bundle requires. Kept in sync with install.go's +// operatorMemoryTokenSecrets and secret_bootstrap_test.go's installTimeSecrets. +var MemoryTokenSecretNames = []string{ + "spicebox-channelsd-memory-token", + "spicebox-webd-memory-token", + "agentprimitives-authzd-memory-token", +} + +// EnsureTokenSecret creates a `token`-keyed random-bearer Secret if absent, and +// is a no-op (created=false) if it already exists — it never rotates. reader +// must be a live/uncached reader (the operator's Secret cache is label-filtered +// and would hide an as-yet-unlabeled Secret). AlreadyExists from Create — a +// leader-failover race, or a cache miss on an existing Secret — is tolerated and +// reported as created=false. +func EnsureTokenSecret(ctx context.Context, reader client.Reader, writer client.Writer, namespace, name string) (created bool, err error) { + var existing corev1.Secret + switch err := reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &existing); { + case err == nil: + return false, nil + case !apierrors.IsNotFound(err): + return false, fmt.Errorf("get %s secret: %w", name, err) + } + + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return false, fmt.Errorf("generate token: %w", err) + } + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{TokenSecretKey: []byte(hex.EncodeToString(buf))}, + } + adoptguard.WithAdoptedLabel(sec) + if err := writer.Create(ctx, sec); err != nil { + if apierrors.IsAlreadyExists(err) { + return false, nil + } + return false, fmt.Errorf("create %s secret: %w", name, err) + } + return true, nil +} diff --git a/pkg/platform/bootstrap/token_test.go b/pkg/platform/bootstrap/token_test.go new file mode 100644 index 00000000..0dd880ea --- /dev/null +++ b/pkg/platform/bootstrap/token_test.go @@ -0,0 +1,49 @@ +package bootstrap + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/authzed/openagentprimitives/pkg/authz/adoptguard" +) + +func newFakeClient(objs ...client.Object) client.Client { + return fake.NewClientBuilder().WithObjects(objs...).Build() +} + +func TestEnsureTokenSecret_CreatesWhenAbsent(t *testing.T) { + c := newFakeClient() + created, err := EnsureTokenSecret(context.Background(), c, c, TokenSecretNamespace, "spicebox-webd-memory-token") + require.NoError(t, err) + assert.True(t, created, "a fresh mint reports created=true") + + var got corev1.Secret + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: TokenSecretNamespace, Name: "spicebox-webd-memory-token"}, &got)) + assert.Len(t, string(got.Data[TokenSecretKey]), 64, "token is 64 hex chars") + assert.Equal(t, corev1.SecretTypeOpaque, got.Type) + assert.Contains(t, got.Labels, adoptguard.AdoptedLabel, "minted Secret carries the adopted label") +} + +func TestEnsureTokenSecret_IdempotentNoRotate(t *testing.T) { + existing := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "spicebox-webd-memory-token", Namespace: TokenSecretNamespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{TokenSecretKey: []byte("preexisting-value")}, + } + c := newFakeClient(existing) + created, err := EnsureTokenSecret(context.Background(), c, c, TokenSecretNamespace, "spicebox-webd-memory-token") + require.NoError(t, err) + assert.False(t, created, "an existing Secret reports created=false") + + var got corev1.Secret + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: TokenSecretNamespace, Name: "spicebox-webd-memory-token"}, &got)) + assert.Equal(t, "preexisting-value", string(got.Data[TokenSecretKey]), "existing token is never rotated") +} diff --git a/pkg/platform/manifests/install.yaml b/pkg/platform/manifests/install.yaml index 2d06755c..4f63b66b 100644 --- a/pkg/platform/manifests/install.yaml +++ b/pkg/platform/manifests/install.yaml @@ -25848,6 +25848,13 @@ rules: - apps resources: - deployments + verbs: + - get + - list + - patch +- apiGroups: + - apps + resources: - statefulsets verbs: - get diff --git a/pkg/platform/manifests/rbac_sufficiency_test.go b/pkg/platform/manifests/rbac_sufficiency_test.go index a3f11938..d3a2967c 100644 --- a/pkg/platform/manifests/rbac_sufficiency_test.go +++ b/pkg/platform/manifests/rbac_sufficiency_test.go @@ -576,8 +576,11 @@ func operatorComponent() component { })...) // admind's cluster-health snapshot reads component Deployments/StatefulSets // (replica/ready counts) for the admin Overview — read-only, no watch. + // deployments also gets `patch`: the bootstrap Job's NATS-creds re-mint + // (pkg/platform/bootstrap, via the operator SA) rolls consumer Deployments + // when a grant gains a subject on upgrade. required = append(required, reqs("apps", map[string][]string{ - "deployments": {"get", "list"}, + "deployments": {"get", "list", "patch"}, "statefulsets": {"get", "list"}, })...) // The same health snapshot sums live PodMetrics (metrics.k8s.io) for the diff --git a/pkg/platform/manifests/secret_bootstrap_test.go b/pkg/platform/manifests/secret_bootstrap_test.go index 3d385040..8cea2a0a 100644 --- a/pkg/platform/manifests/secret_bootstrap_test.go +++ b/pkg/platform/manifests/secret_bootstrap_test.go @@ -50,10 +50,10 @@ var installTimeSecrets = map[string]string{ "spicebox-channelsd-memory-token": "ensureChannelsdMemoryToken (init.go)", "spicebox-webd-memory-token": "ensureWebdMemoryToken (init.go)", "agentprimitives-authzd-memory-token": "ensureAuthzdMemoryToken (init.go)", - "spicebox-spicedb-token": "ensureSpiceDBToken (install.go)", - "spicebox-channelsd-nats-creds": "ensureNATSClientCreds (install.go)", - "spicebox-webd-nats-creds": "ensureNATSClientCreds (install.go)", - "spicebox-passthrough-link-key": "ensurePassthroughLinkSigningKey (install.go)", + "spicebox-spicedb-token": "EnsureSpiceDBToken (mint.go)", + "spicebox-channelsd-nats-creds": "EnsureNATSClientCreds (nats.go)", + "spicebox-webd-nats-creds": "EnsureNATSClientCreds (nats.go)", + "spicebox-passthrough-link-key": "EnsurePassthroughLinkKey (appsecrets.go)", "spicebox-webhook-tls": "ensureWebhookTLSWithClient (install.go)", // No row for spicebox-anthropic-api-key: nothing in the repo creates it. // Every reference to it (authzd's env, channelsd's volume) is optional:true, @@ -141,6 +141,9 @@ func TestInstallTimeSecretsMapIsLoadBearing(t *testing.T) { src := newGoIndex(t, filepath.Join("..", "..", "..", "cmd", "oap"), filepath.Join("..", "..", "apis", "v1alpha1"), + // NATS creds minting moved to the shared bootstrap package (the pre-start + // Job mints them); the creator resolver must scan it too. + filepath.Join("..", "bootstrap"), ) referenced := allReferencedSecretNames(t) @@ -294,7 +297,10 @@ func newGoIndex(t *testing.T, funcDir string, constDirs ...string) goIndex { ix := goIndex{funcs: map[string]*ast.FuncDecl{}, consts: map[string]string{}} fset := token.NewFileSet() for _, root := range append([]string{funcDir}, constDirs...) { - collectFuncs := root == funcDir + // Creators now live in more than one root (cmd/oap AND the shared + // pkg/platform/bootstrap), so index functions from every root, not just + // funcDir. Constant collection already spans all roots below. + collectFuncs := true err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return err diff --git a/site/content/docs/cli-reference.mdx b/site/content/docs/cli-reference.mdx index 842c58f5..db1bf843 100644 --- a/site/content/docs/cli-reference.mdx +++ b/site/content/docs/cli-reference.mdx @@ -38,6 +38,7 @@ from the CLI, so it matches the binary you have. Pick a family for its subcomman | [`oap plangate`](/docs/oap-plangate) | Inspect what the plan gate recorded for a session | | [`oap platform`](/docs/oap-platform) | Manage platform-level admin access (the admin UI gate) | | [`oap preferences`](/docs/oap-preferences) | Inspect a session's resolved per-user preferences | +| [`oap render`](/docs/oap-render) | Render the install as a GitOps-ready kustomize overlay (no cluster mutation) | | [`oap sandbox`](/docs/oap-sandbox) | Inspect SpiceboxSession resources (per-bundle running sandbox pods) | | [`oap session`](/docs/oap-session) | Inspect AgentSession resources | | [`oap settings`](/docs/oap-settings) | Manage cluster-wide agent settings (security defaults wizard + apply). | diff --git a/site/content/docs/gitops.mdx b/site/content/docs/gitops.mdx new file mode 100644 index 00000000..2e25f372 --- /dev/null +++ b/site/content/docs/gitops.mdx @@ -0,0 +1,134 @@ +export const meta = { + title: 'GitOps with Argo CD / Flux', + section: 'Guides', + group: 'Operations', + order: 130, + description: 'Render the install as a kustomize overlay, commit it, and let Argo CD or Flux reconcile the cluster — instead of oap install mutating it directly.', +} + +# GitOps with Argo CD / Flux + +`oap install` brings a cluster up imperatively. If instead you run GitOps — the +desired state lives in git and a controller (Argo CD or Flux) reconciles the +cluster toward it — use `oap render` to produce a committable kustomize overlay, +and `oap check` to verify a synced cluster. + +## Render the overlay + +```bash +oap render --cluster-kind gke --output-dir deploy/oap \ + --image-registry \ + --with-postgres +``` + +`oap render` never mutates a cluster. Its only remote call is read-only image +digest resolution (skip it with `--no-digest-pin`). It writes a self-contained +overlay: + +``` +deploy/oap/ + kustomization.yaml # resources + patches + components + base/ + install.yaml # the core bundle (operator, webd, authzd, spicedb-operator, CRDs) + channelsd.yaml # channelsd (applied separately by oap install; included here) + nats.yaml # nats + patches/ + cluster-kind.yaml # AP_CLUSTER_KIND stamped on operator + webd + components/ # postgres — only when --with-postgres is set + routing.yaml # Gateway + HTTPRoute(s) — only when --trusted-hostname is set +``` + +Only `--with-postgres` is offered today: its token is minted by the overlay, so +the component works end to end. The Graphiti/Neo4j memory backends need a +user-supplied LLM API key the overlay cannot mint, so their GitOps support is a +follow-up; use `oap install` for them in the meantime. + +The output is a pure function of the flags: re-rendering identical inputs is +byte-identical, so Argo CD shows no phantom drift. Commit the directory and +point an Argo CD `Application` (or a Flux `Kustomization`) at it. + +`oap render` refuses a memory-datastore cluster kind (`local` / `desktop`): +those run an ephemeral, single-replica SpiceDB that seeds its schema at startup, +which is fatal against the durable, multi-replica datastore a GitOps install +targets. + +## Secrets + +Agent Primitives' secrets are **app-internal** — none are API keys you supply. +The overlay includes a **pre-start bootstrap Job** (named +`spicebox-bootstrap-secrets-`, running the operator image's +`bootstrap-secrets` entrypoint) that mints them idempotently before the operator +and data-plane start, so every component mounts its Secret cleanly at first boot. +Nothing secret lands in git, and nothing is ever rotated. The Job's name carries +a hash of its pod template, so an operator-image upgrade produces a new Job +object (the controller prunes the old one) rather than failing against the +immutable pod template of the existing Job. + +The Job mints: the three memory tokens, the admind token, the passthrough signing +key, the NATS identity/TLS/client credentials, and — unless you bring your own +instance — `spicebox-spicedb-token` and `spicebox-postgres-token`. + +The operator's webhook serving cert (`spicebox-webhook-tls`) is provisioned by an +emitted **cert-manager** `Certificate` (a self-signed `Issuer` + the ca-injector +annotation on the webhook configuration). This makes **cert-manager a dependency** +of the rendered bundle. On a cluster without it, pass `--external-webhook-tls` and +provide `spicebox-webhook-tls` + the webhook `caBundle` yourself. + +- **External SpiceDB / Postgres:** pass `--external-spicedb` / `--external-postgres` + so the Job does **not** mint that token; you provide it (e.g. via the External + Secrets Operator or a sealed secret). +- **Link an existing memory token:** use the per-component + `--channelsd-/--webd-/--authzd-memory-token-secret` flags (one Secret each). The + webd token **must name a different Secret** than the channelsd/authzd tokens: + webd is browser-facing and its token is read-only, while the channelsd/authzd + tokens can write the audit log — sharing one value would hand webd that write + capability, so render refuses it. +- **Nothing else to provide by default:** every internal Secret is provisioned by + the overlay (the Job + the cert-manager webhook Certificate). `oap render + --validate` reports nothing to provide unless you opted an instance external via + `--external-spicedb`, `--external-postgres`, or `--external-webhook-tls` — then + it lists exactly those. + +## Validate before you commit + +```bash +oap render --cluster-kind gke --output-dir deploy/oap --validate +``` + +`--validate` builds the overlay in-process and checks that it will sync: every +non-optional Secret reference has a creator (shipped, operator-minted, or +linked), `AP_CLUSTER_KIND` is stamped, and first-party images are pinned. It +prints the app-internal secrets you still need to provide. It cannot verify that +a *linked* or externally-provided Secret actually exists in the cluster — that is +a cluster-state fact `oap check` reports after the sync. + +## Verify the synced cluster + +After Argo CD / Flux reports synced, confirm the install is healthy with the +same command the imperative path uses: + +```bash +oap check +``` + +`oap check` is provenance-agnostic — it verifies live cluster state by name, so +it works identically whether the cluster came up via `oap install` or a GitOps +sync. It includes a check that the memory-token Secrets are present; their +absence is the signal that the bootstrap Job never minted them (wrong image, its +backoff was exhausted, or the operator ServiceAccount is missing Secret RBAC). + +## First-sync notes + +- The **bootstrap Job mints every internal Secret before the operator and + data-plane start** — there is no first-sync operator mint-and-roll step. The + Job and the workloads are applied together; the Job retries (its + `backoffLimit`) until the namespace and operator ServiceAccount it depends on + exist, and the operator crash-loops until the Secrets it mounts at startup + appear. Both are designed-for, self-healing cold-start states, not drift. +- During that window the workloads whose token mounts are required sit in + `CreateContainerConfigError` — expected, and cleared the moment the Job + completes. No pod-template annotation is written, so after the Job finishes the + bundle is at rest and Argo CD / Flux report `Synced` with no phantom drift. +- The one roll that can happen later is unrelated to first sync: if the webd + NATS publish-grant drifts, the operator re-mints the client credentials and + rolls the mounting Deployments once to remount them. diff --git a/site/content/docs/oap-render.mdx b/site/content/docs/oap-render.mdx new file mode 100644 index 00000000..7c90e378 --- /dev/null +++ b/site/content/docs/oap-render.mdx @@ -0,0 +1,14 @@ +export const meta = { + title: 'oap render', + section: 'Reference', + group: 'CLI', + order: 2940, + description: 'Render the install as a GitOps-ready kustomize overlay (no cluster mutation)', +} + +# oap render + +Render the Agent Primitives install as a self-contained kustomize overlay for Argo CD / Flux. + +Unlike `oap install`, render never mutates a cluster (its only remote call is read-only image-digest resolution, skipped with --no-digest-pin). Commit the output directory and point your GitOps controller at it. The overlay provisions every internal Secret: a pre-start Job mints the tokens/NATS material before the operator starts, and a cert-manager Certificate populates the webhook cert (so cert-manager is a dependency). Use --external-spicedb/--external-postgres/--external-webhook-tls to provide any of those yourself, or link an existing memory token with the per-component --*-memory-token-secret flags. Verify a synced cluster with `oap check`. + diff --git a/site/content/docs/oap-sandbox.mdx b/site/content/docs/oap-sandbox.mdx index ffc4534b..5026bdd9 100644 --- a/site/content/docs/oap-sandbox.mdx +++ b/site/content/docs/oap-sandbox.mdx @@ -2,7 +2,7 @@ export const meta = { title: 'oap sandbox', section: 'Reference', group: 'CLI', - order: 2940, + order: 2950, description: 'Inspect SpiceboxSession resources (per-bundle running sandbox pods)', } diff --git a/site/content/docs/oap-session.mdx b/site/content/docs/oap-session.mdx index c4c8ca22..3dcc70e4 100644 --- a/site/content/docs/oap-session.mdx +++ b/site/content/docs/oap-session.mdx @@ -2,7 +2,7 @@ export const meta = { title: 'oap session', section: 'Reference', group: 'CLI', - order: 2950, + order: 2960, description: 'Inspect AgentSession resources', } diff --git a/site/content/docs/oap-settings.mdx b/site/content/docs/oap-settings.mdx index 2ad1b7a4..108ed715 100644 --- a/site/content/docs/oap-settings.mdx +++ b/site/content/docs/oap-settings.mdx @@ -2,7 +2,7 @@ export const meta = { title: 'oap settings', section: 'Reference', group: 'CLI', - order: 2960, + order: 2970, description: 'Manage cluster-wide agent settings (security defaults wizard + apply).', } diff --git a/site/content/docs/oap-skill.mdx b/site/content/docs/oap-skill.mdx index 6030c452..bec8e16c 100644 --- a/site/content/docs/oap-skill.mdx +++ b/site/content/docs/oap-skill.mdx @@ -2,7 +2,7 @@ export const meta = { title: 'oap skill', section: 'Reference', group: 'CLI', - order: 2970, + order: 2980, description: 'List, view, create, and delete agent Skills and SkillSources', } diff --git a/site/content/docs/oap-spicedb.mdx b/site/content/docs/oap-spicedb.mdx index 51c82228..2e804555 100644 --- a/site/content/docs/oap-spicedb.mdx +++ b/site/content/docs/oap-spicedb.mdx @@ -2,7 +2,7 @@ export const meta = { title: 'oap spicedb', section: 'Reference', group: 'CLI', - order: 2980, + order: 2990, description: 'Apply schema and check permissions against the system SpiceDB.', } diff --git a/site/content/docs/oap-tools.mdx b/site/content/docs/oap-tools.mdx index a334d5ef..2da81d14 100644 --- a/site/content/docs/oap-tools.mdx +++ b/site/content/docs/oap-tools.mdx @@ -2,7 +2,7 @@ export const meta = { title: 'oap tools', section: 'Reference', group: 'CLI', - order: 2990, + order: 3000, description: 'Manage tools used by agents (kind-agnostic)', } diff --git a/site/content/docs/oap-user-identity.mdx b/site/content/docs/oap-user-identity.mdx index 16c58a05..f1cc4f12 100644 --- a/site/content/docs/oap-user-identity.mdx +++ b/site/content/docs/oap-user-identity.mdx @@ -2,7 +2,7 @@ export const meta = { title: 'oap user-identity', section: 'Reference', group: 'CLI', - order: 3000, + order: 3010, description: 'Manage UserIdentity resources (per-user credential catalogs)', }