From ac42a64463ba8fc49ba1e3a798b457aae1b79afa Mon Sep 17 00:00:00 2001 From: Andrew McCabe Date: Tue, 22 Sep 2026 21:25:40 -0400 Subject: [PATCH 1/3] feat(flagsmith): deploy, seed, and validate self-hosted Flagsmith Flagsmith joins the stack as infrastructure: deployed by Flux from k8s/infra/flagsmith like the monitoring stack, seeded by setup.sh like Forgejo, checked by validate-stack.sh, and documented with the rest. It is inert at baseline. No flags are defined and no service reads one; later changes wire the SDKs and add scenarios. The manifests are a namespace, a one-replica Deployment of the unified image pinned by tag and digest, a Service, an Ingress on flagsmith.localhost, a demo-only Secret, a migrate-flagsmith-v1 Job, and a ServiceMonitor. The container runs serve, so migrations run only in the Job, and readiness fails until the Job has applied the schema. Gunicorn runs one worker. There is no CPU limit, like every other Deployment here. The flagsmith database joins orders in the PostgreSQL initdb ConfigMap. The Flux Kustomization depends on apps, which covers both the operator CRDs and the database. Two Flagsmith defaults that reach the internet are off. ENABLE_TELEMETRY posts object counts to api.flagsmith.com from Django's post_migrate signal. The dashboard fetches its own feature flags from Flagsmith's hosted service on every page load: the two settings default to None, the view that builds the frontend config omits a None, and the self-hosted bundle hardcodes a key and edge.api.flagsmith.com, so empty strings replace both. Everything else it could call out to is already off at its default. setup.sh seeds a bootstrap superuser through the signup endpoint, an organization and project named example-stack, and exactly the staging and production environments, with --flagsmith-password for the account. Later runs log in with the password rather than resetting it. The admin token and the two environment keys go into a flagsmith-bootstrap Secret, and the Forgejo token setup.sh already mints goes into forgejo-bootstrap, which the scenario scripts now reuse instead of minting one per run. Both are bootstrap state Flux does not own and does not prune. Every host port the stack opens is bound to 127.0.0.1. k3d's ingress and API ports and the registry port were published with no host address, and the podman machine's forwarder put them on every interface of the host, so a running cluster was reachable from whatever network the machine was on with the demo credentials this repository publishes. validate-stack.sh now asks the routing table for the machine's own address and requires each port to refuse there while answering on 127.0.0.1. It also checks the Flagsmith pod, liveness through the ingress, both seeded environments against their keys, the flagsmith Prometheus target, the flagsmith and monitoring-flux Kustomizations, and the migration Job. Two checks that appended a second 000 to curl's own now read curl's value. The proto code is regenerated for the protoc-gen-go go.mod pins; the diff is the two header lines. The pull request template uses the three headings every pull request here has actually been written under. Assisted-by: Claude:claude-opus-5 [claude-code] Assisted-by: Claude:claude-fable-5-1 [claude-code] Signed-off-by: Andrew McCabe --- .github/PULL_REQUEST_TEMPLATE.md | 8 +- AGENTS.md | 2 +- README.md | 51 +++- docs/scenarios.md | 9 +- docs/services.md | 59 +++- docs/topology.md | 70 +++-- go-grpc/proto/echopb/echo.pb.go | 2 +- go-grpc/proto/pricingpb/pricing.pb.go | 2 +- k8s/apps/base/landing.yaml | 1 + k8s/apps/base/postgres.yaml | 1 + k8s/infra/flagsmith/deployment.yaml | 131 +++++++++ k8s/infra/flagsmith/ingress.yaml | 18 ++ k8s/infra/flagsmith/kustomization.yaml | 10 + k8s/infra/flagsmith/migrate.yaml | 45 +++ k8s/infra/flagsmith/namespace.yaml | 4 + k8s/infra/flagsmith/secret.yaml | 14 + k8s/infra/flagsmith/service.yaml | 20 ++ k8s/infra/flagsmith/servicemonitor.yaml | 15 + k8s/infra/flux/kustomization-flagsmith.yaml | 21 ++ k8s/infra/flux/kustomization.yaml | 1 + scenarios/lib.sh | 42 ++- scripts/build.sh | 4 +- scripts/setup.sh | 305 +++++++++++++++++++- scripts/validate-stack.sh | 113 +++++++- 24 files changed, 873 insertions(+), 75 deletions(-) create mode 100644 k8s/infra/flagsmith/deployment.yaml create mode 100644 k8s/infra/flagsmith/ingress.yaml create mode 100644 k8s/infra/flagsmith/kustomization.yaml create mode 100644 k8s/infra/flagsmith/migrate.yaml create mode 100644 k8s/infra/flagsmith/namespace.yaml create mode 100644 k8s/infra/flagsmith/secret.yaml create mode 100644 k8s/infra/flagsmith/service.yaml create mode 100644 k8s/infra/flagsmith/servicemonitor.yaml create mode 100644 k8s/infra/flux/kustomization-flagsmith.yaml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9140588..428dcbd 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,11 @@ -## Description +## What changes -## Testing +## How it was tested -## Review +## Security review - + diff --git a/AGENTS.md b/AGENTS.md index f0ef351..614f175 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # Agent guide -This repository is a small distributed system on a local k3d cluster that breaks in realistic ways on demand. Go and Rust services live in `go-api/`, `go-grpc/`, `rust-inventory/`, and `migrate/` as one Go module plus one cargo crate; `k8s/` holds the manifests (`apps/base` for the services, `infra/` for Forgejo, Flux, and monitoring); `scenarios/` holds the failure scenarios and their kustomize overlays; `scripts/` holds bring-up, teardown, build, smoke, and validation; `docs/` holds the topology, service, and scenario pages. +This repository is a small distributed system on a local k3d cluster that breaks in realistic ways on demand. Go and Rust services live in `go-api/`, `go-grpc/`, `rust-inventory/`, and `migrate/` as one Go module plus one cargo crate; `k8s/` holds the manifests (`apps/base` for the services, `infra/` for Forgejo, Flux, monitoring, and Flagsmith); `scenarios/` holds the failure scenarios and their kustomize overlays; `scripts/` holds bring-up, teardown, build, smoke, and validation; `docs/` holds the topology, service, and scenario pages. ## Build and test diff --git a/README.md b/README.md index 7f9906d..0dc77c6 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,10 @@ is self-contained after startup (see [Prerequisites](#prerequisites) and - node-exporter - Pre-configured with per-service dashboards, recording rules for error rate and latency, and alerting rules. +- Flagsmith, self-hosted, for feature flags + - Seeded with an `example-stack` organization and project and two + environments, `staging` and `production` + - No flags are defined and no service reads one yet - A local image registry that the cluster pulls from. - Three failure scenarios that break the stack in a specific way and revert. @@ -39,7 +43,7 @@ go-grpc/ Go gRPC pricing and echo service, and its protos rust-inventory/ Rust HTTP inventory service migrate/ goose migration runner and the migration SQL k8s/apps/base/ the services, their Ingresses, dashboards, and alert rules -k8s/infra/ Forgejo, Flux, and the monitoring stack +k8s/infra/ Forgejo, Flux, the monitoring stack, and Flagsmith scenarios/ the failure scenarios and their overlays scripts/ bring-up, teardown, build, smoke, validation docs/ topology, services, scenarios @@ -97,6 +101,7 @@ Starting up the stack downloads from - `gcr.io` - `ghcr.io` - `codeberg.org` +- `docker.flagsmith.com`, which redirects to `docker.io` - `proxy.golang.org` - `crates.io` - `prometheus-community.github.io` @@ -107,6 +112,8 @@ cluster, and the third-party telemetry defaults that do so are turned off in the manifests - Grafana usage reporting and update checks - Forgejo avatar fetching +- Flagsmith's telemetry heartbeat, and the feature flags its own dashboard + would otherwise fetch from Flagsmith's hosted service Flux still re-fetches the chart index hourly, and the other components have not been audited. This project is not air-gapped; block egress at the podman @@ -124,9 +131,11 @@ has more details. The project is intended to be run with make targets that call bash scripts. ```sh -make up # registry, cluster, images, GitOps, monitoring, services, smoke +make up # registry, cluster, images, GitOps, monitoring, services, + # Flagsmith, smoke make smoke # exercise the service APIs -make validate # pods, ingress, metrics, Prometheus targets, Flux +make validate # pods, ingress, host port boundary, metrics, Prometheus + # targets, the Flagsmith seed, Flux make build # registry, proto code, image builds and pushes make down # delete cluster, registry, network, kubeconfig make lint # shellcheck, actionlint, kustomize build, semgrep; needs no cluster @@ -140,7 +149,9 @@ scripts take `--help` and document their supported flags. `setup.sh` calls `build.sh`, so `make up` or a bare `bash scripts/setup.sh` on a clean machine runs everything, and it exits non-zero if the smoke test at -the end fails. The scripts are idempotent and can be safely rerun. Reset any +the end fails. The scripts are idempotent and can be safely rerun, with one +caveat about the Flagsmith password noted under +[Reaching example-stack](#reaching-example-stack). Reset any applied failure scenario before rerunning `setup.sh`, because its seed push restores the manifests in Forgejo but not the database - so a scenario that changed the schema stays broken until its script is run with `--reset`. @@ -167,9 +178,19 @@ with a flag on `setup.sh`: | 6551 | Kubernetes API | `--k8s-api-port` | | 5111 | local image registry | `--registry-port` | +All three are bound to 127.0.0.1 on the host, so nothing in the stack is +reachable from the network the machine is on. The `*.localhost` names resolve +locally and were never the boundary; the loopback binding is. + +`validate-stack.sh` takes the same three port flags, `--ingress-port`, +`--k8s-api-port`, and `--registry-port`. It checks each port for that binding, +so if the defaults were changed on `setup.sh`, the same values have to be +given here or the boundary check probes ports nothing is listening on and +fails. + Everything is currently HTTP. No Ingress declares TLS, so the cluster publishes -no HTTPS port. `--ingress-port` also rewrites the links on the landing page and -in the UI. +no HTTPS port. `--ingress-port` also rewrites the links on the landing page, +in the UI, and in the pages Flagsmith builds for itself. Everything HTTP is reachable at a `*.localhost` hostname. `curl`, Chrome, and Firefox will resolve those names to 127.0.0.1 themselves, which is all the @@ -187,6 +208,7 @@ only under systemd-resolved or nss-myhostname. If you need those, add one | http://forgejo.localhost:8090/ | Forgejo | | http://grafana.localhost:8090/ | Grafana | | http://prometheus.localhost:8090/ | Prometheus | +| http://flagsmith.localhost:8090/ | Flagsmith | `go-grpc` only speaks HTTP/2 and is not exposed through the ingress. You can reach it with a port-forward @@ -209,6 +231,7 @@ way on any machine. None of it is a production secret. | Forgejo | user `bootstrap`, password `password` | `--forgejo-password` on `scripts/setup.sh` | | Grafana | `admin` / `admin` | `k8s/infra/monitoring/helmrelease.yaml` | | Flux Receiver webhook | shared secret `example-stack-webhook-secret` | `k8s/infra/flux/receiver.yaml` | +| Flagsmith | `bootstrap@flagsmith.local` / `example-stack-demo` | `--flagsmith-password` on `scripts/setup.sh` | The PostgreSQL container runs with `POSTGRES_HOST_AUTH_METHOD=trust`, so the password is never checked and any connection from inside the cluster is @@ -217,6 +240,14 @@ a connection string from it). The browser UI never holds the bearer token: nginx injects it server-side from the Secret, through the envsubst step in `k8s/apps/base/ui/nginx.conf.template`, so the token has one home. +Flagsmith's account is created once, on the first bring-up. Later runs do not +reset its password: they log in with the one they are given, so every later +`setup.sh` has to be given the same value. If you pass `--flagsmith-password` +on the first run, pass it on every run. If you change the password in the +Flagsmith web interface, pass the new value from then on, or the seed step +fails at login. Flagsmith applies Django's password validators, so a short or +common value is rejected at signup. + `setup.sh` sets the Forgejo bootstrap password on every run. Pass `--forgejo-password` to override the default or change it afterwards @@ -274,9 +305,11 @@ There are four layers of testing. All exit non-zero on failure. authentication rejection, the seeded inventory, order creation across the full call graph, insufficient stock, an unknown item, and the UI proxy paths. - **`scripts/validate-stack.sh`** checks the infrastructure: pod health, - ingress resolution, `/metrics` endpoints, Prometheus scrape targets, alert - rules loaded, alerts not firing, and Flux reconciliation status. It does not - run the scenarios. + ingress resolution, that each of the three host ports answers on 127.0.0.1 + and not on the machine's own network address, `/metrics` endpoints, + Prometheus scrape targets, alert rules loaded, alerts not firing, that both + seeded Flagsmith environments answer for the keys the bring-up stored, and + Flux reconciliation status. It does not run the scenarios. - **`--verify` on a scenario's `demo.sh`** checks that the scenario broke what it expects to break, and that `--reset` put the stack back correctly. diff --git a/docs/scenarios.md b/docs/scenarios.md index 00ab0c0..3272b4a 100644 --- a/docs/scenarios.md +++ b/docs/scenarios.md @@ -18,8 +18,13 @@ A webhook triggers a Flux reconciliation immediately. `scenarios/lib.sh` has shared code between the scenarios' `demo.sh` scripts. They share flags, the Forgejo token and port-forward, the clone, the overlay -push, the Flux path switch, and a wait helper. Each `demo.sh` contains only -what is specific to its own failure. `demo.sh` replaces its overlay directory +push, the Flux path switch, and a wait helper. The token is read from the +`forgejo-bootstrap` Secret the bring-up wrote and reused while Forgejo still +accepts it, which is the "Reusing the Forgejo access token" line the scripts +print; one is minted through the Forgejo command line when that Secret is +missing or the token check does not answer 200, whether because Forgejo +rejected the token or because the port-forward had not answered yet. Each +`demo.sh` contains only what is specific to its own failure. `demo.sh` replaces its overlay directory in the clone, so a file dropped from a scenario does not survive in Forgejo and re-running the script with nothing to change pushes nothing. A wait that times out exits non-zero. diff --git a/docs/services.md b/docs/services.md index f13c659..29de52e 100644 --- a/docs/services.md +++ b/docs/services.md @@ -184,20 +184,24 @@ infrastructure interface with the ports this stack uses. ## PostgreSQL and the migrations -There is a single PostgreSQL 17 Deployment that holds both databases. -`inventory` is created by the image's own initialization; `orders` is created -by an initialization script mounted from a ConfigMap. Data is on a 100Mi -local-path volume, which survives pod restarts and `k3d cluster stop` and is -lost on cluster delete. - -Schema and seed data are applied by Kubernetes Jobs running -[goose](https://github.com/pressly/goose), not by services. The services -expect the schema to exist and fail if it does not. +There is a single PostgreSQL 17 Deployment that holds every database in the +stack. `inventory` is created by the image's own initialization; `orders` and +`flagsmith` are created by an initialization script mounted from a ConfigMap, +which the image runs only against an empty data directory, so they appear on +a fresh volume and not on a restart. Data is on a 100Mi local-path volume, +which survives pod restarts and `k3d cluster stop` and is lost on cluster +delete. + +Schema and seed data are applied by Kubernetes Jobs, not by services: +[goose](https://github.com/pressly/goose) for the two service databases, +Flagsmith's own Django migrations for its. The services expect the schema +to exist and fail if it does not. | Job | Database | Description | |---|---|---| | `migrate-inventory-v1` | `inventory` | Creates the inventory table and seeds three items | | `migrate-orders-v1` | `orders` | Creates the orders table | +| `migrate-flagsmith-v1` | `flagsmith` | Runs Flagsmith's own Django migrations from its image, not goose | The migration SQL lives in `migrate/migrations/inventory/` and `migrate/migrations/orders/`, embedded into the `migrate` binary @@ -213,6 +217,43 @@ On a fresh cluster the services may restart once or twice while the migration Jobs run. Those restarts show up in `kube_pod_container_status_restarts_total`. +## Flagsmith + +Flagsmith is a self-hosted feature flag server, deployed by Flux from +`k8s/infra/flagsmith` into its own `flagsmith` namespace the way the +monitoring stack is. It is the unified image, so one container serves both +the API and the web interface on port 8000, reachable at +`http://flagsmith.localhost:8090/`. Its schema lives in the `flagsmith` +database in the shared PostgreSQL and is applied by the +`migrate-flagsmith-v1` Job, not by the server. + +| Endpoint | Auth | Description | +|---|---|---| +| `GET /health/liveness/` | No | Liveness. Returns `{"status": "ok"}` without touching the database | +| `GET /health/readiness/` | No | Readiness. 200 while the database is reachable and no migration is outstanding, 500 otherwise | +| `GET /metrics/` on port 9100 | No | Prometheus metrics, from a second HTTP server in the same process. A `ServiceMonitor` registers it | +| `/api/v1/...` | Yes | The Flagsmith REST API. `Authorization: Token ` for the admin API, `X-Environment-Key: ` for the SDK endpoints | + +`scripts/setup.sh` seeds it after Flux has reconciled it: a bootstrap account, +an organization and a project both named `example-stack`, and exactly two +environments, `staging` and `production`. No feature flags are created, and +no service in this stack reads a flag yet. + +The two environments' client-side keys are written into a +`flagsmith-bootstrap` Secret in the `example-stack` namespace, under the keys +`staging` and `production`, alongside the bootstrap account's API token under +`admin-token`. Flagsmith generates all three at seed time, so they cannot live +in the manifests Flux reconciles; `setup.sh` writes the Secret with +`kubectl`. Flux prunes only what it applied, so a reconcile leaves the Secret +alone, and the `app.kubernetes.io/managed-by` label on it records which script +owns it. Nothing mounts it yet. + +The bootstrap login is `bootstrap@flagsmith.local` with the password +`--flagsmith-password` sets, `example-stack-demo` by default. It is a +demo-only credential like every other one in this stack. Flagsmith runs +Django's password validators on signup, so a short or common password is +rejected. + ## How the services are built Every image is a multi-stage Dockerfile built by podman through diff --git a/docs/topology.md b/docs/topology.md index 084c3bf..a9f787c 100644 --- a/docs/topology.md +++ b/docs/topology.md @@ -10,8 +10,9 @@ about each service. `scripts/setup.sh` creates a k3d cluster named `example-stack` with one server node and one agent node, running a k3s image pinned by tag and digest. k3d publishes the cluster's Traefik ingress on host port 8090 -(HTTP) and the Kubernetes API on 6551. No Ingress declares TLS, so no -HTTPS port is published. The cluster writes its own kubeconfig +(HTTP) and the Kubernetes API on 6551, both bound to loopback like every +host port the stack opens. No Ingress declares TLS, so no HTTPS port is +published. The cluster writes its own kubeconfig to `$HOME/.kube/example-stack.yaml` and leaves the default kubeconfig untouched, so the context the shell already had is unchanged. @@ -19,7 +20,7 @@ On a podman machine with 4 CPUs and 8 GiB of memory the stack's containers peak just under 3 GiB of memory together when starting. The two k3s nodes each burst past a full core while the images load, the first run takes under ten minutes from cold start and about -six minutes with the images already built. A re-run reconciles in +seven minutes with the images already built. A re-run reconciles in under two minutes. The cluster takes about 5 GiB of the machine's disk while it is up and cleans that upon teardown. By default teardown keeps the built images and the k3s image, which are under 1 GiB, plus @@ -27,10 +28,19 @@ the Go and Rust build caches podman holds between builds. `setup.sh` enforces a floor of 4 GiB of memory, which is the smallest machine that leaves that peak headroom. +Flagsmith asks for 300Mi of memory and 300m of CPU and is capped at +500Mi of memory, the figures its Helm chart suggests in the +commented-out block of its values file, with no CPU limit like the +rest of the stack: under the chart's 500m its startup throttled at 78% +and fired CPUThrottlingHigh. Measured on this stack after a bring-up, +idle, with the one gunicorn worker the Deployment sets, it holds 331Mi +and about 14m of CPU: above its request, inside its limit. Its image +is the second long pull of a cold start after the k3s one. + Besides the three host ports, scripts open short-lived port-forwards on -13000 (Forgejo bootstrap), 18080, 18081, 18082, 19090, and 19091 (smoke -test and stack validation), and 19092 (the go-grpc health call in a -scenario's `--verify`). +13000 (Forgejo bootstrap), 18000 (Flagsmith bootstrap), 18080, 18081, +18082, 19090, and 19091 (smoke test and stack validation), and 19092 +(the go-grpc health call in a scenario's `--verify`). Namespaces: @@ -40,6 +50,7 @@ Namespaces: | `forgejo` | the in-cluster Git server | | `flux-system` | the Flux controllers, the GitRepository, the Kustomizations, the Receiver | | `monitoring` | kube-prometheus-stack and the HelmRelease that manages it | +| `flagsmith` | the self-hosted Flagsmith server and its migration Job | ## The registry @@ -69,23 +80,43 @@ cluster and Flux reconciles from Forgejo. When starting the cluster, `setup.sh` 2. creates a `bootstrap` admin user and an access token through the Forgejo command line inside the pod, then creates the `endgrainlabs/example-stack` organization and repository through the -Forgejo API. +Forgejo API. That token is stored in a `forgejo-bootstrap` Secret once +the services' namespace exists, and the scenario scripts reuse it +rather than minting one on every run. 3. pushes a snapshot of `k8s/apps/base`, -`k8s/infra/monitoring`, and `k8s/infra/monitoring-flux` into that -repository. +`k8s/infra/monitoring`, `k8s/infra/monitoring-flux`, and +`k8s/infra/flagsmith` into that repository. 4. installs Flux from the pinned upstream manifest for `v2.8.5`. The image reflector and image automation controllers are scaled to zero: the stack defines no image automation resources and they would idle at real cost. 5. applies the Flux objects in `k8s/infra/flux`: a `GitRepository` -pointed at the in-cluster Forgejo, three `Kustomization` objects -(`monitoring`, `apps`, `monitoring-flux`), and a `Receiver`. +pointed at the in-cluster Forgejo, four `Kustomization` objects +(`monitoring`, `apps`, `monitoring-flux`, `flagsmith`), and a +`Receiver`. 6. creates a Forgejo webhook that calls the Receiver, so a push reconciles immediately instead of waiting for the poll interval. +7. seeds Flagsmith through its REST API once that Kustomization +reconciles: a bootstrap account, the `example-stack` organization and +project, and the `staging` and `production` environments. The account's +API token and the two client-side environment keys go into a +`flagsmith-bootstrap` Secret in the `example-stack` namespace, under +the keys `admin-token`, `staging`, and `production`. + +`forgejo-bootstrap` and `flagsmith-bootstrap` are the only two +Kubernetes objects in the `example-stack` namespace that a script +writes rather than Flux. +Both hold values minted while the stack comes up, which cannot be +written into manifests ahead of time. Flux prunes only what it applied, +so neither is removed by a reconcile, and both carry an +`app.kubernetes.io/managed-by=setup.sh` label saying so. The `apps` and `monitoring-flux` kustomizations depend on `monitoring`, because the Prometheus operator custom resource definitions must exist -before a `ServiceMonitor` or a `PrometheusRule` will apply. +before a `ServiceMonitor` or a `PrometheusRule` will apply. `flagsmith` +depends on `apps`: it registers a `ServiceMonitor` too, and its +migration Job needs the `flagsmith` database in the PostgreSQL that +`apps` brings up. A failure scenario works in the same way. It clones the Forgejo repository, commits an overlay, pushes, and patches the `apps` @@ -146,6 +177,7 @@ is plain YAML rendered by kustomize. +-----+ PostgreSQL | | orders db | | inventory db | + | flagsmith db | +-----------------+ ``` @@ -154,10 +186,12 @@ is plain YAML rendered by kustomize. the currency, and writes the order to its own database. All calls between services use in-cluster Service names. -A single PostgreSQL Deployment holds both databases. `rust-inventory` -owns `inventory`, `go-api` owns `orders`. Neither service creates its -own schema. Migration Jobs do this before the services need it, and a -service fails at startup if the schema is missing. +A single PostgreSQL Deployment holds all three databases. +`rust-inventory` owns `inventory`, `go-api` owns `orders`, and the +Flagsmith server in its own namespace owns `flagsmith`. No service +creates its own schema. Migration Jobs do this before the services need +it. `go-api` and `rust-inventory` fail at startup if the schema is +missing; Flagsmith stays unready until its Job has run. Data resides on a 100Mi local-path volume. It survives pod restarts and `k3d cluster stop`, and is lost on `k3d cluster delete`, which is what @@ -171,8 +205,8 @@ Traefik routes by hostname on port 8090, over HTTP only landing page - `ui.localhost` to the order-management UI - `goapi.localhost` and `rust.localhost` to their services -- `forgejo.localhost`, `grafana.localhost`, and `prometheus.localhost` to the - infrastructure +- `forgejo.localhost`, `grafana.localhost`, `prometheus.localhost`, and + `flagsmith.localhost` to the infrastructure The UI is an nginx container that serves static files and proxies the API paths to `go-api` and `rust-inventory`, injecting the demo bearer diff --git a/go-grpc/proto/echopb/echo.pb.go b/go-grpc/proto/echopb/echo.pb.go index 46f1fe6..6c8f9d5 100644 --- a/go-grpc/proto/echopb/echo.pb.go +++ b/go-grpc/proto/echopb/echo.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 +// protoc-gen-go v1.36.12 // protoc v7.34.1 // source: echo.proto diff --git a/go-grpc/proto/pricingpb/pricing.pb.go b/go-grpc/proto/pricingpb/pricing.pb.go index 931aa25..bd4ad23 100644 --- a/go-grpc/proto/pricingpb/pricing.pb.go +++ b/go-grpc/proto/pricingpb/pricing.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 +// protoc-gen-go v1.36.12 // protoc v7.34.1 // source: pricing.proto diff --git a/k8s/apps/base/landing.yaml b/k8s/apps/base/landing.yaml index ed68247..7a86bcb 100644 --- a/k8s/apps/base/landing.yaml +++ b/k8s/apps/base/landing.yaml @@ -44,6 +44,7 @@ data:
  • Grafana - kube-prometheus-stack dashboards. Per-service and consolidated views with service level objective gauges. Demo-only login: admin / admin.
  • Prometheus - metrics store, service level objective recording rules, alerting rules.
  • Forgejo - in-cluster Git source of truth. Flux reconciles the endgrainlabs/example-stack repository from here. Demo-only login: bootstrap / password.
    • +
  • Flagsmith - self-hosted feature flags. Seeded with the example-stack organization and project and the staging and production environments; no flags are defined and no service reads one yet. Demo-only login: bootstrap@flagsmith.local / example-stack-demo.

    • Authenticated endpoints

    diff --git a/k8s/apps/base/postgres.yaml b/k8s/apps/base/postgres.yaml index 76da36b..d67bc29 100644 --- a/k8s/apps/base/postgres.yaml +++ b/k8s/apps/base/postgres.yaml @@ -9,6 +9,7 @@ data: set -e psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL SELECT 'CREATE DATABASE orders' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'orders')\gexec + SELECT 'CREATE DATABASE flagsmith' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'flagsmith')\gexec EOSQL --- apiVersion: v1 diff --git a/k8s/infra/flagsmith/deployment.yaml b/k8s/infra/flagsmith/deployment.yaml new file mode 100644 index 0000000..8dcc1de --- /dev/null +++ b/k8s/infra/flagsmith/deployment.yaml @@ -0,0 +1,131 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flagsmith + namespace: flagsmith + labels: + app: flagsmith +spec: + replicas: 1 + selector: + matchLabels: + app: flagsmith + template: + metadata: + labels: + app: flagsmith + spec: + containers: + - name: flagsmith + image: docker.flagsmith.com/flagsmith/flagsmith:2.273.0@sha256:6e9e8bac587865f272a0f81ad19b2c999b5b21a9514f878a7ecfabc1ab0ed7b1 + # The image's own default is migrate-and-serve. Migrations run in + # the Job beside this file instead, so the server only serves; the + # readiness probe below holds the pod out of the Service until the + # Job has applied them. + args: ["serve"] + ports: + - name: http + containerPort: 8000 + - name: prom + containerPort: 9100 + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: flagsmith + key: database-url + - name: DJANGO_SECRET_KEY + valueFrom: + secretKeyRef: + name: flagsmith + key: django-secret-key + # Flagsmith builds the links in its own pages, including the + # password-reset link, from this, so it names the host ingress + # and not the in-cluster Service. setup.sh sets the port for + # --ingress-port. + - name: FLAGSMITH_DOMAIN + value: "flagsmith.localhost:8090" + # Demo-only: Django's host allowlist is open because this stack is + # reached on a local ingress under a name that follows + # --ingress-port, and the alternative is enumerating every name a + # developer might use to reach a throwaway cluster. What the + # allowlist usually protects is absolute links built from the Host + # header; the links Flagsmith puts in its emails come from + # FLAGSMITH_DOMAIN above instead, so opening it costs little here. + - name: DJANGO_ALLOWED_HOSTS + value: "*" + # Background work runs in a thread inside this process. The stack + # runs no separate task processor, which is the only other way + # Flagsmith has of running it. + - name: TASK_RUN_METHOD + value: "SEPARATE_THREAD" + - name: PROMETHEUS_ENABLED + value: "true" + # Flagsmith posts a heartbeat of object counts to + # api.flagsmith.com on every migrate, and defaults to on + # (ENABLE_TELEMETRY in api/app/settings/common.py). The migration + # Job beside this file is where it would fire; it is set here too + # so both containers of this pair are configured the same way. + - name: ENABLE_TELEMETRY + value: "false" + # The dashboard runs its own feature flags against Flagsmith's + # hosted service, on every page load, before anyone has logged in. + # Both settings default to None, and the view that builds the + # frontend's config omits a None, which leaves the key and URL the + # self-hosted bundle hardcodes in place + # (frontend/env/project_selfhosted.js). Empty strings are not None, + # so they replace them. The key is what stops the call: the SDK + # refuses an empty environment key before it opens a connection. + # The URL is blanked with it because the SDK's own fallback is the + # same hosted address, so clearing that alone would change nothing. + - name: FLAGSMITH_ON_FLAGSMITH_API_URL + value: "" + - name: FLAGSMITH_ON_FLAGSMITH_API_KEY + value: "" + # One of local, dev, staging, production. It picks defaults inside + # Flagsmith and is not the name of a Flagsmith environment. + - name: ENVIRONMENT + value: "production" + # One browser on a laptop needs one worker. The image defaults to + # three, and three Django processes do not fit under the memory + # limit below. + - name: GUNICORN_WORKERS + value: "1" + # Django and gunicorn take tens of seconds to answer on a laptop, so + # a startup probe holds the other two back rather than letting + # liveness restart a pod that is still booting. Readiness also fails + # while the migration Job has not finished, so the five minutes this + # budgets covers waiting for the Job as well as the boot. + startupProbe: + httpGet: + path: /health/readiness/ + port: http + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 2 + livenessProbe: + httpGet: + path: /health/liveness/ + port: http + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 5 + # Readiness runs django-health-check, which fails while the database + # is unreachable and while migrations are outstanding. + readinessProbe: + httpGet: + path: /health/readiness/ + port: http + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 10 + # Requests and the memory limit are the figures the Flagsmith chart + # suggests in its commented-out api.resources block. No CPU limit, + # like every other Deployment here: Django's startup throttled at + # 78% under the chart's 500m and fired CPUThrottlingHigh. + resources: + requests: + memory: 300Mi + cpu: 300m + limits: + memory: 500Mi diff --git a/k8s/infra/flagsmith/ingress.yaml b/k8s/infra/flagsmith/ingress.yaml new file mode 100644 index 0000000..bf93eb9 --- /dev/null +++ b/k8s/infra/flagsmith/ingress.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: flagsmith + namespace: flagsmith +spec: + ingressClassName: traefik + rules: + - host: flagsmith.localhost + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: flagsmith + port: + number: 8000 diff --git a/k8s/infra/flagsmith/kustomization.yaml b/k8s/infra/flagsmith/kustomization.yaml new file mode 100644 index 0000000..0022490 --- /dev/null +++ b/k8s/infra/flagsmith/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - secret.yaml + - migrate.yaml + - deployment.yaml + - service.yaml + - ingress.yaml + - servicemonitor.yaml diff --git a/k8s/infra/flagsmith/migrate.yaml b/k8s/infra/flagsmith/migrate.yaml new file mode 100644 index 0000000..32944c8 --- /dev/null +++ b/k8s/infra/flagsmith/migrate.yaml @@ -0,0 +1,45 @@ +# Named migrate--v1 like the migration Jobs in k8s/apps/base: when the +# image's migration set changes, bump the suffix, and Flux creates a new Job +# rather than trying to patch the completed one. Flagsmith's migrations +# are idempotent, so a re-run against an already migrated database is a no-op. +# +# The image's own bootstrap command would create the first superuser here, +# but only with an unusable password and a reset link. scripts/setup.sh +# creates the account through the signup endpoint instead, which takes a +# password and makes that first account a superuser. +apiVersion: batch/v1 +kind: Job +metadata: + name: migrate-flagsmith-v1 + namespace: flagsmith +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + app: migrate-flagsmith + spec: + restartPolicy: OnFailure + containers: + - name: migrate + image: docker.flagsmith.com/flagsmith/flagsmith:2.273.0@sha256:6e9e8bac587865f272a0f81ad19b2c999b5b21a9514f878a7ecfabc1ab0ed7b1 + args: ["migrate"] + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: flagsmith + key: database-url + - name: DJANGO_SECRET_KEY + valueFrom: + secretKeyRef: + name: flagsmith + key: django-secret-key + - name: ENVIRONMENT + value: "production" + # Flagsmith's telemetry heartbeat is wired to post_migrate and + # defaults to on, so this Job is what would post the stack's + # object counts to api.flagsmith.com. + - name: ENABLE_TELEMETRY + value: "false" diff --git a/k8s/infra/flagsmith/namespace.yaml b/k8s/infra/flagsmith/namespace.yaml new file mode 100644 index 0000000..9f76ec8 --- /dev/null +++ b/k8s/infra/flagsmith/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flagsmith diff --git a/k8s/infra/flagsmith/secret.yaml b/k8s/infra/flagsmith/secret.yaml new file mode 100644 index 0000000..d7a20fa --- /dev/null +++ b/k8s/infra/flagsmith/secret.yaml @@ -0,0 +1,14 @@ +# Demo-only credentials, committed the same way k8s/apps/base/secrets.yaml is. +# Every value here is a fixed default so the stack comes up the same way on any +# machine; none of it is a production secret. The database URL names the +# PostgreSQL Service in the example-stack namespace, which holds the flagsmith +# database alongside inventory and orders. +apiVersion: v1 +kind: Secret +metadata: + name: flagsmith + namespace: flagsmith +type: Opaque +stringData: + database-url: "postgres://postgres:postgres@postgres.example-stack.svc.cluster.local:5432/flagsmith" + django-secret-key: "demo-only-django-secret-key-not-a-real-secret" diff --git a/k8s/infra/flagsmith/service.yaml b/k8s/infra/flagsmith/service.yaml new file mode 100644 index 0000000..43faeb1 --- /dev/null +++ b/k8s/infra/flagsmith/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: flagsmith + namespace: flagsmith + labels: + app: flagsmith +spec: + selector: + app: flagsmith + ports: + - name: http + port: 8000 + targetPort: 8000 + # Flagsmith serves Prometheus metrics from a second HTTP server in a + # daemon thread, not from the API port, so the scrape target is its own + # port. 9100 is the port that server binds. + - name: prom + port: 9100 + targetPort: 9100 diff --git a/k8s/infra/flagsmith/servicemonitor.yaml b/k8s/infra/flagsmith/servicemonitor.yaml new file mode 100644 index 0000000..71c1601 --- /dev/null +++ b/k8s/infra/flagsmith/servicemonitor.yaml @@ -0,0 +1,15 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: flagsmith + namespace: flagsmith + labels: + app: flagsmith +spec: + selector: + matchLabels: + app: flagsmith + endpoints: + - port: prom + path: /metrics/ + interval: 15s diff --git a/k8s/infra/flux/kustomization-flagsmith.yaml b/k8s/infra/flux/kustomization-flagsmith.yaml new file mode 100644 index 0000000..2956986 --- /dev/null +++ b/k8s/infra/flux/kustomization-flagsmith.yaml @@ -0,0 +1,21 @@ +# dependsOn apps, not nothing: the ServiceMonitor needs the Prometheus +# operator custom resource definitions the monitoring Kustomization installs, +# and the migration Job needs the flagsmith database in the PostgreSQL the +# apps Kustomization brings up. Depending on apps covers both, because apps +# already depends on monitoring. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: flagsmith + namespace: flux-system +spec: + interval: 5m + dependsOn: + - name: apps + sourceRef: + kind: GitRepository + name: example-stack + path: ./k8s/infra/flagsmith + prune: true + wait: true + timeout: 10m diff --git a/k8s/infra/flux/kustomization.yaml b/k8s/infra/flux/kustomization.yaml index 02608ea..8d8d3d8 100644 --- a/k8s/infra/flux/kustomization.yaml +++ b/k8s/infra/flux/kustomization.yaml @@ -5,4 +5,5 @@ resources: - kustomization-apps.yaml - kustomization-monitoring.yaml - kustomization-monitoring-flux.yaml + - kustomization-flagsmith.yaml - receiver.yaml diff --git a/scenarios/lib.sh b/scenarios/lib.sh index 468e784..cacb0d8 100644 --- a/scenarios/lib.sh +++ b/scenarios/lib.sh @@ -98,12 +98,36 @@ scenario_cleanup() { } trap scenario_cleanup EXIT -# Generates a Forgejo access token through the command line inside the pod and -# opens a port-forward to Forgejo, so the clone below can reach it from the -# host. The token name carries the scenario and a timestamp: Forgejo rejects a -# duplicate name, and a demo.sh may be run more than once. -scenario_open_forgejo() { - local token_output +# Puts a working Forgejo access token in FORGEJO_TOKEN. The bring-up stores +# the token it minted in the forgejo-bootstrap Secret, and that one is reused +# while Forgejo still accepts it, so a stack that has run many scenarios does +# not accumulate a token per run. A missing Secret, or a token Forgejo no +# longer accepts, falls back to minting one through the command line inside +# the pod. The minted name carries the scenario and a timestamp: Forgejo +# rejects a duplicate name, and a demo.sh may be run more than once. +# +# Needs the port-forward open: the reused token is checked against the API +# before anything is cloned with it. +scenario_forgejo_token() { + local token_output status + + FORGEJO_TOKEN=$(kubectl -n "${NAMESPACE}" get secret forgejo-bootstrap \ + -o jsonpath='{.data.token}' 2>/dev/null | base64 --decode 2>/dev/null || echo "") + + if [ -n "${FORGEJO_TOKEN}" ]; then + # curl prints 000 itself when no connection is made, so no fallback: + # one would print a second 000 after it. + status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 \ + -H "Authorization: token ${FORGEJO_TOKEN}" \ + "http://localhost:${FORGEJO_PF_PORT}/api/v1/user" 2>/dev/null || true) + status="${status:-000}" + if [ "${status}" = "200" ]; then + echo "==> Reusing the Forgejo access token from the forgejo-bootstrap Secret" + return 0 + fi + echo "==> The stored Forgejo access token did not authenticate (${status}), generating one" + FORGEJO_TOKEN="" + fi echo "==> Generating Forgejo access token" token_output=$(kubectl -n "${FORGEJO_NS}" exec deploy/forgejo -- \ @@ -117,7 +141,11 @@ scenario_open_forgejo() { echo "${token_output}" >&2 exit 1 fi +} +# Opens a port-forward to Forgejo and clones the in-cluster repository through +# it, so the overlay below can be pushed from the host. +scenario_open_forgejo() { echo "==> Port-forwarding Forgejo" kubectl -n "${FORGEJO_NS}" port-forward "svc/forgejo" "${FORGEJO_PF_PORT}:3000" &>/dev/null & FORGEJO_PF_PID=$! @@ -130,6 +158,8 @@ scenario_open_forgejo() { sleep 0.5 done + scenario_forgejo_token + WORK_DIR=$(mktemp -d) cd "${WORK_DIR}" || exit 1 git clone -q \ diff --git a/scripts/build.sh b/scripts/build.sh index e1b8de5..987c32e 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -66,9 +66,11 @@ if podman container exists "${REGISTRY_NAME}" 2>/dev/null; then fi else echo "==> Creating local registry: ${REGISTRY_NAME}" + # Bound to loopback: only build.sh pushes through the host port. The + # cluster's nodes reach the registry over the podman network by name. podman run -d \ --name "${REGISTRY_NAME}" \ - -p "${REGISTRY_PORT}:5000" \ + -p "127.0.0.1:${REGISTRY_PORT}:5000" \ --restart always \ "${REGISTRY_IMAGE}" fi diff --git a/scripts/setup.sh b/scripts/setup.sh index a74d41b..d580329 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -3,7 +3,9 @@ set -euo pipefail # Brings up the example stack: local registry, k3d cluster, images, an # in-cluster Forgejo holding the manifests, Flux reconciling from it, the -# monitoring stack, the services, and a smoke test. +# monitoring stack, the services, a seeded Flagsmith, and a smoke test. The +# Forgejo and Flagsmith access tokens it mints are left behind in the +# forgejo-bootstrap and flagsmith-bootstrap Secrets. # # Phases are ordered so a failure costs as little as possible. The registry and # the image builds run before any cluster exists, so a broken build never @@ -13,7 +15,7 @@ set -euo pipefail # # A. Builds local registry up, service images built and pushed # B. Cluster k3d cluster created and wired to the registry -# C. Deploy Forgejo, Flux, monitoring, services +# C. Deploy Forgejo, Flux, monitoring, services, Flagsmith # D. Smoke smoke test against the running services # # Invoke with: bash scripts/setup.sh @@ -34,6 +36,9 @@ K8S_API_PORT="6551" # --cluster-name alone gives that cluster its own kubeconfig file. KUBECONFIG_PATH="" FORGEJO_PASSWORD="password" +# Flagsmith runs Django's password validators on signup, so the demo value +# has to be long enough and not a common word. +FLAGSMITH_PASSWORD="example-stack-demo" usage() { cat <&2; echo "" >&2; usage >&2; exit 1 ;; + --cluster-name=*) CLUSTER_NAME="${1#*=}" ;; + --registry-name=*) REGISTRY_NAME="${1#*=}" ;; + --registry-port=*) REGISTRY_PORT="${1#*=}" ;; + --ingress-port=*) INGRESS_PORT="${1#*=}" ;; + --k8s-api-port=*) K8S_API_PORT="${1#*=}" ;; + --kubeconfig-path=*) KUBECONFIG_PATH="${1#*=}" ;; + --forgejo-password=*) FORGEJO_PASSWORD="${1#*=}" ;; + --flagsmith-password=*) FLAGSMITH_PASSWORD="${1#*=}" ;; + --help|-h) usage; exit 0 ;; + *) echo "ERROR: unknown flag '$1'" >&2; echo "" >&2; usage >&2; exit 1 ;; esac shift done @@ -82,6 +92,14 @@ FORGEJO_ADMIN_EMAIL="bootstrap@forgejo.local" # the host. FORGEJO_PF_PORT="13000" MONITORING_NS="monitoring" +FLAGSMITH_NS="flagsmith" +FLAGSMITH_ORG="example-stack" +FLAGSMITH_PROJECT="example-stack" +# Synthetic address for the demo-only bootstrap account. Nothing sends mail. +FLAGSMITH_ADMIN_EMAIL="bootstrap@flagsmith.local" +# Local port for the bootstrap port-forward: a port unlikely to be in use on +# the host. +FLAGSMITH_PF_PORT="18000" # kube-prometheus-stack, PostgreSQL, and two node containers do not fit in # less than this. MIN_MACHINE_MEMORY_MIB="4096" @@ -109,6 +127,7 @@ echo " kubeconfig: ${KUBECONFIG_PATH}" # --- Cleanup tracking --- FORGEJO_PF_PID="" +FLAGSMITH_PF_PID="" WORK_DIR="" cleanup() { @@ -116,6 +135,10 @@ cleanup() { kill "${FORGEJO_PF_PID}" 2>/dev/null || true wait "${FORGEJO_PF_PID}" 2>/dev/null || true fi + if [ -n "${FLAGSMITH_PF_PID}" ]; then + kill "${FLAGSMITH_PF_PID}" 2>/dev/null || true + wait "${FLAGSMITH_PF_PID}" 2>/dev/null || true + fi if [ -n "${WORK_DIR}" ] && [ -d "${WORK_DIR}" ]; then rm -rf "${WORK_DIR}" fi @@ -128,6 +151,91 @@ forgejo_exec() { kubectl -n "${FORGEJO_NS}" exec deploy/forgejo -- "$@" } +# One Python program answers every question the Flagsmith seed asks of a +# Flagsmith API response. The response arrives as a string, so it is piped +# back in, the way prom_fact does it in validate-stack.sh. A regular +# expression over JSON would break the first time a field moved. +# +# field KEY one key of an object +# find NAME KEY one key of the first list entry whose name is NAME +# names the name of every list entry, one per line +# +# A list answer is either a bare array or a paginated object with "results". +# Nothing found is a non-zero exit, so callers fall back with || echo "". +flagsmith_fact() { + local response="$1" + shift + printf '%s' "${response}" | python3 -c ' +import json +import sys + +mode = sys.argv[1] +try: + doc = json.load(sys.stdin) +except ValueError: + sys.exit(1) + +if mode == "field": + value = doc.get(sys.argv[2]) if isinstance(doc, dict) else None + if value is None: + sys.exit(1) + print(value) + sys.exit(0) + +items = doc.get("results", []) if isinstance(doc, dict) else doc +if mode == "names": + for item in items: + print(item["name"]) + sys.exit(0) + +for item in items: + if item["name"] == sys.argv[2]: + print(item[sys.argv[3]]) + sys.exit(0) +sys.exit(1) +' "$@" 2>/dev/null +} + +# Writes one bootstrap Secret into the services' namespace. These values are +# minted while the stack comes up, so they cannot live in the manifests Flux +# reconciles; Flux prunes only what it applied, so a Secret written here +# survives every reconcile. The label says which script owns it. +write_bootstrap_secret() { + local name="$1" + local app="$2" + shift 2 + local literal + local args + args=() + for literal in "$@"; do + args+=(--from-literal="${literal}") + done + kubectl -n "${NAMESPACE}" create secret generic "${name}" \ + "${args[@]}" --dry-run=client -o yaml \ + | kubectl -n "${NAMESPACE}" apply -f - > /dev/null + kubectl -n "${NAMESPACE}" label secret "${name}" --overwrite \ + "app=${app}" "app.kubernetes.io/managed-by=setup.sh" > /dev/null +} + +# A call to the Flagsmith API with the bootstrap token attached. The token and +# the base URL are set in phase C9, before the first call. +flagsmith_api() { + local method="$1" + local path="$2" + local body="${3:-}" + if [ -n "${body}" ]; then + curl -s -X "${method}" \ + -H "Authorization: Token ${FLAGSMITH_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "${body}" \ + "${FLAGSMITH_API}${path}" || echo "" + else + curl -s -X "${method}" \ + -H "Authorization: Token ${FLAGSMITH_TOKEN}" \ + "${FLAGSMITH_API}${path}" || echo "" + fi +} + # A node container that has been stopped and started comes back on a different # address in the podman network, but k3s keeps the address it first registered # in the node's flannel annotation. The node then reports Ready and shuts its @@ -284,11 +392,14 @@ YAML # --kubeconfig-update-default=false: the default kubeconfig is left alone, # so this cluster never becomes the shell's current context. + # Both host ports are bound to loopback. Without an address, the podman + # machine's forwarder binds every interface of the host, which put the + # whole stack on whatever network the machine was on. k3d cluster create "${CLUSTER_NAME}" \ --image "${K3S_IMAGE}" \ --agents 1 \ - --api-port "${K8S_API_PORT}" \ - --port "${INGRESS_PORT}:80@loadbalancer" \ + --api-port "127.0.0.1:${K8S_API_PORT}" \ + --port "127.0.0.1:${INGRESS_PORT}:80@loadbalancer" \ --registry-config "${REGISTRIES_YAML}" \ --kubeconfig-update-default=false \ --kubeconfig-switch-context=false \ @@ -453,13 +564,16 @@ mkdir -p "${WORK_DIR}/k8s/apps" "${WORK_DIR}/k8s/infra" cp -R "${REPO_ROOT}/k8s/apps/base" "${WORK_DIR}/k8s/apps/base" cp -R "${REPO_ROOT}/k8s/infra/monitoring" "${WORK_DIR}/k8s/infra/monitoring" cp -R "${REPO_ROOT}/k8s/infra/monitoring-flux" "${WORK_DIR}/k8s/infra/monitoring-flux" +cp -R "${REPO_ROOT}/k8s/infra/flagsmith" "${WORK_DIR}/k8s/infra/flagsmith" # The landing page and the UI print browser URLs, which carry the host ingress -# port. The manifests hold the default; the seeded copies get whatever +# port, and Flagsmith builds the links in its own pages from FLAGSMITH_DOMAIN. +# The manifests hold the default; the seeded copies get whatever # --ingress-port was given, so those links stay clickable. if [ "${INGRESS_PORT}" != "8090" ]; then - echo " Rewriting landing page and UI links for ingress port ${INGRESS_PORT}" + echo " Rewriting landing page, UI, and Flagsmith links for ingress port ${INGRESS_PORT}" for f in "${WORK_DIR}/k8s/apps/base/landing.yaml" \ + "${WORK_DIR}/k8s/infra/flagsmith/deployment.yaml" \ "${WORK_DIR}/k8s/apps/base/ui/index.html"; do sed -e "s/localhost:8090/localhost:${INGRESS_PORT}/g" \ -e "s/host port 8090/host port ${INGRESS_PORT}/g" \ @@ -568,6 +682,12 @@ kubectl -n flux-system wait --for=condition=Ready --timeout=20m kustomization/mo echo "==> Waiting for the apps Kustomization to reconcile" kubectl -n flux-system wait --for=condition=Ready --timeout=5m kustomization/apps +# The namespace exists now, which is why this is not back in C2 where the +# token was minted. The scenario scripts reuse this token instead of minting +# one of their own on every run. +echo "==> Storing the Forgejo access token for the scenario scripts" +write_bootstrap_secret "forgejo-bootstrap" "forgejo" "token=${FORGEJO_TOKEN}" + # --- C8: Roll the services so they pick up freshly built images --- # # Builds run before the cluster exists, so the first reconciliation already @@ -578,6 +698,158 @@ echo "==> Rolling restart of the services to pick up rebuilt images" kubectl -n "${NAMESPACE}" rollout restart deployment/go-api deployment/go-grpc deployment/rust-inventory 2>/dev/null || true kubectl -n "${NAMESPACE}" rollout status deployment --timeout=120s || true +# --- C9: Seed Flagsmith --- +# +# Flagsmith has no way to create its first user with a known password from the +# environment: the image's bootstrap command creates one with an unusable +# password and prints a reset link. The account is created through the signup +# endpoint instead, which takes a password, answers with the API token the +# rest of the seed uses, and accepts superuser on a self-hosted instance that +# has no users yet. No feature flags are created; nothing reads one yet. + +mark "flagsmith reconcile wait begin (the second long pull: the Flagsmith image)" +echo "==> Waiting for the flagsmith Kustomization to reconcile" +kubectl -n flux-system wait --for=condition=Ready --timeout=10m kustomization/flagsmith + +echo "==> Port-forwarding Flagsmith for bootstrap" +kubectl -n "${FLAGSMITH_NS}" port-forward "svc/flagsmith" "${FLAGSMITH_PF_PORT}:8000" &>/dev/null & +FLAGSMITH_PF_PID=$! +FLAGSMITH_LIVENESS="http://localhost:${FLAGSMITH_PF_PORT}/health/liveness/" +for _ in $(seq 1 40); do + if curl -s -o /dev/null --max-time 1 "${FLAGSMITH_LIVENESS}" 2>/dev/null; then + break + fi + sleep 0.5 +done +if ! curl -s -o /dev/null --max-time 1 "${FLAGSMITH_LIVENESS}" 2>/dev/null; then + echo "ERROR: the Flagsmith port-forward on localhost:${FLAGSMITH_PF_PORT} did not answer within 20s" >&2 + exit 1 +fi + +FLAGSMITH_API="http://localhost:${FLAGSMITH_PF_PORT}/api/v1" + +echo "==> Bootstrapping the Flagsmith admin user" +FLAGSMITH_SIGNUP=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"${FLAGSMITH_ADMIN_EMAIL}\",\"password\":\"${FLAGSMITH_PASSWORD}\",\"first_name\":\"Bootstrap\",\"last_name\":\"User\",\"sign_up_type\":\"NO_INVITE\",\"superuser\":true}" \ + "${FLAGSMITH_API}/auth/users/" || echo "") +FLAGSMITH_TOKEN=$(flagsmith_fact "${FLAGSMITH_SIGNUP}" field key || echo "") + +if [ -n "${FLAGSMITH_TOKEN}" ]; then + echo " Admin user '${FLAGSMITH_ADMIN_EMAIL}' created" +else + # A re-run against a live cluster finds the account already there, and + # signup answers 400 rather than a token. + echo " Admin user '${FLAGSMITH_ADMIN_EMAIL}' already exists, logging in" + FLAGSMITH_LOGIN=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"${FLAGSMITH_ADMIN_EMAIL}\",\"password\":\"${FLAGSMITH_PASSWORD}\"}" \ + "${FLAGSMITH_API}/auth/login/" || echo "") + FLAGSMITH_TOKEN=$(flagsmith_fact "${FLAGSMITH_LOGIN}" field key || echo "") +fi + +if [ -z "${FLAGSMITH_TOKEN}" ]; then + echo "ERROR: could not obtain a Flagsmith API token for ${FLAGSMITH_ADMIN_EMAIL}" >&2 + echo " Signup answered: ${FLAGSMITH_SIGNUP}" >&2 + if [ -n "${FLAGSMITH_LOGIN:-}" ]; then + echo " Login answered: ${FLAGSMITH_LOGIN}" >&2 + fi + exit 1 +fi + +echo "==> Ensuring the Flagsmith organization and project exist" +FLAGSMITH_ORG_ID=$(flagsmith_fact "$(flagsmith_api GET /organisations/)" find "${FLAGSMITH_ORG}" id || echo "") +if [ -n "${FLAGSMITH_ORG_ID}" ]; then + echo " Organization '${FLAGSMITH_ORG}' already exists" +else + # The JSON bodies are built into a variable first. bash 3.2 loses the + # quoting of an escaped-quote body nested inside "$(...)", and brace- + # expands the {"a":1,"b":2} into two words, so every body with a comma + # was sent as two broken requests. + body='{"name":"'"${FLAGSMITH_ORG}"'"}' + response=$(flagsmith_api POST /organisations/ "${body}") + FLAGSMITH_ORG_ID=$(flagsmith_fact "${response}" field id || echo "") + if [ -z "${FLAGSMITH_ORG_ID}" ]; then + echo "ERROR: could not create the Flagsmith organization '${FLAGSMITH_ORG}'" >&2 + echo " Flagsmith answered: ${response}" >&2 + exit 1 + fi + echo " Organization '${FLAGSMITH_ORG}' created" +fi + +FLAGSMITH_PROJECT_ID=$(flagsmith_fact "$(flagsmith_api GET /projects/)" find "${FLAGSMITH_PROJECT}" id || echo "") +if [ -n "${FLAGSMITH_PROJECT_ID}" ]; then + echo " Project '${FLAGSMITH_PROJECT}' already exists" +else + body='{"name":"'"${FLAGSMITH_PROJECT}"'","organisation":'"${FLAGSMITH_ORG_ID}"'}' + response=$(flagsmith_api POST /projects/ "${body}") + FLAGSMITH_PROJECT_ID=$(flagsmith_fact "${response}" field id || echo "") + if [ -z "${FLAGSMITH_PROJECT_ID}" ]; then + echo "ERROR: could not create the Flagsmith project '${FLAGSMITH_PROJECT}'" >&2 + echo " Flagsmith answered: ${response}" >&2 + exit 1 + fi + echo " Project '${FLAGSMITH_PROJECT}' created" +fi + +# The two environment names the services will read later. Anything else the +# project carries is removed, so a Flagsmith release that starts creating a +# default environment on project create does not leave one behind. +echo "==> Ensuring the Flagsmith environments exist" +FLAGSMITH_ENVS=$(flagsmith_api GET "/environments/?project=${FLAGSMITH_PROJECT_ID}") + +while IFS= read -r env_name; do + [ -n "${env_name}" ] || continue + case "${env_name}" in + staging|production) continue ;; + esac + env_key=$(flagsmith_fact "${FLAGSMITH_ENVS}" find "${env_name}" api_key || echo "") + if [ -n "${env_key}" ]; then + echo " Removing the default environment '${env_name}'" + flagsmith_api DELETE "/environments/${env_key}/" > /dev/null + fi +done <&2 + echo " Flagsmith answered: ${response}" >&2 + exit 1 + fi + echo " Environment '${env_name}' created" + fi + case "${env_name}" in + staging) FLAGSMITH_STAGING_KEY="${env_key}" ;; + production) FLAGSMITH_PRODUCTION_KEY="${env_key}" ;; + esac +done + +# The admin token joins the two environment keys so a later run, a scenario, +# or a person has an authenticated way into Flagsmith without signing in +# again. validate-stack.sh reads the two environment keys back out to check +# the seeded project; nothing else reads any of it yet, and the SDKs will +# read the environment keys. +echo "==> Writing the Flagsmith bootstrap Secret into the ${NAMESPACE} namespace" +write_bootstrap_secret "flagsmith-bootstrap" "flagsmith" \ + "admin-token=${FLAGSMITH_TOKEN}" \ + "production=${FLAGSMITH_PRODUCTION_KEY}" \ + "staging=${FLAGSMITH_STAGING_KEY}" + +kill "${FLAGSMITH_PF_PID}" 2>/dev/null || true +wait "${FLAGSMITH_PF_PID}" 2>/dev/null || true +FLAGSMITH_PF_PID="" + # ============================================================================= # Phase D: Smoke # ============================================================================= @@ -588,7 +860,7 @@ mark "Phase D begin" echo "" echo "==> Namespaces:" -kubectl get ns | grep -E "${NAMESPACE}|${FORGEJO_NS}|flux-system|${MONITORING_NS}" || true +kubectl get ns | grep -E "${NAMESPACE}|${FORGEJO_NS}|flux-system|${MONITORING_NS}|${FLAGSMITH_NS}" || true echo "" echo "==> Pods in ${NAMESPACE}:" @@ -626,6 +898,7 @@ echo " - go-api: http://goapi.localhost:${INGRESS_PORT}/" echo " - rust-inventory: http://rust.localhost:${INGRESS_PORT}/" echo " - Forgejo: http://forgejo.localhost:${INGRESS_PORT}/ (${FORGEJO_ADMIN_USER} / ${FORGEJO_PASSWORD}, demo-only)" echo " - Grafana: http://grafana.localhost:${INGRESS_PORT}/ (admin / admin, demo-only)" +echo " - Flagsmith: http://flagsmith.localhost:${INGRESS_PORT}/ (${FLAGSMITH_ADMIN_EMAIL} / ${FLAGSMITH_PASSWORD}, demo-only)" echo " - Prometheus: http://prometheus.localhost:${INGRESS_PORT}/" echo "" echo " kubectl reaches this cluster with KUBECONFIG=${KUBECONFIG_PATH}." diff --git a/scripts/validate-stack.sh b/scripts/validate-stack.sh index f9efa59..c57af62 100644 --- a/scripts/validate-stack.sh +++ b/scripts/validate-stack.sh @@ -2,14 +2,17 @@ set -euo pipefail # Checks that the stack's infrastructure is wired correctly: pods running, -# ingress resolving, metrics endpoints responding, Prometheus scraping every -# expected target, Flux reconciling. Complements smoke-test.sh, which checks -# service behavior. +# ingress resolving, the three host ports answering on 127.0.0.1 and nowhere +# else, metrics endpoints responding, Prometheus scraping every expected +# target, both seeded Flagsmith environments answering for their keys, Flux +# reconciling. Complements smoke-test.sh, which checks service behavior. # # Invoke with: bash scripts/validate-stack.sh KUBECONFIG_PATH="${HOME}/.kube/example-stack.yaml" INGRESS_PORT="8090" +K8S_API_PORT="6551" +REGISTRY_PORT="5111" usage() { cat <&2; echo "" >&2; usage >&2; exit 1 ;; @@ -41,6 +48,7 @@ NAMESPACE="example-stack" MONITORING_NS="monitoring" FLUX_NS="flux-system" FORGEJO_NS="forgejo" +FLAGSMITH_NS="flagsmith" FAILED=0 pass() { echo " PASS $1"; } @@ -74,7 +82,10 @@ check_url() { local url="$2" local expect="${3:-200}" local status - status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "${url}" 2>/dev/null || echo "000") + # curl prints 000 itself when no connection is made, so no fallback: + # one would print a second 000 after it. + status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "${url}" 2>/dev/null || true) + status="${status:-000}" if [ "${status}" = "${expect}" ]; then pass "${description} (${status})" else @@ -119,6 +130,33 @@ check_body() { fail "${description} (no '${needle}' in ${url} within ${timeout}s: status ${status:-none}, ${#body} bytes, body kept at ${saved})" } +# Asks Flagsmith for one environment's flag list with that environment's +# client-side key, read back out of the flagsmith-bootstrap Secret the +# bring-up wrote. A key that no longer names a live environment is answered +# with a 401, so this checks the seeded project still has the environment as +# well as that the key is good. +check_flagsmith_environment() { + local env_name="$1" + local key status + key=$(kubectl -n "${NAMESPACE}" get secret flagsmith-bootstrap \ + -o "jsonpath={.data.${env_name}}" 2>/dev/null | base64 --decode 2>/dev/null || echo "") + if [ -z "${key}" ]; then + fail "flagsmith ${env_name} environment (no '${env_name}' key in the flagsmith-bootstrap Secret)" + return + fi + # curl prints 000 itself when no connection is made, so no fallback: + # one would print a second 000 after it. + status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 \ + -H "X-Environment-Key: ${key}" \ + "http://flagsmith.localhost:${INGRESS_PORT}/api/v1/flags/" 2>/dev/null || true) + status="${status:-000}" + if [ "${status}" = "200" ]; then + pass "flagsmith ${env_name} environment (${status})" + else + fail "flagsmith ${env_name} environment (got ${status}, expected 200)" + fi +} + # --- Pods --- echo "==> Pod health" @@ -155,10 +193,12 @@ check_job() { check_job "${NAMESPACE}" "migrate-inventory-v1" check_job "${NAMESPACE}" "migrate-orders-v1" +check_job "${FLAGSMITH_NS}" "migrate-flagsmith-v1" echo "" echo "==> Pod health (continued)" check_pods "${FORGEJO_NS}" "app=forgejo" "forgejo" +check_pods "${FLAGSMITH_NS}" "app=flagsmith" "flagsmith" check_pods "${MONITORING_NS}" "app.kubernetes.io/name=grafana" "grafana" check_pods "${MONITORING_NS}" "app.kubernetes.io/name=prometheus" "prometheus" check_pods "${FLUX_NS}" "app=source-controller" "flux source-controller" @@ -182,6 +222,64 @@ check_body "forgejo clone URL names the ingress host" \ "forgejo.localhost:${INGRESS_PORT}/endgrainlabs/example-stack.git" check_url "grafana" "http://grafana.localhost:${INGRESS_PORT}/" "302" check_url "prometheus" "http://prometheus.localhost:${INGRESS_PORT}/" "302" +check_url "flagsmith /health/liveness/" "http://flagsmith.localhost:${INGRESS_PORT}/health/liveness/" + +# --- Host port boundary --- +# +# The stack's three host ports must be bound to loopback. Published without +# an address, the podman machine's forwarder puts them on every interface, +# and the whole stack, demo credentials and all, is on the local network. +# The .localhost names resolve locally and prove nothing about that. +# +# This is a self-check and touches no other host. The address probed is the +# machine's own: python asks the routing table which local address would be +# used toward 192.0.2.1, an address RFC 5737 reserves for documentation that +# is never routed, and a UDP connect sends nothing. Each port must then +# refuse a connection on that address while answering on 127.0.0.1. It +# proves the ports are not exposed on the interface the stack shares with +# the network; it does not try to prove more than that. The address is +# printed only on failure, so a pasted validate run does not carry it. + +echo "" +echo "==> Host port boundary" +HOST_ADDR=$(python3 -c ' +import socket +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.connect(("192.0.2.1", 1)) +print(s.getsockname()[0]) +' 2>/dev/null || echo "") + +check_loopback_only() { + local port="$1" + local description="$2" + local lan_status loop_status + if [ -z "${HOST_ADDR}" ] || [ "${HOST_ADDR}" = "127.0.0.1" ]; then + echo " SKIP ${description} loopback-only on ${port} (no network address to probe)" + return + fi + lan_status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "http://${HOST_ADDR}:${port}/" 2>/dev/null || true) + loop_status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "http://127.0.0.1:${port}/" 2>/dev/null || true) + lan_status="${lan_status:-000}" + loop_status="${loop_status:-000}" + if [ "${lan_status}" = "000" ] && [ "${loop_status}" != "000" ]; then + pass "${description} loopback-only on ${port}" + elif [ "${lan_status}" != "000" ]; then + fail "${description} on ${port} answered on ${HOST_ADDR} (status ${lan_status}); host ports must be bound to 127.0.0.1" + else + fail "${description} on ${port} did not answer on 127.0.0.1 either (status ${loop_status})" + fi +} + +check_loopback_only "${INGRESS_PORT}" "ingress" +check_loopback_only "${K8S_API_PORT}" "kubernetes api" +check_loopback_only "${REGISTRY_PORT}" "registry" + +# --- Flagsmith seed --- + +echo "" +echo "==> Flagsmith environments" +check_flagsmith_environment "staging" +check_flagsmith_environment "production" # --- Metrics endpoints (via port-forward) --- @@ -231,12 +329,12 @@ echo "==> Prometheus targets" # which lands a minute or two after the apps Kustomization reconciles, and a # target it has just discovered reports "unknown" health until its first # scrape. Run straight after a bring-up the service targets are not there or -# not yet scraped, so the list is polled until all three are up. +# not yet scraped, so the list is polled until all four are up. PROM_TARGETS='{"data":{"activeTargets":[]}}' for _ in $(seq 1 30); do PROM_TARGETS=$(curl -s --max-time 5 "http://prometheus.localhost:${INGRESS_PORT}/api/v1/targets?state=active" || echo '{"data":{"activeTargets":[]}}') NOT_UP=0 - for job in go-api go-grpc rust-inventory; do + for job in go-api go-grpc rust-inventory flagsmith; do [ "$(prom_fact "${PROM_TARGETS}" target-health "${job}")" = "up" ] || NOT_UP=1 done if [ "${NOT_UP}" -eq 0 ]; then @@ -262,6 +360,7 @@ check_prom_target() { check_prom_target "go-api scrape" "go-api" check_prom_target "go-grpc scrape" "go-grpc" check_prom_target "rust-inventory scrape" "rust-inventory" +check_prom_target "flagsmith scrape" "flagsmith" check_prom_target "kps-prometheus self-scrape" "kps-prometheus" check_prom_target "kps-alertmanager" "kps-alertmanager" check_prom_target "grafana" "kube-prometheus-stack-grafana" @@ -421,7 +520,7 @@ fi echo "" echo "==> Flux reconciliation" -for ks in apps monitoring; do +for ks in apps monitoring monitoring-flux flagsmith; do READY=$(kubectl -n "${FLUX_NS}" get kustomization "${ks}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") if [ "${READY}" = "True" ]; then pass "kustomization/${ks} ready" From 981c963e4af0cf23f8202f7b63884d4650224811 Mon Sep 17 00:00:00 2001 From: Andrew McCabe Date: Tue, 22 Sep 2026 22:52:40 -0400 Subject: [PATCH 2/3] fix(k8s): stop Flux re-running the migration Jobs every ten minutes Each migration Job had ttlSecondsAfterFinished: 600. Kubernetes deleted the completed Job after ten minutes, Flux re-created it on the next reconcile, and the migrations ran on a loop for as long as the cluster was up. The TTL is gone from all four Jobs; completed Jobs stay, and the versioned names already handle a change. validate-stack.sh and smoke-test.sh no longer pass a missing Job as garbage-collected. docs/topology.md takes back one sentence: Flagsmith does exit at startup when it comes up ahead of its Job and finds no schema. Assisted-by: Claude:claude-fable-5-1 [claude-code] Signed-off-by: Andrew McCabe --- README.md | 2 +- docs/services.md | 5 +++-- docs/topology.md | 3 +-- k8s/apps/base/migrate.yaml | 6 ++---- k8s/apps/base/prometheus-rules.yaml | 4 ++-- k8s/infra/flagsmith/migrate.yaml | 1 - .../overlay/migrate-inventory-v2.yaml | 1 - scripts/smoke-test.sh | 9 ++------- scripts/validate-stack.sh | 9 ++------- 9 files changed, 13 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 0dc77c6..5bb9591 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,7 @@ There are four layers of testing. All exit non-zero on failure. run the `rust-inventory` handlers that answer before a query: authentication, request validation, and liveness. - **`scripts/smoke-test.sh`** checks service behavior on a live stack: - migration Jobs succeeded or already garbage-collected, health and readiness, + migration Jobs succeeded, health and readiness, authentication rejection, the seeded inventory, order creation across the full call graph, insufficient stock, an unknown item, and the UI proxy paths. - **`scripts/validate-stack.sh`** checks the infrastructure: pod health, diff --git a/docs/services.md b/docs/services.md index 29de52e..cf3c962 100644 --- a/docs/services.md +++ b/docs/services.md @@ -210,8 +210,9 @@ filesystem directory after the embedded set, against the same `goose_db_version` table, which is how scenario 2 adds a migration without a variant image. Job names carry a version suffix: when migration content changes, the suffix is bumped, so Flux creates a new -Job and leaves the completed one alone. Finished Jobs are -garbage-collected after ten minutes. +Job and leaves the completed one alone. Completed Jobs stay: a TTL would +delete them, and Flux would re-create and re-run them on its next +reconcile. On a fresh cluster the services may restart once or twice while the migration Jobs run. Those restarts show up in diff --git a/docs/topology.md b/docs/topology.md index a9f787c..04d7783 100644 --- a/docs/topology.md +++ b/docs/topology.md @@ -190,8 +190,7 @@ A single PostgreSQL Deployment holds all three databases. `rust-inventory` owns `inventory`, `go-api` owns `orders`, and the Flagsmith server in its own namespace owns `flagsmith`. No service creates its own schema. Migration Jobs do this before the services need -it. `go-api` and `rust-inventory` fail at startup if the schema is -missing; Flagsmith stays unready until its Job has run. +it, and a service fails at startup if the schema is missing. Data resides on a 100Mi local-path volume. It survives pod restarts and `k3d cluster stop`, and is lost on `k3d cluster delete`, which is what diff --git a/k8s/apps/base/migrate.yaml b/k8s/apps/base/migrate.yaml index 9d025fa..c9fe904 100644 --- a/k8s/apps/base/migrate.yaml +++ b/k8s/apps/base/migrate.yaml @@ -1,7 +1,7 @@ # Migration Jobs are named with a version suffix. When migration content # changes, bump the suffix (for example v1 to v2). Flux then creates a new Job -# and leaves the completed one alone. Finished Jobs are garbage-collected after -# 10 minutes. +# and leaves the completed one alone. Completed Jobs stay: a TTL would delete +# them, and Flux would re-create and re-run them on its next reconcile. apiVersion: batch/v1 kind: Job metadata: @@ -9,7 +9,6 @@ metadata: namespace: example-stack spec: backoffLimit: 3 - ttlSecondsAfterFinished: 600 template: metadata: labels: @@ -36,7 +35,6 @@ metadata: namespace: example-stack spec: backoffLimit: 3 - ttlSecondsAfterFinished: 600 template: metadata: labels: diff --git a/k8s/apps/base/prometheus-rules.yaml b/k8s/apps/base/prometheus-rules.yaml index f1280bd..4d31f80 100644 --- a/k8s/apps/base/prometheus-rules.yaml +++ b/k8s/apps/base/prometheus-rules.yaml @@ -146,8 +146,8 @@ spec: description: "Container {{ $labels.container }} in pod {{ $labels.pod }} has restarted {{ $value }} times in the last 10 minutes." # A Job pod that has finished is never ready again, so completed and - # failed pods are excluded. Without this the migration Jobs keep the - # alert firing until their ten minute garbage collection. + # failed pods are excluded. Without this the migration Jobs' pods + # would keep the alert firing for as long as they exist. - alert: ExampleAppNotReady expr: | kube_pod_status_ready{namespace="example-stack", condition="true"} == 0 diff --git a/k8s/infra/flagsmith/migrate.yaml b/k8s/infra/flagsmith/migrate.yaml index 32944c8..2c7fd1a 100644 --- a/k8s/infra/flagsmith/migrate.yaml +++ b/k8s/infra/flagsmith/migrate.yaml @@ -14,7 +14,6 @@ metadata: namespace: flagsmith spec: backoffLimit: 3 - ttlSecondsAfterFinished: 600 template: metadata: labels: diff --git a/scenarios/scenario-2-bad-migration/overlay/migrate-inventory-v2.yaml b/scenarios/scenario-2-bad-migration/overlay/migrate-inventory-v2.yaml index ff79bed..4aa115f 100644 --- a/scenarios/scenario-2-bad-migration/overlay/migrate-inventory-v2.yaml +++ b/scenarios/scenario-2-bad-migration/overlay/migrate-inventory-v2.yaml @@ -8,7 +8,6 @@ metadata: namespace: example-stack spec: backoffLimit: 3 - ttlSecondsAfterFinished: 600 template: metadata: labels: diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index b5d70ad..7bb8afd 100644 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -122,14 +122,9 @@ check_grpc() { # --- Migration jobs --- echo "==> Migration jobs" -# Finished Jobs are garbage-collected ten minutes after completion, so on a -# stack that has been up a while the Job is gone. Its absence is not a failure: -# the seed-data checks below prove the migration ran. +# Nothing deletes a completed Job, so a missing one means Flux never applied +# it. for job in migrate-inventory-v1 migrate-orders-v1; do - if ! kubectl -n "${NAMESPACE}" get job "${job}" >/dev/null 2>&1; then - echo " PASS ${job} already garbage-collected" - continue - fi STATUS=$(kubectl -n "${NAMESPACE}" get job "${job}" -o jsonpath='{.status.succeeded}' 2>/dev/null || echo "0") if [ "${STATUS}" = "1" ]; then echo " PASS ${job} succeeded" diff --git a/scripts/validate-stack.sh b/scripts/validate-stack.sh index c57af62..f476b26 100644 --- a/scripts/validate-stack.sh +++ b/scripts/validate-stack.sh @@ -172,17 +172,12 @@ check_pods "${NAMESPACE}" "app=postgres" "postgres" echo "" echo "==> Migration jobs" -# Finished Jobs are garbage-collected ten minutes after completion; a missing -# Job is not a failure, and the seed data the smoke test checks is the proof -# the migration ran. +# Nothing deletes a completed Job, so a missing one means Flux never applied +# it. check_job() { local ns="$1" local name="$2" local succeeded - if ! kubectl -n "${ns}" get job "${name}" >/dev/null 2>&1; then - pass "${name} already garbage-collected" - return 0 - fi succeeded=$(kubectl -n "${ns}" get job "${name}" -o jsonpath='{.status.succeeded}' 2>/dev/null || echo "0") if [ "${succeeded}" = "1" ]; then pass "${name} succeeded" From aa7e43127625fa8948f99533f0f538912e701ce7 Mon Sep 17 00:00:00 2001 From: Andrew McCabe Date: Tue, 22 Sep 2026 23:47:47 -0400 Subject: [PATCH 3/3] fix(flagsmith): point the dashboard's own flags at the stack Blanking FLAGSMITH_ON_FLAGSMITH_API_KEY and _API_URL stopped the dashboard's call to Flagsmith's hosted service, but the frontend's only failure handler for its own flags renders the Maintenance page, so the dashboard was unusable. Follow Flagsmith's "Running Flagsmith on Flagsmith" docs instead. setup.sh finds or creates a flagsmith-dashboard project with one environment, writes its client-side key to a flagsmith-dashboard Secret in the flagsmith namespace, and restarts Flagsmith when the key changed. The Deployment reads the key from that Secret, optionally, and points the API URL at the stack's ingress, which setup.sh already rewrites for --ingress-port. validate-stack.sh checks that /config/project-overrides serves the key. Assisted-by: Claude:claude-opus-5 [claude-code] Signed-off-by: Andrew McCabe --- README.md | 6 +- docs/services.md | 12 +++ docs/topology.md | 15 ++-- k8s/infra/flagsmith/deployment.yaml | 26 ++++--- scripts/setup.sh | 115 +++++++++++++++++++++++++--- scripts/validate-stack.sh | 23 +++++- 6 files changed, 166 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 5bb9591..9845750 100644 --- a/README.md +++ b/README.md @@ -112,8 +112,10 @@ cluster, and the third-party telemetry defaults that do so are turned off in the manifests - Grafana usage reporting and update checks - Forgejo avatar fetching -- Flagsmith's telemetry heartbeat, and the feature flags its own dashboard - would otherwise fetch from Flagsmith's hosted service +- Flagsmith's telemetry heartbeat. Its dashboard's own feature flags are read + from the stack's Flagsmith rather than Flagsmith's hosted service, as + Flagsmith's docs describe; a dashboard opened during bring-up, before the + seed, still reads them from the hosted service Flux still re-fetches the chart index hourly, and the other components have not been audited. This project is not air-gapped; block egress at the podman diff --git a/docs/services.md b/docs/services.md index cf3c962..c78c2ce 100644 --- a/docs/services.md +++ b/docs/services.md @@ -249,6 +249,18 @@ in the manifests Flux reconciles; `setup.sh` writes the Secret with alone, and the `app.kubernetes.io/managed-by` label on it records which script owns it. Nothing mounts it yet. +The dashboard reads its own feature flags from a Flagsmith project too, the +vendor's hosted one by default. The seed creates a second organization and +project, both `flagsmith-dashboard`, with one environment, `dashboard`, and +no flags, since the dashboard has a built-in default for each. The +organization exists because Flagsmith's free plan, the default when +self-hosting too, allows one project per organization. Its client-side key +goes into a +`flagsmith-dashboard` Secret in the `flagsmith` namespace, under +`client-key`, which the Deployment reads as `FLAGSMITH_ON_FLAGSMITH_API_KEY`; +`setup.sh` restarts Flagsmith when the key changes, because Django reads it +only at startup. + The bootstrap login is `bootstrap@flagsmith.local` with the password `--flagsmith-password` sets, `example-stack-demo` by default. It is a demo-only credential like every other one in this stack. Flagsmith runs diff --git a/docs/topology.md b/docs/topology.md index 04d7783..3afc42d 100644 --- a/docs/topology.md +++ b/docs/topology.md @@ -20,7 +20,7 @@ On a podman machine with 4 CPUs and 8 GiB of memory the stack's containers peak just under 3 GiB of memory together when starting. The two k3s nodes each burst past a full core while the images load, the first run takes under ten minutes from cold start and about -seven minutes with the images already built. A re-run reconciles in +nine minutes with the images already built. A re-run reconciles in under two minutes. The cluster takes about 5 GiB of the machine's disk while it is up and cleans that upon teardown. By default teardown keeps the built images and the k3s image, which are under 1 GiB, plus @@ -101,14 +101,19 @@ reconciles: a bootstrap account, the `example-stack` organization and project, and the `staging` and `production` environments. The account's API token and the two client-side environment keys go into a `flagsmith-bootstrap` Secret in the `example-stack` namespace, under -the keys `admin-token`, `staging`, and `production`. +the keys `admin-token`, `staging`, and `production`. A second +organization and project, both `flagsmith-dashboard`, hold the +Flagsmith dashboard's own feature flags; its client-side key goes into a `flagsmith-dashboard` Secret in +the `flagsmith` namespace, under `client-key`, and Flagsmith is +restarted to read it. `forgejo-bootstrap` and `flagsmith-bootstrap` are the only two Kubernetes objects in the `example-stack` namespace that a script -writes rather than Flux. -Both hold values minted while the stack comes up, which cannot be +writes rather than Flux; the third Secret a script writes, +`flagsmith-dashboard`, is in the `flagsmith` namespace. +All three hold values minted while the stack comes up, which cannot be written into manifests ahead of time. Flux prunes only what it applied, -so neither is removed by a reconcile, and both carry an +so none is removed by a reconcile, and all three carry an `app.kubernetes.io/managed-by=setup.sh` label saying so. The `apps` and `monitoring-flux` kustomizations depend on `monitoring`, diff --git a/k8s/infra/flagsmith/deployment.yaml b/k8s/infra/flagsmith/deployment.yaml index 8dcc1de..0e45b52 100644 --- a/k8s/infra/flagsmith/deployment.yaml +++ b/k8s/infra/flagsmith/deployment.yaml @@ -68,20 +68,22 @@ spec: # so both containers of this pair are configured the same way. - name: ENABLE_TELEMETRY value: "false" - # The dashboard runs its own feature flags against Flagsmith's - # hosted service, on every page load, before anyone has logged in. - # Both settings default to None, and the view that builds the - # frontend's config omits a None, which leaves the key and URL the - # self-hosted bundle hardcodes in place - # (frontend/env/project_selfhosted.js). Empty strings are not None, - # so they replace them. The key is what stops the call: the SDK - # refuses an empty environment key before it opens a connection. - # The URL is blanked with it because the SDK's own fallback is the - # same hosted address, so clearing that alone would change nothing. + # The dashboard reads its own feature flags from a Flagsmith, the + # vendor's hosted one by default. These point it at this one, per + # Flagsmith's "Running Flagsmith on Flagsmith" docs. The browser + # calls the URL, so it names the host ingress, and setup.sh sets + # the port for --ingress-port. The key belongs to the + # flagsmith-dashboard project setup.sh seeds, so it is optional: + # until the seed's restart, a dashboard opened during bring-up + # uses the vendor default. - name: FLAGSMITH_ON_FLAGSMITH_API_URL - value: "" + value: "http://flagsmith.localhost:8090/api/v1/" - name: FLAGSMITH_ON_FLAGSMITH_API_KEY - value: "" + valueFrom: + secretKeyRef: + name: flagsmith-dashboard + key: client-key + optional: true # One of local, dev, staging, production. It picks defaults inside # Flagsmith and is not the name of a Flagsmith environment. - name: ENVIRONMENT diff --git a/scripts/setup.sh b/scripts/setup.sh index d580329..b4b5079 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -5,7 +5,8 @@ set -euo pipefail # in-cluster Forgejo holding the manifests, Flux reconciling from it, the # monitoring stack, the services, a seeded Flagsmith, and a smoke test. The # Forgejo and Flagsmith access tokens it mints are left behind in the -# forgejo-bootstrap and flagsmith-bootstrap Secrets. +# forgejo-bootstrap and flagsmith-bootstrap Secrets, and the key the Flagsmith +# dashboard reads its own feature flags with in the flagsmith-dashboard Secret. # # Phases are ordered so a failure costs as little as possible. The registry and # the image builds run before any cluster exists, so a broken build never @@ -95,6 +96,10 @@ MONITORING_NS="monitoring" FLAGSMITH_NS="flagsmith" FLAGSMITH_ORG="example-stack" FLAGSMITH_PROJECT="example-stack" +# The project the Flagsmith dashboard reads its own feature flags from, and +# the one environment in it whose client-side key the dashboard is given. +FLAGSMITH_DASHBOARD_PROJECT="flagsmith-dashboard" +FLAGSMITH_DASHBOARD_ENV="dashboard" # Synthetic address for the demo-only bootstrap account. Nothing sends mail. FLAGSMITH_ADMIN_EMAIL="bootstrap@flagsmith.local" # Local port for the bootstrap port-forward: a port unlikely to be in use on @@ -196,11 +201,17 @@ sys.exit(1) ' "$@" 2>/dev/null } -# Writes one bootstrap Secret into the services' namespace. These values are -# minted while the stack comes up, so they cannot live in the manifests Flux -# reconciles; Flux prunes only what it applied, so a Secret written here -# survives every reconcile. The label says which script owns it. +# Writes one bootstrap Secret, into the services' namespace unless a leading +# -n names another. These values are minted while the stack comes up, so they +# cannot live in the manifests Flux reconciles; Flux prunes only what it +# applied, so a Secret written here survives every reconcile. The label says +# which script owns it. write_bootstrap_secret() { + local ns="${NAMESPACE}" + if [ "$1" = "-n" ]; then + ns="$2" + shift 2 + fi local name="$1" local app="$2" shift 2 @@ -210,10 +221,10 @@ write_bootstrap_secret() { for literal in "$@"; do args+=(--from-literal="${literal}") done - kubectl -n "${NAMESPACE}" create secret generic "${name}" \ + kubectl -n "${ns}" create secret generic "${name}" \ "${args[@]}" --dry-run=client -o yaml \ - | kubectl -n "${NAMESPACE}" apply -f - > /dev/null - kubectl -n "${NAMESPACE}" label secret "${name}" --overwrite \ + | kubectl -n "${ns}" apply -f - > /dev/null + kubectl -n "${ns}" label secret "${name}" --overwrite \ "app=${app}" "app.kubernetes.io/managed-by=setup.sh" > /dev/null } @@ -567,8 +578,9 @@ cp -R "${REPO_ROOT}/k8s/infra/monitoring-flux" "${WORK_DIR}/k8s/infra/monitoring cp -R "${REPO_ROOT}/k8s/infra/flagsmith" "${WORK_DIR}/k8s/infra/flagsmith" # The landing page and the UI print browser URLs, which carry the host ingress -# port, and Flagsmith builds the links in its own pages from FLAGSMITH_DOMAIN. -# The manifests hold the default; the seeded copies get whatever +# port, Flagsmith builds the links in its own pages from FLAGSMITH_DOMAIN, and +# the Flagsmith dashboard calls FLAGSMITH_ON_FLAGSMITH_API_URL from the +# browser. The manifests hold the default; the seeded copies get whatever # --ingress-port was given, so those links stay clickable. if [ "${INGRESS_PORT}" != "8090" ]; then echo " Rewriting landing page, UI, and Flagsmith links for ingress port ${INGRESS_PORT}" @@ -705,7 +717,8 @@ kubectl -n "${NAMESPACE}" rollout status deployment --timeout=120s || true # password and prints a reset link. The account is created through the signup # endpoint instead, which takes a password, answers with the API token the # rest of the seed uses, and accepts superuser on a self-hosted instance that -# has no users yet. No feature flags are created; nothing reads one yet. +# has no users yet. No feature flags are created: no service reads one yet, +# and the dashboard has a built-in default for each of its own. mark "flagsmith reconcile wait begin (the second long pull: the Flagsmith image)" echo "==> Waiting for the flagsmith Kustomization to reconcile" @@ -846,10 +859,90 @@ write_bootstrap_secret "flagsmith-bootstrap" "flagsmith" \ "production=${FLAGSMITH_PRODUCTION_KEY}" \ "staging=${FLAGSMITH_STAGING_KEY}" +# The dashboard reads its own feature flags from a Flagsmith, the vendor's +# hosted one unless the Deployment names another. This project is what it is +# pointed at instead, per Flagsmith's "Running Flagsmith on Flagsmith" docs. +# It lives in its own organization because Flagsmith's free plan, the default +# on a self-hosted instance too, allows one project per organization. +# The API creates no environment with a project (the dashboard's own +# create-project form adds them), so the one environment is found or created +# by name like the others. +echo "==> Ensuring the Flagsmith organization and project for the dashboard's own flags exist" +FLAGSMITH_DASHBOARD_ORG_ID=$(flagsmith_fact "$(flagsmith_api GET /organisations/)" find "${FLAGSMITH_DASHBOARD_PROJECT}" id || echo "") +if [ -n "${FLAGSMITH_DASHBOARD_ORG_ID}" ]; then + echo " Organization '${FLAGSMITH_DASHBOARD_PROJECT}' already exists" +else + body='{"name":"'"${FLAGSMITH_DASHBOARD_PROJECT}"'"}' + response=$(flagsmith_api POST /organisations/ "${body}") + FLAGSMITH_DASHBOARD_ORG_ID=$(flagsmith_fact "${response}" field id || echo "") + if [ -z "${FLAGSMITH_DASHBOARD_ORG_ID}" ]; then + echo "ERROR: could not create the Flagsmith organization '${FLAGSMITH_DASHBOARD_PROJECT}'" >&2 + echo " Flagsmith answered: ${response}" >&2 + exit 1 + fi + echo " Organization '${FLAGSMITH_DASHBOARD_PROJECT}' created" +fi +FLAGSMITH_DASHBOARD_PROJECT_ID=$(flagsmith_fact "$(flagsmith_api GET /projects/)" find "${FLAGSMITH_DASHBOARD_PROJECT}" id || echo "") +if [ -n "${FLAGSMITH_DASHBOARD_PROJECT_ID}" ]; then + echo " Project '${FLAGSMITH_DASHBOARD_PROJECT}' already exists" +else + body='{"name":"'"${FLAGSMITH_DASHBOARD_PROJECT}"'","organisation":'"${FLAGSMITH_DASHBOARD_ORG_ID}"'}' + response=$(flagsmith_api POST /projects/ "${body}") + FLAGSMITH_DASHBOARD_PROJECT_ID=$(flagsmith_fact "${response}" field id || echo "") + if [ -z "${FLAGSMITH_DASHBOARD_PROJECT_ID}" ]; then + echo "ERROR: could not create the Flagsmith project '${FLAGSMITH_DASHBOARD_PROJECT}'" >&2 + echo " Flagsmith answered: ${response}" >&2 + exit 1 + fi + echo " Project '${FLAGSMITH_DASHBOARD_PROJECT}' created" +fi + +FLAGSMITH_DASHBOARD_ENVS=$(flagsmith_api GET "/environments/?project=${FLAGSMITH_DASHBOARD_PROJECT_ID}") +FLAGSMITH_DASHBOARD_KEY=$(flagsmith_fact "${FLAGSMITH_DASHBOARD_ENVS}" find "${FLAGSMITH_DASHBOARD_ENV}" api_key || echo "") +if [ -n "${FLAGSMITH_DASHBOARD_KEY}" ]; then + echo " Environment '${FLAGSMITH_DASHBOARD_ENV}' already exists" +else + body='{"name":"'"${FLAGSMITH_DASHBOARD_ENV}"'","project":'"${FLAGSMITH_DASHBOARD_PROJECT_ID}"'}' + response=$(flagsmith_api POST /environments/ "${body}") + FLAGSMITH_DASHBOARD_KEY=$(flagsmith_fact "${response}" field api_key || echo "") + if [ -z "${FLAGSMITH_DASHBOARD_KEY}" ]; then + echo "ERROR: could not create the Flagsmith environment '${FLAGSMITH_DASHBOARD_ENV}'" >&2 + echo " Flagsmith answered: ${response}" >&2 + exit 1 + fi + echo " Environment '${FLAGSMITH_DASHBOARD_ENV}' created" +fi + +# What the running server hands the dashboard, read before the port-forward +# closes. Comparing against this rather than the Secret's old value means a +# run interrupted between the Secret write and the restart, or a Secret +# edited by hand, still ends with a restart. +FLAGSMITH_DASHBOARD_SERVED=$(curl -s --max-time 5 \ + "http://localhost:${FLAGSMITH_PF_PORT}/config/project-overrides" 2>/dev/null || true) + kill "${FLAGSMITH_PF_PID}" 2>/dev/null || true wait "${FLAGSMITH_PF_PID}" 2>/dev/null || true FLAGSMITH_PF_PID="" +# The Deployment reads the key from this Secret, optionally, so Flagsmith +# starts before the seed has run. Django reads its settings once, at process +# start, so a key it is not already serving reaches the dashboard only +# through a restart. +echo "==> Writing the dashboard's client-side key into the ${FLAGSMITH_NS} namespace" +write_bootstrap_secret -n "${FLAGSMITH_NS}" "flagsmith-dashboard" "flagsmith" \ + "client-key=${FLAGSMITH_DASHBOARD_KEY}" +case "${FLAGSMITH_DASHBOARD_SERVED}" in + *"\"flagsmith\": \"${FLAGSMITH_DASHBOARD_KEY}\""*) + echo " Flagsmith already serves this key to the dashboard, not restarting" + ;; + *) + mark "flagsmith restart begin" + echo "==> Restarting Flagsmith so the dashboard reads its own flags from this instance" + kubectl -n "${FLAGSMITH_NS}" rollout restart deployment/flagsmith + kubectl -n "${FLAGSMITH_NS}" rollout status deployment/flagsmith --timeout=5m + ;; +esac + # ============================================================================= # Phase D: Smoke # ============================================================================= diff --git a/scripts/validate-stack.sh b/scripts/validate-stack.sh index f476b26..04428df 100644 --- a/scripts/validate-stack.sh +++ b/scripts/validate-stack.sh @@ -4,7 +4,8 @@ set -euo pipefail # Checks that the stack's infrastructure is wired correctly: pods running, # ingress resolving, the three host ports answering on 127.0.0.1 and nowhere # else, metrics endpoints responding, Prometheus scraping every expected -# target, both seeded Flagsmith environments answering for their keys, Flux +# target, both seeded Flagsmith environments answering for their keys, the +# Flagsmith dashboard reading its own flags from this Flagsmith, Flux # reconciling. Complements smoke-test.sh, which checks service behavior. # # Invoke with: bash scripts/validate-stack.sh @@ -157,6 +158,25 @@ check_flagsmith_environment() { fi } +# The dashboard learns where to read its own feature flags from +# /config/project-overrides, a script that sets window.projectOverrides from +# the server's settings. The key there has to be the one the bring-up wrote +# into the flagsmith-dashboard Secret: a missing one means the dashboard +# still reads the vendor's hosted Flagsmith, and a different one means the +# server has not been restarted since the Secret changed. +check_flagsmith_dashboard() { + local key + key=$(kubectl -n "${FLAGSMITH_NS}" get secret flagsmith-dashboard \ + -o 'jsonpath={.data.client-key}' 2>/dev/null | base64 --decode 2>/dev/null || echo "") + if [ -z "${key}" ]; then + fail "flagsmith dashboard flags (no 'client-key' in the flagsmith-dashboard Secret)" + return + fi + check_body "flagsmith dashboard reads its own flags from this Flagsmith" \ + "http://flagsmith.localhost:${INGRESS_PORT}/config/project-overrides" \ + "\"flagsmith\": \"${key}\"" +} + # --- Pods --- echo "==> Pod health" @@ -275,6 +295,7 @@ echo "" echo "==> Flagsmith environments" check_flagsmith_environment "staging" check_flagsmith_environment "production" +check_flagsmith_dashboard # --- Metrics endpoints (via port-forward) ---