diff --git a/.github/workflows/cache-cleanup.yml b/.github/workflows/cache-cleanup.yml new file mode 100644 index 00000000..1e32cc18 --- /dev/null +++ b/.github/workflows/cache-cleanup.yml @@ -0,0 +1,28 @@ +name: Cache cleanup + +# Pull request runs only restore caches, but the BuildKit and tool caches of a closed PR +# ref are dead weight against the 10 GB repository quota, so drop them when the PR closes. +on: + pull_request: + types: [closed] + +permissions: + actions: write + contents: read + +jobs: + delete-pr-caches: + # The token of a pull request from a fork is read-only, so the delete would fail; such a + # run only restores caches and has nothing of its own to remove. + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + steps: + - name: Delete the caches of this pull request + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + REF: refs/pull/${{ github.event.pull_request.number }}/merge + run: | + ids="$(gh cache list --ref "$REF" --limit 100 --json id --jq '.[].id')" + if [ -z "$ids" ]; then echo "no caches for $REF"; exit 0; fi + for id in $ids; do gh cache delete "$id" || true; done diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 306bde49..a6f14cb0 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -66,10 +66,19 @@ jobs: with: distribution: temurin java-version: "25" - cache: maven + # Cache policy (see the cache note above the cpp job): legs only restore. The one shared + # key is written by the `warm-maven` job on main. + - name: Restore the Maven repository + uses: actions/cache/restore@v6.1.0 + with: + path: ~/.m2/repository + key: maven-${{ runner.os }}-${{ hashFiles('examples/**/pom.xml') }} + restore-keys: | + maven-${{ runner.os }}- - name: Build env: DIR: ${{ matrix.dir }} + MAVEN_ARGS: -Daether.transport.http.retryHandler.count=8 -Daether.transport.http.retryHandler.interval=5000 -Daether.transport.http.retryHandler.serviceUnavailable=429,503 run: | test -f "$DIR/pom.xml" || { echo "::error::no pom.xml in $DIR"; exit 1; } mvn -B -ntp -q -f "$DIR/pom.xml" package -DskipTests @@ -110,7 +119,16 @@ jobs: - uses: actions/setup-go@v7 with: go-version-file: ${{ matrix.dir }}/go.mod - cache-dependency-path: ${{ matrix.dir }}/go.sum + cache: false + # Cache policy (see the cache note above the cpp job): legs only restore. The one shared + # key is written by the `warm-go` job on main. + - name: Restore the Go module cache + uses: actions/cache/restore@v6.1.0 + with: + path: ~/go/pkg/mod + key: go-${{ runner.os }}-${{ hashFiles('examples/**/go.sum') }} + restore-keys: | + go-${{ runner.os }}- - name: Vet and build working-directory: ${{ matrix.dir }} run: go vet ./... && go build -o /dev/null ./... && go test ./... @@ -138,20 +156,66 @@ jobs: pip install -r requirements.txt python -m compileall -q . + # Cache policy. The repository gets 10 GB of Actions cache, shared by every branch. The rule: + # every run RESTORES, only a push to main WRITES, and every ecosystem keeps a small, fixed + # number of entries, whatever the number of projects. + # - Maven and Go: ONE shared entry each, keyed on the hash of every pom.xml / go.sum under + # examples/. The build legs only restore it (actions/cache/restore). The `warm-maven` / + # `warm-go` jobs on main resolves the dependencies of ALL projects once and saves it, then deletes + # the superseded entries. A leg cannot be the writer: it downloads only its own + # dependencies, so whichever leg won the save would freeze a cache holding one project. + # Tradeoff: any pom/go.sum change on main re-warms everything (Maven: 62 builds, + # Go: `go mod download` x31), which costs minutes of main-only CI time and never delays + # a pull request. Go keeps only the module cache; the build cache is not worth its bytes. + # - Conan: one entry per distinct conan.lock (22 of 31), pruned of sources, build trees and + # downloads (about 90% of the bytes; only the built packages are kept) before main saves + # it. The per-lock entries stay small enough to fit; one shared entry would need a warm + # job that compiles all 22 dependency trees (hours), or a single leg's packages would + # stand in for everyone's. + # - BuildKit image layers (type=gha, mode=min): cache-from always, cache-to on main only. + # - Sizing, measured with `gh api repos/.../actions/caches` before this change: unpruned + # Conan entries 210-668 MB (avg 408 MB, 20 entries); pruned is ~10% (25-caching/cpp: + # 1.6 GB -> 163 MB), budgeted here at 20% = 135 MB for the largest. Per-pom Maven entries + # were 56-350 MB and per-go.sum Go entries 22-248 MB (avg 111 MB), so those two are + # what outgrew the quota, not Conan. Steady state on main: + # Conan 22 x <=135 MB (typical 82) <= 3.0 GB + # Maven 1 shared union <= 1.0 GB (estimate) + # Go 1 shared module cache <= 0.6 GB (estimate) + # layers 27 blobs = 200 MB measured 0.25 GB + # Pages bundler cache <= 0.05 GB (estimate) + # typical 22 x 82 MB + 1.0 + 0.6 + 0.25 + 0.05 = ~3.7 GB; bound ~4.9 GB. + # Peak ~8 GB: a Conan key bump or bulk lock change leaves the superseded per-lock + # entries in place until they age out, and layer blobs stay 7 days unused. + # Measured: unpruned Conan entries (API), BuildKit layers. Partly measured: the prune + # ratio (one project, 25-caching/cpp). Estimated: Maven and Go unions, Pages. After the + # first merge, check the real saved Maven and Go entry sizes (`gh cache list --key`) + # and correct these numbers. PR refs hold nothing. + # - Hygiene: .github/workflows/cache-cleanup.yml deletes the caches of a PR ref when the + # PR closes; CLAUDE.md ("CI cache budget") has the commands for inspecting and clearing. + # # Each C++ service builds in its own Containerfile (Conan 2.33.0 + GCC 15 / C++23 on # UBI 10). The expensive part is compiling the Conan dependency tree (gRPC, Drogon, # OpenTelemetry C++, libpq...): 30+ minutes cold for the largest projects on 4 CPUs. Its # home is a BuildKit cache mount (id cndp-conan2), which neither the runner nor # type=gha carries to the next job, so the cache is round-tripped by hand: - # 1. actions/cache restores the directory `conan-cache`, keyed on the runner OS, the - # toolchain versions (UBI 10.2, GCC 15, Conan 2.33.0) and this project's conan.lock. - # A lock change misses the exact key and falls back to the newest cache of the same - # toolchain (restore-keys), so only the changed packages recompile. - # 2. buildkit-cache-dance injects that directory into the cache mount before the build - # and extracts it afterwards (skipped on an exact key hit, so nothing is re-saved). - # The image layers use type=gha,mode=min: mode=max across 31 scopes would exceed the 10 GB - # Actions cache quota, and the layers that matter are the Conan cache above. - # Bump the toolchain key when the Containerfile base image, GCC or Conan version changes. + # 1. actions/cache restores the directory `conan-cache`, keyed + # `-conan2-ubi10.2-gcc15-conan2.33.0-` + # (for example Linux-conan2-ubi10.2-gcc15-conan2.33.0-b4900...). A lock change misses + # the exact key and falls back to the newest cache of the same toolchain + # (restore-keys), so only the changed packages recompile. On a PR this is + # actions/cache/restore (nothing is saved); on main it is actions/cache, whose post + # step saves the directory when the exact key missed. + # 2. buildkit-cache-dance injects that directory into the cache mount before the build. + # Its post step extracts the mount afterwards, before the cache post step saves it; + # extraction is skipped everywhere except a push to main with an exact-key miss. + # 3. Just before that extraction, on main with a miss, a throwaway build prunes the mount + # and fails the job if no built package is left, so an empty cache is never saved. + # On a main miss the builder stage is forced to run (no-cache-filters): the type=gha layer + # cache can otherwise satisfy the stage from cache, BuildKit then skips the + # `conan install` RUN, the cache mount stays empty and an empty Conan cache would be saved + # under the exact key until the next conan.lock change. + # Bump the `conan2-ubi10.2-gcc15-conan2.33.0` part of CONAN_KEY below when the + # Containerfile base image, GCC or Conan version changes. # The build context is the service directory, or its parent cpp/ directory # for the multi-service examples; scripts/ci_changed_projects.py cpp-context # decides, mirroring each example's compose.yaml. @@ -161,6 +225,9 @@ jobs: if: needs.changes.outputs.cpp != '[]' runs-on: ubuntu-latest timeout-minutes: 120 + env: + # Toolchain part of the Conan cache key; the runner OS is added per step. + CONAN_KEY: conan2-ubi10.2-gcc15-conan2.33.0 strategy: fail-fast: false matrix: @@ -178,14 +245,24 @@ jobs: echo "scope=cpp-${DIR//\//-}" >> "$GITHUB_OUTPUT" - uses: docker/setup-buildx-action@v4 id: buildx - - name: Restore the Conan cache - id: conan-cache + - name: Restore the Conan cache (pull request) + id: conan-restore + if: github.ref != 'refs/heads/main' + uses: actions/cache/restore@v6.1.0 + with: + path: conan-cache + key: ${{ runner.os }}-${{ env.CONAN_KEY }}-${{ hashFiles(format('{0}/conan.lock', matrix.dir)) }} + restore-keys: | + ${{ runner.os }}-${{ env.CONAN_KEY }}- + - name: Restore the Conan cache (main; saved after the build on a miss) + id: conan-main + if: github.ref == 'refs/heads/main' uses: actions/cache@v6.1.0 with: path: conan-cache - key: conan-${{ runner.os }}-ubi10.2-gcc15-conan2.33.0-${{ hashFiles(format('{0}/conan.lock', matrix.dir)) }} + key: ${{ runner.os }}-${{ env.CONAN_KEY }}-${{ hashFiles(format('{0}/conan.lock', matrix.dir)) }} restore-keys: | - conan-${{ runner.os }}-ubi10.2-gcc15-conan2.33.0- + ${{ runner.os }}-${{ env.CONAN_KEY }}- - name: Inject the Conan cache into the BuildKit cache mount uses: reproducible-containers/buildkit-cache-dance@v3.4.0 with: @@ -194,14 +271,176 @@ jobs: { "conan-cache": { "target": "/root/.conan2", "id": "cndp-conan2" } } - skip-extraction: ${{ steps.conan-cache.outputs.cache-hit }} + skip-extraction: ${{ github.ref != 'refs/heads/main' || steps.conan-main.outputs.cache-hit == 'true' }} + # Exact tag (newest published on ghcr; the action's default is :latest). + utility-image: ghcr.io/containerd/busybox:1.36@sha256:7b3ccabffc97de872a30dfd234fd972a66d247c8cfc69b0550f276481852627c - uses: docker/build-push-action@v7 with: context: ${{ steps.ctx.outputs.context }} file: ${{ matrix.dir }}/Containerfile push: false cache-from: type=gha,scope=${{ steps.ctx.outputs.scope }} - cache-to: type=gha,mode=min,scope=${{ steps.ctx.outputs.scope }} + cache-to: ${{ github.ref == 'refs/heads/main' && format('type=gha,mode=min,scope={0}', steps.ctx.outputs.scope) || '' }} + # Every Containerfile has a `builder` stage (two also run conan in `schemagen`). + no-cache-filters: ${{ github.ref == 'refs/heads/main' && steps.conan-main.outputs.cache-hit != 'true' && 'builder,schemagen' || '' }} + - name: Prune the Conan cache mount before it is extracted and saved + if: github.ref == 'refs/heads/main' && steps.conan-main.outputs.cache-hit != 'true' + env: + BUILDER: ${{ steps.buildx.outputs.name }} + run: | + # Conan keeps build trees (p/b//b), recipe sources (p//s) and download + # archives (p/*/d) next to the built packages (p/b//p): about 90% of the bytes. + # A rebuild fetches them again, so only the packages are worth saving. Measured on + # 25-caching/cpp: 1.6 GB -> 163 MB, and the next conan install is all cache hits. + # The RUN fails (and so does the job) when no built package (p/b//p) is left, so + # an empty cache is never extracted and saved. On 25-caching/cpp 13 are left. + # The STAMP build arg keeps BuildKit from reusing a cached RUN; --no-cache would + # not do, because it gives the RUN an empty cache mount instead of the real one. + empty="$(mktemp -d)" + docker buildx build --builder "$BUILDER" --progress=plain \ + --build-arg "STAMP=$(date +%s%N)" -f - "$empty" <<'EOF' + FROM registry.access.redhat.com/ubi10/ubi-minimal:10.2-1791444377 + ARG STAMP + RUN --mount=type=cache,id=cndp-conan2,target=/c,sharing=locked \ + echo "$STAMP" >/dev/null && du -sh /c \ + && find /c/p -mindepth 2 -maxdepth 3 -type d \( -name b -o -name s -o -name d \) -prune -exec rm -rf {} + \ + && du -sh /c \ + && n="$(find /c/p/b -mindepth 2 -maxdepth 2 -type d -name p 2>/dev/null | wc -l)" \ + && echo "built packages left: $n" \ + && [ "$n" -gt 0 ] + EOF + + # Writes the shared Maven and Go caches (see the cache note above the cpp job). Main only, + # one job per ecosystem, so a slow or flaky warm-up never touches a pull request. A failed + # warm-up fails the run and saves nothing (the save is the last step); the restore of the + # previous entry plus the exact-key check make a successful run a no-op until a pom.xml or + # go.sum changes. Not part of `examples-ok`: PRs never run it. + warm-maven: + name: Warm the Maven cache + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 90 + concurrency: + group: warm-maven-${{ github.ref }} + cancel-in-progress: false + permissions: + actions: write + contents: read + env: + MAVEN_ARGS: -Daether.transport.http.retryHandler.count=8 -Daether.transport.http.retryHandler.interval=5000 -Daether.transport.http.retryHandler.serviceUnavailable=429,503 + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + steps: + - uses: actions/checkout@v7 + # Exact-key probe: on a hit nothing below runs. + - id: exact + uses: actions/cache/restore@v6.1.0 + with: + path: ~/.m2/repository + key: maven-${{ runner.os }}-${{ hashFiles('examples/**/pom.xml') }} + lookup-only: true + # Miss: start from the newest older entry so the warm-up only adds what changed. + - id: m2 + if: steps.exact.outputs.cache-hit != 'true' + uses: actions/cache/restore@v6.1.0 + with: + path: ~/.m2/repository + key: maven-${{ runner.os }}-${{ hashFiles('examples/**/pom.xml') }} + restore-keys: | + maven-${{ runner.os }}- + - if: steps.exact.outputs.cache-hit != 'true' + uses: actions/setup-java@v6 + with: + distribution: temurin + java-version: "25" + - name: Resolve the dependencies of every Maven project + if: steps.exact.outputs.cache-hit != 'true' + run: | + failed=0 + while IFS= read -r dir; do + echo "::group::$dir" + mvn -B -ntp -q -f "$dir/pom.xml" package -DskipTests || { echo "::error::mvn failed in $dir"; failed=1; } + echo "::endgroup::" + done < <(python3 scripts/ci_changed_projects.py --all | jq -r '.jvm[]') + exit "$failed" + - name: Save the Maven repository + if: steps.exact.outputs.cache-hit != 'true' + uses: actions/cache/save@v6.1.0 + with: + path: ~/.m2/repository + key: ${{ steps.m2.outputs.cache-primary-key }} + # actions/cache/save only warns when the save fails, so confirm the new entry exists + # before removing the old ones. `gh cache list --key` matches by key prefix. + - name: Delete the superseded Maven entries + if: steps.exact.outputs.cache-hit != 'true' + env: + KEEP: ${{ steps.m2.outputs.cache-primary-key }} + run: | + gh cache list --key "$KEEP" --limit 1000 --json key --jq '.[].key' | grep -qx "$KEEP" \ + || { echo "::warning::$KEEP not saved; keeping old entries"; exit 0; } + gh cache list --key "maven-$RUNNER_OS-" --limit 1000 --json id,key \ + --jq ".[] | select(.key != \"$KEEP\") | .id" | xargs -r -n1 gh cache delete || true + + warm-go: + name: Warm the Go cache + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 30 + concurrency: + group: warm-go-${{ github.ref }} + cancel-in-progress: false + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + steps: + - uses: actions/checkout@v7 + # Exact-key probe: on a hit nothing below runs. + - id: exact + uses: actions/cache/restore@v6.1.0 + with: + path: ~/go/pkg/mod + key: go-${{ runner.os }}-${{ hashFiles('examples/**/go.sum') }} + lookup-only: true + - id: gomod + if: steps.exact.outputs.cache-hit != 'true' + uses: actions/cache/restore@v6.1.0 + with: + path: ~/go/pkg/mod + key: go-${{ runner.os }}-${{ hashFiles('examples/**/go.sum') }} + restore-keys: | + go-${{ runner.os }}- + # Same toolchain as the build legs (every go.mod says go 1.27.2). + - if: steps.exact.outputs.cache-hit != 'true' + uses: actions/setup-go@v7 + with: + go-version-file: examples/01-cloud-native-principles/go/go.mod + cache: false + - name: Download the modules of every Go project + if: steps.exact.outputs.cache-hit != 'true' + run: | + failed=0 + while IFS= read -r dir; do + (cd "$dir" && go mod download) || { echo "::error::go mod download failed in $dir"; failed=1; } + done < <(python3 scripts/ci_changed_projects.py --all | jq -r '.go[]') + exit "$failed" + - name: Save the Go module cache + if: steps.exact.outputs.cache-hit != 'true' + uses: actions/cache/save@v6.1.0 + with: + path: ~/go/pkg/mod + key: ${{ steps.gomod.outputs.cache-primary-key }} + - name: Delete the superseded Go entries + if: steps.exact.outputs.cache-hit != 'true' + env: + KEEP: ${{ steps.gomod.outputs.cache-primary-key }} + run: | + gh cache list --key "$KEEP" --limit 1000 --json key --jq '.[].key' | grep -qx "$KEEP" \ + || { echo "::warning::$KEEP not saved; keeping old entries"; exit 0; } + gh cache list --key "go-$RUNNER_OS-" --limit 1000 --json id,key \ + --jq ".[] | select(.key != \"$KEEP\") | .id" | xargs -r -n1 gh cache delete || true # Compose files and verify.sh scripts are cheap to check, so each kind is one # job that loops over its selection and reports every failure at the end. diff --git a/CLAUDE.md b/CLAUDE.md index ceb5e522..887ad5bc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,6 +71,48 @@ Five services exchanging the `order.placed` fact: **order, payment, inventory, notification, shipping**. Local = Docker; prod = plain Kubernetes (no managed cloud) with CloudNativePG, Strimzi/Kafka, Apicurio, Istio, KEDA, Debezium, and OTel → the LGTM stack. +## Infrastructure images (pinned, fully qualified) + +All shared infra lives in `examples/_infra/` and uses fully qualified names, never `:latest`: +`docker.io/apache/kafka:4.3.1` (KRaft), `docker.io/library/postgres:18.6-alpine` (the volume mounts at +`/var/lib/postgresql`, not `/data`: PGDATA moved to `/var/lib/postgresql/18/docker`), +`docker.io/library/redis:8.10.2-alpine`, `quay.io/debezium/connect:3.7.0.Final`, +`docker.io/apicurio/apicurio-registry:3.3.3`, `ghcr.io/open-feature/flagd:v0.17.0`, +`docker.io/grafana/otel-lgtm:0.36.0`, `docker.io/envoyproxy/envoy:v1.39.3`. kafka-ui is gone: inspect topics +with `kcat` or `kafka-console-consumer.sh` inside the `kafka` container. Test tools: newman 6.2.3 + +newman-reporter-htmlextra 1.23.1, hey v0.1.5, ghz v0.121.0, conftest 0.71.1. +Redis 8 is tri-licensed (RSALv2 / SSPLv1 / AGPLv3); we take the free AGPLv3 open-source option, no license key needed. + +## CI cache budget + +The repo gets 10 GB of Actions cache. Policy in `examples.yml`: every run restores caches, only a push to +`main` writes them, and each ecosystem keeps a fixed number of entries. Maven and Go are one shared entry each +(`maven-Linux-`, `go-Linux-`), written by the main-only `warm-maven` / +`warm-go` jobs, which resolve every project and delete the superseded entry. Conan is one entry per +distinct `conan.lock` (22), pruned of sources, build trees and downloads before it is saved; on a main miss the +builder stage is forced to run so the cache mount is never empty. BuildKit layers are `type=gha,mode=min`. +Steady state on main: typical about 3.7 GB (22 x 82 MB Conan + Maven 1.0 + Go 0.6 + layers 0.25 + Pages 0.05), +bound 4.9 GB, peak about 8 GB when a Conan key bump or bulk lock change leaves superseded per-lock entries and +layer blobs age out after 7 days unused. Measured: unpruned Conan entries, BuildKit layers (27 blobs = 200 MB). +Partly measured: prune ratio (one project). Estimated: Maven <=1.0 GB, Go <=0.6 GB, Pages. After the first merge, +check the real saved Maven and Go entry sizes and correct the numbers (arithmetic in the `examples.yml` comment). +`cache-cleanup.yml` drops a PR ref's caches when the PR closes (same-repo PRs only). + +Check usage with `gh api repos/patterncatalyst/cloud-native-design-patterns/actions/cache/usage`. Drop one PR's +caches with +`gh cache list --ref refs/pull//merge --limit 100 --json id --jq '.[].id' | xargs -r -n1 gh cache delete`. +The pre-S8 entries use other key prefixes than the current ones, so deleting them is safe at any time, even +after the first main run with the new keys: `conan-Linux-` (old Conan), `setup-java-Linux-x64-maven-` and +`setup-go-Linux-x64-ubuntu24-go-` (old Maven and Go). Current keys start `Linux-conan2-`, `maven-Linux-`, `go-Linux-`. + +```bash +for k in conan-Linux- setup-java-Linux-x64-maven- setup-go-Linux-x64-ubuntu24-go-; do + gh cache list --key "$k" --limit 100 --json id --jq '.[].id' | xargs -r -n1 gh cache delete +done +``` + +Do not use `gh cache delete --all`: it also removes the entries the current policy just wrote. + ## Locked language stacks (do not substitute) - **Go** 1.27: `net/http` stdlib (1.22 ServeMux, no chi/gin/echo); `franz-go` (Kafka); pgx v5; diff --git a/_docs/09-api-registry.md b/_docs/09-api-registry.md index a81b7a64..854f0b18 100644 --- a/_docs/09-api-registry.md +++ b/_docs/09-api-registry.md @@ -49,7 +49,7 @@ curl -sf -X POST \ curl -s -o /dev/null -w "%{http_code}" -X POST \ http://apicurio.registry.svc/apis/registry/v3/groups/orders/artifacts/order-placed/versions \ -H "Content-Type: application/json" \ - --data-binary @order-placed.avsc # → 200 if compatible, 409 if not + --data-binary @order-placed.avsc # → 200 if compatible, 400 if not ``` The same call works from application or pipeline code in any language, returning @@ -72,7 +72,7 @@ public class RegistryClient { .body(schema) .retrieve() .toBodilessEntity(); - return resp.getStatusCode().value(); // 200 if compatible, 409 if rejected + return resp.getStatusCode().value(); // 200 if compatible, 400 if rejected } } ``` @@ -87,7 +87,7 @@ public interface RegistryClient { Response publish(@PathParam("group") String group, @PathParam("artifactId") String artifactId, byte[] schema); - // caller checks response.getStatus(): 200 = compatible, 409 = rejected + // caller checks response.getStatus(): 200 = compatible, 400 = rejected } ``` @@ -104,7 +104,7 @@ public class RegistryClient(HttpClient http) content.Headers.ContentType = new("application/json"); var resp = await http.PostAsync( $"{Registry}/groups/{group}/artifacts/{artifactId}/versions", content); - return (int)resp.StatusCode; // 200 if compatible, 409 if rejected + return (int)resp.StatusCode; // 200 if compatible, 400 if rejected } } ``` @@ -121,7 +121,7 @@ def publish(group: str, artifact_id: str, schema: bytes) -> int: headers={"Content-Type": "application/json"}, data=schema, ) - return r.status_code # 200 if compatible, 409 if the rule rejects it + return r.status_code # 200 if compatible, 400 if the rule rejects it ``` {% raw %} @@ -137,7 +137,7 @@ inline int publish(std::string_view group, std::string_view artifact_id, auto r = cpr::Post(cpr::Url{url}, cpr::Header{{"Content-Type", "application/json"}}, cpr::Body{std::string(schema)}); - return r.status_code; // 200 if compatible, 409 if rejected + return r.status_code; // 200 if compatible, 400 if rejected } ``` {% endraw %} @@ -153,7 +153,7 @@ func publish(group, artifactID string, schema []byte) (int, error) { return 0, err } defer resp.Body.Close() - return resp.StatusCode, nil // 200 if compatible, 409 if rejected + return resp.StatusCode, nil // 200 if compatible, 400 if rejected } ``` @@ -161,13 +161,13 @@ func publish(group, artifactID string, schema []byte) (int, error) { This is the guardrail that makes the whole contract story real. The CI pipeline posts the new schema version to the registry; if it violates the artifact's -compatibility rule, Apicurio returns **`409 Conflict`** and the step fails the +compatibility rule, Apicurio returns **`400 Bad Request`** and the step fails the build, blocking the merge. The contract is enforced *before* code ships — not discovered broken in production by a downstream team three days later. {% include excalidraw.html file="09-registry-gate" - alt="A pull request with a schema change posts the new version to the Apicurio registry; the registry checks it against the compatibility rule and either allows merge and publish, or returns 409 and blocks the merge; consumers then fetch the published schema and generate stubs" + alt="A pull request with a schema change posts the new version to the Apicurio registry; the registry checks it against the compatibility rule and either allows merge and publish, or returns 400 and blocks the merge; consumers then fetch the published schema and generate stubs" caption="Figure 9.2 — The registry checks compatibility at publish time and blocks the merge on a breaking change" %} ```yaml @@ -177,7 +177,7 @@ discovered broken in production by a downstream team three days later. code=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ "$REGISTRY/groups/orders/artifacts/order-placed/versions" \ -H "Content-Type: application/json" --data-binary @order-placed.avsc) - if [ "$code" = "409" ]; then + if [ "$code" = "400" ]; then echo "::error::Incompatible schema change — blocking the merge" exit 1 fi @@ -187,7 +187,7 @@ The rule is the contract about the contract. `BACKWARD` (new consumers can read old data) is the common default for events, because consumers upgrade on their own schedule and must tolerate data already on the topic. `FORWARD` flips it for cases where old consumers must read new data, and `FULL` demands both. Picking the rule -is a design decision; once picked, the registry enforces it for you and the `409` +is a design decision; once picked, the registry enforces it for you and the `400` is non-negotiable. ## Schema evolution strategies @@ -219,7 +219,7 @@ order of increasing cost: The registry enforces the *boundary* (compatible or not), but the strategy determines how often you approach that boundary. Teams that evolve additively rarely trigger a -`409`; teams that rename or restructure hit it constantly. +`400`; teams that rename or restructure hit it constantly. ## Validate before you publish @@ -255,7 +255,7 @@ public interface DryRunValidator { Response validate(@PathParam("group") String group, @PathParam("artifactId") String artifactId, byte[] schema); - // 200 = compatible (no version created); 409 = incompatible + // 200 = compatible (no version created); 400 = incompatible } ``` @@ -326,11 +326,11 @@ func isCompatible(group, artifactID string, schema []byte) (bool, error) { Prove the gate bites. Register `order-placed.avsc`, then make a genuinely breaking change — rename a required field — and `curl` the new version: you should get a -`409`, and the CI step should exit non-zero. Then make an additive change — a new +`400`, and the CI step should exit non-zero. Then make an additive change — a new *optional* field — and confirm it returns `200` and publishes a new version. The breaking change failing the build and the additive change passing it is the entire point of the registry. --- *Verification status: verified — [`examples/09-api-registry/`](https://github.com/patterncatalyst/cloud-native-design-patterns/tree/main/examples/09-api-registry/) passes 8/8 checks on Docker Engine (2026-10-10) -(schema registration, BACKWARD compatibility rule, rejection of a breaking change with 409, acceptance of an additive change, versioned retrieval).* +(schema registration, BACKWARD compatibility rule, rejection of a breaking change with 400, acceptance of an additive change, versioned retrieval).* diff --git a/_docs/15-appendix-b-versioning.md b/_docs/15-appendix-b-versioning.md index 5bd335e5..90865cae 100644 --- a/_docs/15-appendix-b-versioning.md +++ b/_docs/15-appendix-b-versioning.md @@ -330,7 +330,7 @@ The protocol-specific rules are all faces of four ideas: Prove the additive/breaking line with the registry from **API Registry**. Add an optional field to a schema and confirm the `BACKWARD` check passes; change a field's -type or reuse a protobuf tag and confirm it fails with `409`. The mechanical check +type or reuse a protobuf tag and confirm it fails with `400`. The mechanical check catching the breaking change — instead of a reviewer's memory — is the principle made real. diff --git a/_docs/28-appendix-o-newman.md b/_docs/28-appendix-o-newman.md index 10a4c05a..dee85286 100644 --- a/_docs/28-appendix-o-newman.md +++ b/_docs/28-appendix-o-newman.md @@ -129,7 +129,7 @@ audience. `cli` is for a human watching; `junit` emits the XML that CI systems t reports; `htmlextra` (a community reporter) produces a shareable page. ```bash -# install once (Node.js): npm i -g newman newman-reporter-htmlextra +# install once (Node.js): npm install -g --prefix ~/.local newman@6.2.3 newman-reporter-htmlextra@1.23.1 newman run orders.postman_collection.json \ --environment local.postman_environment.json \ --reporters cli,junit,htmlextra \ @@ -209,7 +209,7 @@ jobs: - uses: actions/checkout@v4 - run: docker compose up -d --build --wait # order, payment, inventory, … - run: ./scripts/wait-for-ready.sh # poll /readyz before testing - - run: npm i -g newman + - run: npm i -g newman@6.2.3 newman-reporter-htmlextra@1.23.1 - run: | newman run orders.postman_collection.json \ -e ci.postman_environment.json \ diff --git a/_example_pages/prerequisites.md b/_example_pages/prerequisites.md index 3fea374c..8f3d3f48 100644 --- a/_example_pages/prerequisites.md +++ b/_example_pages/prerequisites.md @@ -159,7 +159,17 @@ conftest --version `~/.local/bin` is on the default `PATH` for Fedora and RHEL login shells. -### hey (HTTP load testing) +### kcat (Kafka inspection) + +Lists topics and reads or produces Kafka messages from the host (example 04 uses it); it +replaces the removed kafka-ui. Optional, since `kafka-console-consumer.sh` inside the +`kafka` container does the same: + +```bash +sudo dnf install -y kcat +``` + +### hey v0.1.5 (HTTP load testing) Used by examples that demonstrate scaling or performance patterns. Download a binary from the [hey releases](https://github.com/rakyll/hey/releases) page, or build it with Go: @@ -168,10 +178,22 @@ the [hey releases](https://github.com/rakyll/hey/releases) page, or build it wit go install github.com/rakyll/hey@v0.1.5 ``` -### ghz (gRPC load testing) +### ghz v0.121.0 (gRPC load testing) -Used by examples with gRPC endpoints. Download a binary from the -[ghz releases](https://github.com/bojand/ghz/releases) page. +Used by examples with gRPC endpoints. Download the pinned binary from the +[ghz releases](https://github.com/bojand/ghz/releases) page: + +```bash +GHZ_VERSION=0.121.0 +BASE="https://github.com/bojand/ghz/releases/download/v${GHZ_VERSION}" +cd "$(mktemp -d)" +curl -fsSLO "${BASE}/ghz-linux-x86_64.tar.gz" +curl -fsSLO "${BASE}/ghz-linux-x86_64.tar.gz.sha256" +echo "$(cat ghz-linux-x86_64.tar.gz.sha256) ghz-linux-x86_64.tar.gz" | sha256sum -c - +mkdir -p ~/.local/bin +tar -xzf ghz-linux-x86_64.tar.gz -C ~/.local/bin ghz +ghz --version +``` ### newman 6.2.3 (Postman CLI, required for example 28) @@ -180,7 +202,7 @@ Used by example 28 (Newman API testing). Requires Node.js. Install into the user ```bash sudo dnf install -y nodejs npm -npm install -g --prefix ~/.local newman@6.2.3 +npm install -g --prefix ~/.local newman@6.2.3 newman-reporter-htmlextra@1.23.1 ``` Verify: @@ -188,6 +210,7 @@ Verify: ```bash curl --version jq --version +kcat -V 2>&1 | head -1 # optional hey -h 2>&1 | head -1 # optional ghz --version # optional newman --version # required for example 28 diff --git a/_plans/modernization-plan.md b/_plans/modernization-plan.md index b6420ba3..403cb9df 100644 --- a/_plans/modernization-plan.md +++ b/_plans/modernization-plan.md @@ -30,7 +30,7 @@ Each step gets its own branch, a PR, green CI and a squash-merge before the next | S5 | chore/python-3.14 | 5 | done (19/19 Python + 28 PASS after the Opus review) | | S6 | chore/go-toolchain | 2+4 | done (18/18 Go + 06 in all six languages PASS after the Opus review; closes #34) | | S7 | chore/cpp-toolchain-ubi10 | 2+4 | done (111/111 PASS after two Opus repair rounds; closes #35) | -| S8 | chore/infra-images | 2 | pending | +| S8 | chore/infra-images | 2 | validated (awaiting PR/CI) | | S9 | docs/minikube-docker-driver (+ 05 on minikube) | 7 | pending | | S10 | feature/openshift-local-appendix | 8 | pending | | S11 | docs/professional-voice | 9 | pending | @@ -51,12 +51,12 @@ Each step gets its own branch, a PR, green CI and a squash-merge before the next | Postgres | postgres:18.6-alpine (mount /var/lib/postgresql) | | Redis | redis:8.10.2-alpine | | Debezium | quay.io/debezium/connect:3.7.0.Final | -| Apicurio | apicurio-registry:3.3.3 | +| Apicurio | docker.io/apicurio/apicurio-registry:3.3.3 | | flagd | ghcr.io/open-feature/flagd:v0.17.0 | | LGTM | grafana/otel-lgtm:0.36.0 | | Envoy | envoyproxy/envoy:v1.39.3 | | kafka-ui | removed (use kcat) | -| Test tools | newman 6.2.3, htmlextra 1.23.1, hey v0.1.5, ghz v0.121.0 | +| Test tools | newman 6.2.3, htmlextra 1.23.1, hey v0.1.5, ghz v0.121.0, conftest 0.71.1 | | minikube | v1.39.0, Kubernetes v1.36.5 | | Operators | Strimzi 1.2.0, CNPG v1.30.1, KEDA v2.21.0, Istio 1.31.1 | | CRC | crc 2.64.0 / OCP 4.22.14; AMQ Streams per lgtm-crc (check packagemanifest) | @@ -208,3 +208,45 @@ Carried forward: - Images `ubi9/dotnet-100:9.8-1791421718` and `ubi9/dotnet-100-aspnet:9.8-1791363520`, non-root UID 1001, group-0 ownership, BuildKit NuGet cache mount. HOLD: Red Hat publishes no ubi10 .NET image. - Locked stack: 06 stream processor rewritten on Streamiz (same contract as Quarkus/Spring). Kafka producers/consumers stay on Confluent.Kafka; MassTransit dropped by user decision. - 30: second dashboard instance on port 8081 behind the same Redis backplane; verify.sh now has 10 checks (backplane both directions, restart). Circuit reconnection is framework default and not scripted. + +### S8 infra images (2026-10-10) + +Docker 29.8.2, compose 5.5.1, Fedora 44. Pins re-verified with skopeo list-tags: all targets were the newest stable (Kafka 4.3.1, Postgres 18.6-alpine, Redis 8.10.2-alpine, Debezium 3.7.0.Final on quay.io, Apicurio 3.3.3, flagd v0.17.0, otel-lgtm 0.36.0, Envoy v1.39.3); test tools newman 6.2.3, htmlextra 1.23.1, hey v0.1.5, ghz v0.121.0, conftest 0.71.1 were also still the newest. + +**111/111 targets PASS** (`OUT=.../matrix-s8 scripts/verify-matrix.sh`). `grep ':latest|-latest|kafka-ui'` over examples, _docs and _example_pages has no hits except the deliberate `sketchy:latest` policy fixture. + +Breaking changes handled: + +| Change | Fix | +|---|---| +| Debezium 3.7 was OOM-killed (exit 137) at 1 GB: it scans every bundled connector | `compose-debezium.yaml`: `mem_limit: 2g`, `HEAP_OPTS=-Xms256M -Xmx1G` | +| Apicurio 3.3 serves `/health/ready` on the management port 9000, not 8080 | `compose-apicurio.yaml` healthcheck on 9000, host port 19000 published; 09 verify.sh and README updated | +| Apicurio 3.3 answers 400 (was 409) to an incompatible schema | 09 verify.sh, README, `_docs/09-api-registry.md`, appendix B, registry-gate and architecture diagrams | +| Tempo 3 `/api/search` returns no traces without a time window and indexes later | 11 verify.sh passes start/end and retries up to 60 s; quarkus README example | +| otel-lgtm 0.36 ships `grafana-datasources.yaml`; our file beside it was ignored | `compose-lgtm.yaml` mounts ours over that name (Prometheus default, localhost URLs kept) | +| Postgres 18 PGDATA moved | volume mounted at `/var/lib/postgresql` | +| kafka-ui removed | compose overrides, port tables and the 04 README now use kcat or `kafka-console-consumer.sh` | + +No change needed: Kafka 4 KRaft env vars and the `nc` healthcheck, the Debezium connector JSON (`topic.prefix`, `table.include.list`, `pgoutput`), Envoy 1.39 `envoy.yaml`, flagd flag file and start args, the `/apis/registry/v3` paths, Prometheus on 19090, Redis commands. Redis 8 is RSALv2/SSPLv1/AGPLv3; the AGPLv3 option is free and needs no key. + +CI cache (12.4 GB, 170 entries before): Conan 8.6 GB in 20 entries (350-700 MB each, one per conan.lock, written by PRs too), Go 2.3 GB in 20 entries, BuildKit layers 0.2 GB. Now every run restores and only main writes. Maven and Go are one shared entry each, written by main-only `warm-maven` / `warm-go` jobs that resolve all projects (62 poms, 31 go.mod) and delete the superseded entry; Conan stays one entry per distinct conan.lock (22), pruned on main (measured on 25-caching/cpp: 1.6 GB to 163 MB, the following `conan install` all cache hits) and with the builder stage forced to run on a miss. Steady state on main: typical 22 x 82 MB + Maven 1.0 + Go 0.6 + layers 0.25 + Pages 0.05 = about 3.7 GB, bound 4.9 GB, peak about 8 GB when a Conan key bump or bulk lock change leaves superseded per-lock entries and layer blobs age out after 7 days unused. Measured: unpruned Conan entries (API), BuildKit layers (27 blobs = 200 MB); partly measured: prune ratio (one project); estimated: Maven <=1.0 GB, Go <=0.6 GB, Pages. Check the first real Maven/Go saved entry sizes after merge. `cache-cleanup.yml` deletes a closed same-repo PR's caches. New key prefixes (`Linux-conan2-`, `maven-Linux-`, `go-Linux-`) differ from the old ones (`conan-Linux-`, `setup-java-Linux-x64-maven-`, `setup-go-Linux-x64-ubuntu24-go-`), so delete the old entries by prefix (command in CLAUDE.md, "CI cache budget"), safe at any time; never `gh cache delete --all`. + +#### Repair round 1 (validator findings) + +- Conan miss on main (HIGH): reproduced with a fresh buildx builder and a local layer cache standing in for `type=gha`: with `cache-from` alone the `conan install` RUN is CACHED and the `cndp-conan2` mount stays empty. With `--no-cache-filter builder` the RUN executes and the mount is populated (1.6 GB, 13 built packages after pruning to 163 MB). `examples.yml` now passes `no-cache-filters: builder,schemagen` on a main miss (all 31 Containerfiles have a `builder` stage; an unknown stage name is accepted), and the prune RUN fails when no `p/b//p` package is left (negative test on an empty mount fails, populated mount passes). The key-shape comment now reads `-conan2-ubi10.2-gcc15-conan2.33.0-`. +- Cache size (HIGH): the estimate had ignored per-pom Maven keys (62 projects) and per-go.sum Go keys (31). Maven and Go are now one shared key each, restored by every leg and written only by the main-only `warm-maven` / `warm-go` jobs, which resolve every project and delete the superseded entry (a build leg cannot be the writer: it only downloads its own dependencies). Go caches the module cache only. Conan stays per-lock (22 x <=135 MB). Arithmetic: bound 22 x 135 = 3.0 GB + 1.0 + 0.6 + 0.25 + 0.05 = 4.9 GB; typical about 3.7 GB; peak about 8 GB (see above). Maven retry args (`aether.transport.http.retryHandler.*`) are set on the warm job and the build leg. +- Cleanup advice (MEDIUM): `gh cache delete --all` replaced by deleting the old prefixes (`conan-Linux-`, `setup-java-Linux-x64-maven-`, `setup-go-Linux-x64-ubuntu24-go-`), safe at any time because the new keys use other prefixes (CLAUDE.md). +- 09 verify.sh: the breaking-schema check needs HTTP 400 and `RuleViolationException` in the body; the version count uses `jq -e '.count == 2'`. Check count unchanged (8). 11 verify.sh (shared by all six variants, service name `order-service` in each): the Tempo search filters `service.name=order-service`. +- `cache-cleanup.yml` skips fork pull requests and tolerates a failed delete. `_infra/README.md` lists compose-apicurio.yaml, ports 8081 and 19000 and `cndp-apicurio`. kcat (`sudo dnf install -y kcat`, Fedora package `kcat` 1.7.1) is in both prerequisites files. The ghz install creates `~/.local/bin` and verifies the published `.sha256`; the Newman appendix installs with `--prefix ~/.local`; conftest 0.71.1 is in the test-tools table. +- buildkit-cache-dance `utility-image` pinned to `ghcr.io/containerd/busybox:1.36@sha256:7b3c...2627c` (1.36 is the newest tag on ghcr, multi-arch; the action default is `:latest`). +- Verification: actionlint 1.7.12 clean on all workflows; `validate_site.py` 0 errors; live `verify-matrix.sh`: 09 (8 checks), 11 spring-boot, quarkus, dotnet, python, cpp, go (10 checks each) and 25-caching/cpp (26) all PASS. + +#### Repair round 2 + +- Eviction guard: the "Delete the superseded" steps first confirm the new key exists (`gh cache list --key` is a prefix match, `--limit 1000`) and keep the old entries with a warning otherwise, since `actions/cache/save` only warns on failure. +- `warm-maven` and `warm-go` have job-level `concurrency` (`warm-maven-` / `warm-go-`, no cancel). +- Both start with an exact-key `lookup-only` probe; on a hit every later step is skipped. On a miss: restore with restore-keys, warm, save, evict. +- `warm-go` uses `go-version-file` (examples/01-cloud-native-principles/go/go.mod, go 1.27.2) instead of `stable`. +- Stale `warm-caches` references renamed to `warm-maven` / `warm-go` in examples.yml and CLAUDE.md. +- Estimate relabelled measured / partly measured / estimated; typical 3.7 GB, bound 4.9 GB, peak about 8 GB; check real Maven/Go entry sizes after merge. +- Verification: actionlint clean on all workflows; `validate_site.py` 0 errors. diff --git a/assets/diagrams/09-registry-gate.excalidraw b/assets/diagrams/09-registry-gate.excalidraw index ae249a90..25b269ef 100644 --- a/assets/diagrams/09-registry-gate.excalidraw +++ b/assets/diagrams/09-registry-gate.excalidraw @@ -280,13 +280,13 @@ "updated": 1, "link": null, "locked": false, - "text": "409 \u2192 block the merge", + "text": "400 \u2192 block the merge", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, - "originalText": "409 \u2192 block the merge", + "originalText": "400 \u2192 block the merge", "lineHeight": 1.25, "baseline": 14 }, diff --git a/assets/diagrams/09-registry-gate.svg b/assets/diagrams/09-registry-gate.svg index 2f801093..c7ae49ef 100644 --- a/assets/diagrams/09-registry-gate.svg +++ b/assets/diagrams/09-registry-gate.svg @@ -8,7 +8,7 @@ merge & publish -409 → block the merge +400 → block the merge consumers fetch + gen stubs diff --git a/examples/02-communications/README.md b/examples/02-communications/README.md index c78974d9..1f54198f 100644 --- a/examples/02-communications/README.md +++ b/examples/02-communications/README.md @@ -108,5 +108,4 @@ Open Grafana at http://localhost:3000 and explore: | Grafana | 3000 | | Postgres | 5432 | | Kafka (host) | 9092 | -| Kafka UI | 8090 | | OTLP HTTP | 4318 | diff --git a/examples/02-communications/dotnet/compose.yaml b/examples/02-communications/dotnet/compose.yaml index 12c84d80..a00b2df1 100644 --- a/examples/02-communications/dotnet/compose.yaml +++ b/examples/02-communications/dotnet/compose.yaml @@ -9,9 +9,6 @@ services: volumes: - ../db/init:/docker-entrypoint-initdb.d:Z - kafka-ui: - ports: !override [] - inventory: build: context: . diff --git a/examples/04-data/README.md b/examples/04-data/README.md index 269e3323..567ed0fe 100644 --- a/examples/04-data/README.md +++ b/examples/04-data/README.md @@ -99,5 +99,4 @@ Open Grafana at http://localhost:3000: | Grafana | 3000 | | Postgres | 5432 | | Kafka (host) | 9092 | -| Kafka UI | 8090 | | OTLP HTTP | 4318 | diff --git a/examples/04-data/quarkus/README.md b/examples/04-data/quarkus/README.md index 6604ed17..c824ec1c 100644 --- a/examples/04-data/quarkus/README.md +++ b/examples/04-data/quarkus/README.md @@ -74,14 +74,15 @@ curl http://localhost:8080/orders # List outbox events curl http://localhost:8080/outbox -# Check Kafka topic (via Kafka UI) -open http://localhost:8090 -# Navigate to Topics → order.placed → Messages +# Read the topic from inside the broker container +docker compose exec kafka /opt/kafka/bin/kafka-console-consumer.sh \ + --bootstrap-server kafka:9094 --topic order.placed --from-beginning --timeout-ms 5000 +# or, from the host with kcat: kcat -b localhost:9092 -t order.placed -C -e ``` ## Observe -- **Kafka UI**: http://localhost:8090 → see `order.placed` topic +- **Kafka topic**: `kafka-console-consumer.sh` inside the `kafka` container (or `kcat -b localhost:9092 -t order.placed -C -e`) → see `order.placed` events - **Grafana**: http://localhost:3000 → traces, logs, metrics - **Kafka Connect**: http://localhost:8083/connectors → see `outbox-connector` status diff --git a/examples/05-event-driven/README.md b/examples/05-event-driven/README.md index 0b5a4bea..a4caf3dd 100644 --- a/examples/05-event-driven/README.md +++ b/examples/05-event-driven/README.md @@ -94,5 +94,4 @@ Open Grafana at http://localhost:3000: | Grafana | 3000 | | Postgres | 5432 | | Kafka (host) | 9092 | -| Kafka UI | 8090 | | OTLP HTTP | 4318 | diff --git a/examples/05-event-driven/dotnet/compose.yaml b/examples/05-event-driven/dotnet/compose.yaml index c2299260..fd3fed88 100644 --- a/examples/05-event-driven/dotnet/compose.yaml +++ b/examples/05-event-driven/dotnet/compose.yaml @@ -9,9 +9,6 @@ services: volumes: - ../db/init:/docker-entrypoint-initdb.d:Z - kafka-ui: - ports: !override [] - order-service: build: context: . diff --git a/examples/06-stream-processing/README.md b/examples/06-stream-processing/README.md index 340c6c50..80537585 100644 --- a/examples/06-stream-processing/README.md +++ b/examples/06-stream-processing/README.md @@ -96,5 +96,4 @@ Open Grafana at http://localhost:3000: | Grafana | 3000 | | Postgres | 5432 | | Kafka (host) | 9092 | -| Kafka UI | 8090 | | OTLP HTTP | 4318 | diff --git a/examples/06-stream-processing/dotnet/compose.yaml b/examples/06-stream-processing/dotnet/compose.yaml index 495ba4a1..2c1d5547 100644 --- a/examples/06-stream-processing/dotnet/compose.yaml +++ b/examples/06-stream-processing/dotnet/compose.yaml @@ -9,9 +9,6 @@ services: volumes: - ../db/init:/docker-entrypoint-initdb.d:Z - kafka-ui: - ports: !override [] - order-service: build: context: . diff --git a/examples/09-api-registry/README.md b/examples/09-api-registry/README.md index 93a56cdd..9667715a 100644 --- a/examples/09-api-registry/README.md +++ b/examples/09-api-registry/README.md @@ -3,7 +3,7 @@ Demonstrates **Apicurio Registry** as the single authoritative home for API contracts. The verify script registers an Avro schema, sets a `BACKWARD` compatibility rule, then proves the gate: a **breaking change** (renamed -fields) is rejected with `409 Conflict`, while an **additive change** (new +fields) is rejected with `400 Bad Request`, while an **additive change** (new optional field) is accepted. ## Prerequisites @@ -22,7 +22,7 @@ credentials, and the container naming convention. |---------|-------|------| | Schema registration | `POST /groups/orders/artifacts` | Register an Avro schema as a versioned artifact | | Compatibility rules | `POST .../rules` | Set `BACKWARD` — new consumers must read old data | -| Breaking change gate | `POST .../versions` | Renamed fields → `409 Conflict` (blocked) | +| Breaking change gate | `POST .../versions` | Renamed fields → `400 Bad Request` (blocked) | | Additive change | `POST .../versions` | New optional field → `200 OK` (accepted) | ## Architecture @@ -38,7 +38,7 @@ docker compose up -d Wait for Apicurio to be ready: ```bash -curl -sf http://localhost:8081/health/ready | jq . +curl -sf http://localhost:19000/health/ready | jq . ``` ## Drive it @@ -70,7 +70,7 @@ curl -s -w "\n%{http_code}" -X POST \ http://localhost:8081/apis/registry/v3/groups/orders/artifacts/order-placed/versions \ -H "Content-Type: application/json" \ -d '{ "version": "2.0.0", "content": { "content": "...", "contentType": "application/json" } }' -# → 409 Conflict +# → 400 Bad Request # Try an additive change # → 200 OK @@ -92,6 +92,6 @@ Open Grafana at http://localhost:3000: | Service | Port | |---------|------| -| Apicurio Registry | 8081 | +| Apicurio Registry | 8081 (health on 19000) | | Grafana | 3000 | | OTLP HTTP | 4318 | diff --git a/examples/09-api-registry/architecture.excalidraw b/examples/09-api-registry/architecture.excalidraw index 9ffab4b1..486fdb5a 100644 --- a/examples/09-api-registry/architecture.excalidraw +++ b/examples/09-api-registry/architecture.excalidraw @@ -550,13 +550,13 @@ "updated": 1, "link": null, "locked": false, - "text": "breaking \u2192 409", + "text": "breaking \u2192 400", "fontSize": 10, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, - "originalText": "breaking \u2192 409", + "originalText": "breaking \u2192 400", "lineHeight": 1.25, "baseline": 10 }, diff --git a/examples/09-api-registry/architecture.svg b/examples/09-api-registry/architecture.svg index d3ccaf31..2173856a 100644 --- a/examples/09-api-registry/architecture.svg +++ b/examples/09-api-registry/architecture.svg @@ -15,6 +15,6 @@ -breaking → 409 +breaking → 400 additive → 200 \ No newline at end of file diff --git a/examples/09-api-registry/verify.sh b/examples/09-api-registry/verify.sh index ec975bc1..5f7f8a4d 100755 --- a/examples/09-api-registry/verify.sh +++ b/examples/09-api-registry/verify.sh @@ -56,7 +56,7 @@ printf '==> Verifying Example 09: API Registry\n\n' # --- Apicurio is up --- check "Apicurio health check" \ - "curl $CURL_OPTS -sf http://localhost:8081/health/ready" \ + "curl $CURL_OPTS -sf http://localhost:19000/health/ready" \ '"status"' # --- Create a group --- @@ -109,9 +109,9 @@ check "compatibility rule is BACKWARD" \ "curl $CURL_OPTS -sf '$REGISTRY/groups/orders/artifacts/order-placed/rules/COMPATIBILITY'" \ "BACKWARD" -# --- Try a BREAKING change (rename required fields) → expect 409 --- +# --- Try a BREAKING change (rename required fields) → expect 400 --- printf ' \xe2\x86\x92 attempting breaking schema change (renamed fields)...\n' -BREAKING_CODE=$(curl $CURL_OPTS -s -o /dev/null -w '%{http_code}' -X POST \ +BREAKING_RESPONSE=$(curl $CURL_OPTS -s -w '\n%{http_code}' -X POST \ "$REGISTRY/groups/orders/artifacts/order-placed/versions" \ -H "Content-Type: application/json" \ -d "{ @@ -121,12 +121,16 @@ BREAKING_CODE=$(curl $CURL_OPTS -s -o /dev/null -w '%{http_code}' -X POST \ \"contentType\": \"application/json\" } }") +BREAKING_CODE=$(echo "$BREAKING_RESPONSE" | tail -1) +BREAKING_BODY=$(echo "$BREAKING_RESPONSE" | sed '$d') -if [ "$BREAKING_CODE" = "409" ]; then - printf ' \xe2\x9c\x93 breaking change rejected with 409 Conflict\n' +# A 400 alone could be a malformed request; the registry names the rule it enforced. +if [ "$BREAKING_CODE" = "400" ] && echo "$BREAKING_BODY" | grep -q 'RuleViolationException'; then + printf ' \xe2\x9c\x93 breaking change rejected with 400 (RuleViolationException)\n' PASS=$((PASS + 1)) else - printf ' \xe2\x9c\x97 breaking change should return 409 (got %s)\n' "$BREAKING_CODE" + printf ' \xe2\x9c\x97 breaking change should return 400 RuleViolationException (got %s)\n' "$BREAKING_CODE" + printf ' response: %s\n' "$(echo "$BREAKING_BODY" | head -c 300)" FAIL=$((FAIL + 1)) fi @@ -153,8 +157,8 @@ fi # --- Verify we now have 2 versions --- check "artifact has 2 versions" \ - "curl $CURL_OPTS -sf '$REGISTRY/groups/orders/artifacts/order-placed/versions' | jq '.count'" \ - "2" + "curl $CURL_OPTS -sf '$REGISTRY/groups/orders/artifacts/order-placed/versions' | jq -e '.count == 2' >/dev/null && echo count-is-2" \ + "count-is-2" # --- Fetch the latest version and verify it has the new field --- check "latest version contains merchant_id field" \ diff --git a/examples/11-observability/README.md b/examples/11-observability/README.md index b6bfe37c..b9dfa590 100644 --- a/examples/11-observability/README.md +++ b/examples/11-observability/README.md @@ -100,5 +100,4 @@ Open Grafana at http://localhost:3000: | Prometheus | 19090 | | Postgres | 5432 | | Kafka (host) | 9092 | -| Kafka UI | 8090 | | OTLP HTTP | 4318 | diff --git a/examples/11-observability/dotnet/compose.yaml b/examples/11-observability/dotnet/compose.yaml index 37919392..e34fb8b2 100644 --- a/examples/11-observability/dotnet/compose.yaml +++ b/examples/11-observability/dotnet/compose.yaml @@ -9,9 +9,6 @@ services: volumes: - ../db/init:/docker-entrypoint-initdb.d:Z - kafka-ui: - ports: !override [] - inventory: build: context: . diff --git a/examples/11-observability/quarkus/README.md b/examples/11-observability/quarkus/README.md index 2509c252..3052c323 100644 --- a/examples/11-observability/quarkus/README.md +++ b/examples/11-observability/quarkus/README.md @@ -50,7 +50,7 @@ curl -X POST http://localhost:8080/orders \ docker logs cndp-order-service 2>&1 | grep "order placed" # Query traces in Tempo -curl -s http://localhost:3200/api/search | jq '.traces[:3]' +curl -s "http://localhost:3200/api/search?limit=20&start=$(($(date +%s)-900))&end=$(date +%s)" | jq '.traces[:3]' # Query metrics in Prometheus curl -s 'http://localhost:19090/api/v1/query?query=orders_placed_total' | jq '.data.result' diff --git a/examples/11-observability/verify.sh b/examples/11-observability/verify.sh index a7bb7839..7bfee0b3 100755 --- a/examples/11-observability/verify.sh +++ b/examples/11-observability/verify.sh @@ -92,8 +92,16 @@ sleep 10 # --- Traces in Tempo --- printf ' \xe2\x86\x92 checking traces in Tempo...\n' -TEMPO_SEARCH=$(curl $CURL_OPTS -sf "$TEMPO/api/search" 2>/dev/null || echo "") -check "Tempo has traces" \ +# Tempo 3 indexes new traces a little later than the trace-by-id path, and a search needs +# a time window. Retry for up to 60 s. +TEMPO_SEARCH="" +for _ in $(seq 1 12); do + NOW=$(date +%s) + TEMPO_SEARCH=$(curl $CURL_OPTS -sf "$TEMPO/api/search?tags=service.name%3Dorder-service&limit=20&start=$((NOW - 900))&end=$((NOW + 60))" 2>/dev/null || echo "") + case "$TEMPO_SEARCH" in *traceID*) break ;; esac + sleep 5 +done +check "Tempo search finds traces of service order-service" \ "echo '$TEMPO_SEARCH'" \ "traceID" diff --git a/examples/18-errors/quarkus/compose.yaml b/examples/18-errors/quarkus/compose.yaml index 91d8bd7a..ba44103d 100644 --- a/examples/18-errors/quarkus/compose.yaml +++ b/examples/18-errors/quarkus/compose.yaml @@ -1,7 +1,7 @@ name: cndp-18-quarkus services: lgtm: - image: docker.io/grafana/otel-lgtm:0.8.1 + image: docker.io/grafana/otel-lgtm:0.36.0 container_name: cndp-lgtm ports: - "3000:3000" diff --git a/examples/18-errors/spring-boot/compose.yaml b/examples/18-errors/spring-boot/compose.yaml index 8243d169..412faf4d 100644 --- a/examples/18-errors/spring-boot/compose.yaml +++ b/examples/18-errors/spring-boot/compose.yaml @@ -1,7 +1,7 @@ name: cndp-18-spring-boot services: lgtm: - image: docker.io/grafana/otel-lgtm:0.8.1 + image: docker.io/grafana/otel-lgtm:0.36.0 container_name: cndp-lgtm ports: - "3000:3000" diff --git a/examples/22-l7-routing/cpp/compose.yaml b/examples/22-l7-routing/cpp/compose.yaml index d26d2fb7..e72e53d0 100644 --- a/examples/22-l7-routing/cpp/compose.yaml +++ b/examples/22-l7-routing/cpp/compose.yaml @@ -2,7 +2,7 @@ name: cndp-22-cpp services: envoy: - image: docker.io/envoyproxy/envoy:v1.31-latest + image: docker.io/envoyproxy/envoy:v1.39.3 container_name: cndp-envoy user: "101" command: ["envoy", "-c", "/etc/envoy/envoy.yaml"] diff --git a/examples/22-l7-routing/dotnet/compose.yaml b/examples/22-l7-routing/dotnet/compose.yaml index f3420808..dde46ee5 100644 --- a/examples/22-l7-routing/dotnet/compose.yaml +++ b/examples/22-l7-routing/dotnet/compose.yaml @@ -1,7 +1,7 @@ name: cndp-22-dotnet services: envoy: - image: docker.io/envoyproxy/envoy:v1.31-latest + image: docker.io/envoyproxy/envoy:v1.39.3 container_name: cndp-envoy user: "101" command: ["envoy", "-c", "/etc/envoy/envoy.yaml"] diff --git a/examples/22-l7-routing/go/compose.yaml b/examples/22-l7-routing/go/compose.yaml index 5144e644..20a6ae1d 100644 --- a/examples/22-l7-routing/go/compose.yaml +++ b/examples/22-l7-routing/go/compose.yaml @@ -2,7 +2,7 @@ name: cndp-22-go services: envoy: - image: docker.io/envoyproxy/envoy:v1.31-latest + image: docker.io/envoyproxy/envoy:v1.39.3 container_name: cndp-envoy user: "101" command: ["envoy", "-c", "/etc/envoy/envoy.yaml"] diff --git a/examples/22-l7-routing/python/compose.yaml b/examples/22-l7-routing/python/compose.yaml index 4c136c0e..932ef8b4 100644 --- a/examples/22-l7-routing/python/compose.yaml +++ b/examples/22-l7-routing/python/compose.yaml @@ -1,7 +1,7 @@ name: cndp-22-python services: envoy: - image: docker.io/envoyproxy/envoy:v1.31-latest + image: docker.io/envoyproxy/envoy:v1.39.3 container_name: cndp-envoy user: "101" command: ["envoy", "-c", "/etc/envoy/envoy.yaml"] diff --git a/examples/22-l7-routing/quarkus/compose.yaml b/examples/22-l7-routing/quarkus/compose.yaml index 26e572f6..fb4d0fa9 100644 --- a/examples/22-l7-routing/quarkus/compose.yaml +++ b/examples/22-l7-routing/quarkus/compose.yaml @@ -1,7 +1,7 @@ name: cndp-22-quarkus services: envoy: - image: docker.io/envoyproxy/envoy:v1.31-latest + image: docker.io/envoyproxy/envoy:v1.39.3 container_name: cndp-envoy user: "101" command: ["envoy", "-c", "/etc/envoy/envoy.yaml"] diff --git a/examples/22-l7-routing/spring-boot/compose.yaml b/examples/22-l7-routing/spring-boot/compose.yaml index cc076ddc..6ad127ce 100644 --- a/examples/22-l7-routing/spring-boot/compose.yaml +++ b/examples/22-l7-routing/spring-boot/compose.yaml @@ -1,7 +1,7 @@ name: cndp-22-spring-boot services: envoy: - image: docker.io/envoyproxy/envoy:v1.31-latest + image: docker.io/envoyproxy/envoy:v1.39.3 container_name: cndp-envoy user: "101" command: ["envoy", "-c", "/etc/envoy/envoy.yaml"] diff --git a/examples/24-monolith-to-microservices/cpp/compose.yaml b/examples/24-monolith-to-microservices/cpp/compose.yaml index b1bff284..cfd01084 100644 --- a/examples/24-monolith-to-microservices/cpp/compose.yaml +++ b/examples/24-monolith-to-microservices/cpp/compose.yaml @@ -5,9 +5,6 @@ include: - path: ../../_infra/compose-redis.yaml services: - kafka-ui: - ports: !override [] - monolith: build: context: stub-service diff --git a/examples/24-monolith-to-microservices/dotnet/compose.yaml b/examples/24-monolith-to-microservices/dotnet/compose.yaml index 16eb70df..933e20ba 100644 --- a/examples/24-monolith-to-microservices/dotnet/compose.yaml +++ b/examples/24-monolith-to-microservices/dotnet/compose.yaml @@ -4,9 +4,6 @@ include: - path: ../../_infra/compose-redis.yaml services: - kafka-ui: - ports: !override [] - monolith: build: context: . diff --git a/examples/24-monolith-to-microservices/go/compose.yaml b/examples/24-monolith-to-microservices/go/compose.yaml index 7db464c4..396c2658 100644 --- a/examples/24-monolith-to-microservices/go/compose.yaml +++ b/examples/24-monolith-to-microservices/go/compose.yaml @@ -5,9 +5,6 @@ include: - path: ../../_infra/compose-redis.yaml services: - kafka-ui: - ports: !override [] - monolith: build: context: stub-service diff --git a/examples/24-monolith-to-microservices/python/compose.yaml b/examples/24-monolith-to-microservices/python/compose.yaml index e1f966eb..15b8d841 100644 --- a/examples/24-monolith-to-microservices/python/compose.yaml +++ b/examples/24-monolith-to-microservices/python/compose.yaml @@ -4,9 +4,6 @@ include: - path: ../../_infra/compose-redis.yaml services: - kafka-ui: - ports: !override [] - monolith: build: context: stub-service diff --git a/examples/24-monolith-to-microservices/quarkus/compose.yaml b/examples/24-monolith-to-microservices/quarkus/compose.yaml index 763e49de..e8d0f516 100644 --- a/examples/24-monolith-to-microservices/quarkus/compose.yaml +++ b/examples/24-monolith-to-microservices/quarkus/compose.yaml @@ -4,9 +4,6 @@ include: - path: ../../_infra/compose-redis.yaml services: - kafka-ui: - ports: !override [] - monolith: build: context: stub-service diff --git a/examples/24-monolith-to-microservices/spring-boot/compose.yaml b/examples/24-monolith-to-microservices/spring-boot/compose.yaml index 325197f4..d97fcab5 100644 --- a/examples/24-monolith-to-microservices/spring-boot/compose.yaml +++ b/examples/24-monolith-to-microservices/spring-boot/compose.yaml @@ -4,9 +4,6 @@ include: - path: ../../_infra/compose-redis.yaml services: - kafka-ui: - ports: !override [] - monolith: build: context: stub-service diff --git a/examples/25-caching/cpp/compose.yaml b/examples/25-caching/cpp/compose.yaml index 6c3421f3..7d5839b8 100644 --- a/examples/25-caching/cpp/compose.yaml +++ b/examples/25-caching/cpp/compose.yaml @@ -10,7 +10,7 @@ services: - ../db/init:/docker-entrypoint-initdb.d:Z redis: - image: redis:7-alpine + image: docker.io/library/redis:8.10.2-alpine container_name: cndp-redis ports: - "6379:6379" diff --git a/examples/27-feature-flags/cpp/compose.yaml b/examples/27-feature-flags/cpp/compose.yaml index 0486ae30..99d1c402 100644 --- a/examples/27-feature-flags/cpp/compose.yaml +++ b/examples/27-feature-flags/cpp/compose.yaml @@ -2,7 +2,7 @@ name: cndp-27-cpp services: flagd: - image: ghcr.io/open-feature/flagd:latest + image: ghcr.io/open-feature/flagd:v0.17.0 container_name: cndp-flagd command: - start diff --git a/examples/27-feature-flags/dotnet/compose.yaml b/examples/27-feature-flags/dotnet/compose.yaml index 9a08d72e..d6488e84 100644 --- a/examples/27-feature-flags/dotnet/compose.yaml +++ b/examples/27-feature-flags/dotnet/compose.yaml @@ -1,7 +1,7 @@ name: cndp-27-dotnet services: flagd: - image: ghcr.io/open-feature/flagd:latest + image: ghcr.io/open-feature/flagd:v0.17.0 container_name: cndp-flagd command: - start diff --git a/examples/27-feature-flags/go/compose.yaml b/examples/27-feature-flags/go/compose.yaml index 78b49612..84377698 100644 --- a/examples/27-feature-flags/go/compose.yaml +++ b/examples/27-feature-flags/go/compose.yaml @@ -2,7 +2,7 @@ name: cndp-27-go services: flagd: - image: ghcr.io/open-feature/flagd:latest + image: ghcr.io/open-feature/flagd:v0.17.0 container_name: cndp-flagd command: - start diff --git a/examples/27-feature-flags/python/compose.yaml b/examples/27-feature-flags/python/compose.yaml index 2ce35619..37af3f87 100644 --- a/examples/27-feature-flags/python/compose.yaml +++ b/examples/27-feature-flags/python/compose.yaml @@ -1,7 +1,7 @@ name: cndp-27-python services: flagd: - image: ghcr.io/open-feature/flagd:latest + image: ghcr.io/open-feature/flagd:v0.17.0 container_name: cndp-flagd command: - start diff --git a/examples/27-feature-flags/quarkus/compose.yaml b/examples/27-feature-flags/quarkus/compose.yaml index d524903a..905e1a28 100644 --- a/examples/27-feature-flags/quarkus/compose.yaml +++ b/examples/27-feature-flags/quarkus/compose.yaml @@ -1,7 +1,7 @@ name: cndp-27-quarkus services: flagd: - image: ghcr.io/open-feature/flagd:latest + image: ghcr.io/open-feature/flagd:v0.17.0 container_name: cndp-flagd command: - start diff --git a/examples/27-feature-flags/spring-boot/compose.yaml b/examples/27-feature-flags/spring-boot/compose.yaml index bb8464e4..b2531bb0 100644 --- a/examples/27-feature-flags/spring-boot/compose.yaml +++ b/examples/27-feature-flags/spring-boot/compose.yaml @@ -1,7 +1,7 @@ name: cndp-27-spring-boot services: flagd: - image: ghcr.io/open-feature/flagd:latest + image: ghcr.io/open-feature/flagd:v0.17.0 container_name: cndp-flagd command: - start diff --git a/examples/28-newman/README.md b/examples/28-newman/README.md index c30fb657..5829b949 100644 --- a/examples/28-newman/README.md +++ b/examples/28-newman/README.md @@ -9,7 +9,7 @@ JUnit XML for CI. - [Docker Engine](https://docs.docker.com/engine/install/) with the Compose plugin, on Fedora or RHEL (see [Prerequisites](../PREREQUISITES.md)) -- [Newman](https://www.npmjs.com/package/newman): `npm install -g --prefix ~/.local newman@6.2.3` +- [Newman](https://www.npmjs.com/package/newman): `npm install -g --prefix ~/.local newman@6.2.3 newman-reporter-htmlextra@1.23.1` - `curl` and `jq` for driving the API - ~1 GB free memory (Postgres + 1 app service) diff --git a/examples/28-newman/verify.sh b/examples/28-newman/verify.sh index 979ead3b..825b323b 100755 --- a/examples/28-newman/verify.sh +++ b/examples/28-newman/verify.sh @@ -52,7 +52,7 @@ check "healthz returns ok" \ printf '\n--- 1. Newman collection run ---\n' if ! command -v newman &>/dev/null; then - printf ' \xe2\x9c\x97 newman not installed (npm install -g --prefix ~/.local newman@6.2.3)\n' + printf ' \xe2\x9c\x97 newman not installed (npm install -g --prefix ~/.local newman@6.2.3 newman-reporter-htmlextra@1.23.1)\n' FAIL=$((FAIL + 1)) printf '\n==> Results: %d passed, %d failed\n' "$PASS" "$FAIL" printf '==> FAIL\n' diff --git a/examples/PREREQUISITES.md b/examples/PREREQUISITES.md index e6eb2d18..b3bd561e 100644 --- a/examples/PREREQUISITES.md +++ b/examples/PREREQUISITES.md @@ -155,7 +155,17 @@ conftest --version `~/.local/bin` is on the default `PATH` for Fedora and RHEL login shells. -### hey (HTTP load testing) +### kcat (Kafka inspection) + +Lists topics and reads or produces Kafka messages from the host (example 04 uses it); it +replaces the removed kafka-ui. Optional, since `kafka-console-consumer.sh` inside the +`kafka` container does the same: + +```bash +sudo dnf install -y kcat +``` + +### hey v0.1.5 (HTTP load testing) Used by examples that demonstrate scaling or performance patterns. Download a binary from the [hey releases](https://github.com/rakyll/hey/releases) page, or build it with Go: @@ -164,10 +174,22 @@ the [hey releases](https://github.com/rakyll/hey/releases) page, or build it wit go install github.com/rakyll/hey@v0.1.5 ``` -### ghz (gRPC load testing) +### ghz v0.121.0 (gRPC load testing) -Used by examples with gRPC endpoints. Download a binary from the -[ghz releases](https://github.com/bojand/ghz/releases) page. +Used by examples with gRPC endpoints. Download the pinned binary from the +[ghz releases](https://github.com/bojand/ghz/releases) page: + +```bash +GHZ_VERSION=0.121.0 +BASE="https://github.com/bojand/ghz/releases/download/v${GHZ_VERSION}" +cd "$(mktemp -d)" +curl -fsSLO "${BASE}/ghz-linux-x86_64.tar.gz" +curl -fsSLO "${BASE}/ghz-linux-x86_64.tar.gz.sha256" +echo "$(cat ghz-linux-x86_64.tar.gz.sha256) ghz-linux-x86_64.tar.gz" | sha256sum -c - +mkdir -p ~/.local/bin +tar -xzf ghz-linux-x86_64.tar.gz -C ~/.local/bin ghz +ghz --version +``` ### newman 6.2.3 (Postman CLI, required for example 28) @@ -176,7 +198,7 @@ Used by example 28 (Newman API testing). Requires Node.js. Install into the user ```bash sudo dnf install -y nodejs npm -npm install -g --prefix ~/.local newman@6.2.3 +npm install -g --prefix ~/.local newman@6.2.3 newman-reporter-htmlextra@1.23.1 ``` Verify: @@ -184,6 +206,7 @@ Verify: ```bash curl --version jq --version +kcat -V 2>&1 | head -1 # optional hey -h 2>&1 | head -1 # optional ghz --version # optional newman --version # required for example 28 diff --git a/examples/README.md b/examples/README.md index eb3db25a..f60826cb 100644 --- a/examples/README.md +++ b/examples/README.md @@ -60,10 +60,10 @@ The [`_infra/`](_infra/) directory contains composable infrastructure fragments: | Fragment | Provides | |----------|----------| | `compose-lgtm.yaml` | Grafana, Loki, Tempo, Mimir (observability) | -| `compose-postgres.yaml` | PostgreSQL 16 | +| `compose-postgres.yaml` | PostgreSQL 18 | | `compose-postgres-logical.yaml` | PostgreSQL with logical replication | | `compose-kafka.yaml` | Kafka (KRaft mode, no ZooKeeper) | -| `compose-redis.yaml` | Redis 7 | +| `compose-redis.yaml` | Redis 8 | | `compose-debezium.yaml` | Debezium CDC connector | | `compose-flagd.yaml` | flagd feature flag sidecar | diff --git a/examples/_infra/README.md b/examples/_infra/README.md index e8a41670..592abafd 100644 --- a/examples/_infra/README.md +++ b/examples/_infra/README.md @@ -8,12 +8,13 @@ Composable infrastructure services for the runnable examples. Each example | File | Services | Use when | |------|----------|----------| | `compose-lgtm.yaml` | Grafana LGTM (Loki, Grafana, Tempo, Mimir) + OTel Collector | Every example (observability) | -| `compose-postgres.yaml` | Postgres 16 | Example needs relational storage | -| `compose-postgres-logical.yaml` | Postgres 16 with `wal_level=logical` | Example uses CDC / Debezium | -| `compose-kafka.yaml` | Kafka 3.8 (KRaft) + Kafka UI | Example uses event streaming | -| `compose-redis.yaml` | Redis 7 | Example uses caching or pub/sub | +| `compose-postgres.yaml` | Postgres 18 | Example needs relational storage | +| `compose-postgres-logical.yaml` | Postgres 18 with `wal_level=logical` | Example uses CDC / Debezium | +| `compose-kafka.yaml` | Kafka 4.3 (KRaft) | Example uses event streaming | +| `compose-redis.yaml` | Redis 8 | Example uses caching or pub/sub | | `compose-debezium.yaml` | Debezium / Kafka Connect | Example uses change data capture | | `compose-flagd.yaml` | flagd (OpenFeature) | Example uses feature flags | +| `compose-apicurio.yaml` | Apicurio Registry 3.3 (in-memory H2) | Example uses a schema/API registry | ## Usage @@ -59,10 +60,11 @@ not inside the language directory. | Postgres | 5432 | 5432 | | Kafka (host) | 9092 | 9092 | | Kafka (compose) | 9094 | 9094 | -| Kafka UI | 8090 | 8080 | | Redis | 6379 | 6379 | | Kafka Connect | 8083 | 8083 | | flagd (gRPC) | 8013 | 8013 | +| Apicurio Registry (API) | 8081 | 8080 | +| Apicurio management (`/health/ready`) | 19000 | 9000 | ## Credentials @@ -82,10 +84,10 @@ on the same machine: | lgtm | cndp-lgtm | | postgres | cndp-postgres | | kafka | cndp-kafka | -| kafka-ui | cndp-kafka-ui | | redis | cndp-redis | | kafka-connect | cndp-kafka-connect | | flagd | cndp-flagd | +| apicurio | cndp-apicurio | | (app services) | cndp-{service-name} | The shared network is named `cndp`. Service names (used for DNS inside compose) diff --git a/examples/_infra/compose-apicurio.yaml b/examples/_infra/compose-apicurio.yaml index 46fc1477..108e7939 100644 --- a/examples/_infra/compose-apicurio.yaml +++ b/examples/_infra/compose-apicurio.yaml @@ -1,15 +1,16 @@ services: apicurio: - image: docker.io/apicurio/apicurio-registry:3.0.5 + image: docker.io/apicurio/apicurio-registry:3.3.3 container_name: cndp-apicurio ports: - "8081:8080" + - "19000:9000" # management interface: /health/ready environment: - APICURIO_STORAGE_KIND=sql - APICURIO_STORAGE_SQL_KIND=h2 mem_limit: 512m healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:8080/health/ready"] + test: ["CMD", "curl", "-sf", "http://localhost:9000/health/ready"] interval: 5s timeout: 3s retries: 12 diff --git a/examples/_infra/compose-debezium.yaml b/examples/_infra/compose-debezium.yaml index d0b093cd..606169e7 100644 --- a/examples/_infra/compose-debezium.yaml +++ b/examples/_infra/compose-debezium.yaml @@ -1,6 +1,6 @@ services: kafka-connect: - image: docker.io/debezium/connect:2.7.3.Final + image: quay.io/debezium/connect:3.7.0.Final container_name: cndp-kafka-connect ports: - "8083:8083" @@ -13,12 +13,13 @@ services: - CONFIG_STORAGE_REPLICATION_FACTOR=1 - OFFSET_STORAGE_REPLICATION_FACTOR=1 - STATUS_STORAGE_REPLICATION_FACTOR=1 + - HEAP_OPTS=-Xms256M -Xmx1G depends_on: kafka: condition: service_healthy postgres: condition: service_healthy - mem_limit: 1g + mem_limit: 2g healthcheck: test: ["CMD", "curl", "-sf", "http://localhost:8083/"] interval: 10s diff --git a/examples/_infra/compose-flagd.yaml b/examples/_infra/compose-flagd.yaml index 1b9cf5ec..3d837e4d 100644 --- a/examples/_infra/compose-flagd.yaml +++ b/examples/_infra/compose-flagd.yaml @@ -1,6 +1,6 @@ services: flagd: - image: ghcr.io/open-feature/flagd:latest + image: ghcr.io/open-feature/flagd:v0.17.0 container_name: cndp-flagd ports: - "8013:8013" diff --git a/examples/_infra/compose-kafka.yaml b/examples/_infra/compose-kafka.yaml index 8bd090ee..3b13bbab 100644 --- a/examples/_infra/compose-kafka.yaml +++ b/examples/_infra/compose-kafka.yaml @@ -1,6 +1,6 @@ services: kafka: - image: docker.io/apache/kafka:3.8.0 + image: docker.io/apache/kafka:4.3.1 container_name: cndp-kafka ports: - "9092:9092" @@ -33,18 +33,5 @@ services: retries: 12 start_period: 30s - kafka-ui: - image: docker.io/provectuslabs/kafka-ui:latest - container_name: cndp-kafka-ui - ports: - - "8090:8080" - environment: - - KAFKA_CLUSTERS_0_NAME=local - - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:9094 - - DYNAMIC_CONFIG_ENABLED=true - depends_on: - kafka: - condition: service_healthy - volumes: kafka-data: diff --git a/examples/_infra/compose-lgtm.yaml b/examples/_infra/compose-lgtm.yaml index e76e8163..936f5573 100644 --- a/examples/_infra/compose-lgtm.yaml +++ b/examples/_infra/compose-lgtm.yaml @@ -1,6 +1,6 @@ services: lgtm: - image: docker.io/grafana/otel-lgtm:0.8.1 + image: docker.io/grafana/otel-lgtm:0.36.0 container_name: cndp-lgtm ports: - "3000:3000" @@ -14,7 +14,7 @@ services: - GF_AUTH_DISABLE_LOGIN_FORM=true volumes: - ./otelcol/config.yaml:/otel-lgtm/otelcol-config.yaml:Z - - ./grafana/datasources.yaml:/otel-lgtm/grafana/conf/provisioning/datasources/datasources.yaml:Z + - ./grafana/datasources.yaml:/otel-lgtm/grafana/conf/provisioning/datasources/grafana-datasources.yaml:Z healthcheck: test: ["CMD", "curl", "-sf", "http://localhost:3000/api/health"] interval: 5s diff --git a/examples/_infra/compose-postgres-logical.yaml b/examples/_infra/compose-postgres-logical.yaml index 36ae7103..998de546 100644 --- a/examples/_infra/compose-postgres-logical.yaml +++ b/examples/_infra/compose-postgres-logical.yaml @@ -1,6 +1,6 @@ services: postgres: - image: docker.io/library/postgres:16-alpine + image: docker.io/library/postgres:18.6-alpine container_name: cndp-postgres ports: - "5432:5432" @@ -10,7 +10,7 @@ services: - POSTGRES_PASSWORD=apppass - POSTGRES_INITDB_ARGS=--encoding=UTF8 --locale=C volumes: - - postgres-data:/var/lib/postgresql/data + - postgres-data:/var/lib/postgresql mem_limit: 1g command: - "postgres" diff --git a/examples/_infra/compose-postgres.yaml b/examples/_infra/compose-postgres.yaml index 3862ae9f..62281a56 100644 --- a/examples/_infra/compose-postgres.yaml +++ b/examples/_infra/compose-postgres.yaml @@ -1,6 +1,6 @@ services: postgres: - image: docker.io/library/postgres:16-alpine + image: docker.io/library/postgres:18.6-alpine container_name: cndp-postgres ports: - "5432:5432" @@ -10,7 +10,7 @@ services: - POSTGRES_PASSWORD=apppass - POSTGRES_INITDB_ARGS=--encoding=UTF8 --locale=C volumes: - - postgres-data:/var/lib/postgresql/data + - postgres-data:/var/lib/postgresql mem_limit: 1g command: - "postgres" diff --git a/examples/_infra/compose-redis.yaml b/examples/_infra/compose-redis.yaml index 66b8f015..a9598508 100644 --- a/examples/_infra/compose-redis.yaml +++ b/examples/_infra/compose-redis.yaml @@ -1,6 +1,6 @@ services: redis: - image: docker.io/library/redis:7-alpine + image: docker.io/library/redis:8.10.2-alpine container_name: cndp-redis ports: - "6379:6379" diff --git a/scripts/generate_example_diagrams.py b/scripts/generate_example_diagrams.py index ec37a93d..15a65546 100644 --- a/scripts/generate_example_diagrams.py +++ b/scripts/generate_example_diagrams.py @@ -157,7 +157,7 @@ def gen_09(): {"x1": 360, "y1": 140, "x2": 410, "y2": 160}, ], notes=[ - {"x": 420, "y": 195, "text": "breaking → 409", "size": 10, "color": "#b8650a", "bold": True}, + {"x": 420, "y": 195, "text": "breaking → 400", "size": 10, "color": "#b8650a", "bold": True}, {"x": 420, "y": 210, "text": "additive → 200", "size": 10, "color": "#555555"}, ], )