Wpb 27900

.github/workflows/offline-min.yml

@@ -1,14 +1,40 @@
+# Offline Build Workflow
+#
+# This workflow builds offline deployment artifacts for different profiles:
+#   - default: Production deployment (includes external charts, ansible, terraform)
+#   - build-wiab-staging: Wire-in-a-box (wiab-stag) a production like deployment (includes external charts, ansible, terraform)
+#   - wiab-dev: Wire-in-a-box dev deployment (includes databases-ephemeral)
+#   - min: Minimal deployment
+#
+# Build Optimization via PR Labels:
+#   - No label: No builds run (must add label to trigger builds)
+#   - 'build-min': Builds only min profile
+#   - 'build-all': Explicitly builds all profiles (useful for workflow changes)
+#
+# Push to master/develop: Always builds all profiles regardless of labels
+#
 on:
   push:
-    branches: [5.14*]
+    branches: ["**"]
+    tags: [v*]
     paths-ignore:
-      - '*.md'
-      - '**/*.md'
+      - "*.md"
+      - "**/*.md"
+  pull_request:
+    types: [synchronize, reopened, labeled]
+    branches: ["**"]
+    paths-ignore:
+      - "*.md"
+      - "**/*.md"
 jobs:
-  offline:
-    name: Prepare min offline package
-    # Useful to skip expensive CI when writing docs
-    if: "!contains(github.event.head_commit.message, 'skip ci')"
+
+  # Build min profile
+  build-min:
+    name: Build min profile
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
+      contains(github.event.pull_request.labels.*.name, 'build-all') ||
+      contains(github.event.pull_request.labels.*.name, 'build-min')
     runs-on:
       group: wire-server-deploy
     steps:
@@ -24,28 +50,28 @@ jobs:
       - name: Install nix environment
         run: nix-env -f default.nix -iA env
 
+      - name: Login to Quay
+        run: |
+          echo "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_PASSWORD }}" | oras login -u "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_USERNAME }}" --password-stdin quay.io
+          echo "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_PASSWORD }}" | helm registry login -u "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_USERNAME }}" --password-stdin quay.io
+
       - name: Get upload name
         id: upload_name
-        run: |
-          # FIXME: Tag with a nice release name using the github tag...
-          # SOURCE_TAG=${GITHUB_REF#refs/tags/}
-          echo ::set-output name=UPLOAD_NAME::$GITHUB_SHA
-          # echo ::set-output name=UPLOAD_NAME::${SOURCE_TAG:-$GITHUB_SHA}
+        run: echo "UPLOAD_NAME=$GITHUB_SHA" >> $GITHUB_OUTPUT
 
       - name: Process the min profile build
-        run: ./offline/min-build/build.sh
+        run: ./offline/min-build/build_oci.sh
         env:
-          GPG_PRIVATE_KEY: '${{ secrets.GPG_PRIVATE_KEY }}'
-          DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
+          DOCKER_LOGIN: "${{ secrets.DOCKER_LOGIN }}"
 
       - name: Copy min build assets tarball to S3
         run: |
-          # Upload tarball for each profile by specifying their OUTPUT_TAR path
           aws s3 cp offline/min-build/output/assets.tgz s3://public.wire.com/artifacts/wire-server-deploy-static-min-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz
           echo "Uploaded to: https://s3-$AWS_REGION.amazonaws.com/public.wire.com/artifacts/wire-server-deploy-static-min-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz"
-          # remove the archives from the build to optimize the space on the server
-          rm -rf offline/min-build/output/*
         env:
-          AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
-          AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
+          AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
           AWS_REGION: "eu-west-1"
+
+      - name: Cleanup min build assets
+        run: rm -rf offline/min-build/output/

.github/workflows/package-min-bundle.yml

@@ -0,0 +1,86 @@
+name: Package min bundle
+
+on:
+  push:
+    branches:
+      - wpb-27900
+  workflow_dispatch:
+    inputs:
+      release_stream:
+        description: pull the latest bundle from this release stream
+        type: string
+        required: false
+        default: dev-gov
+      oci_link:
+        description: pull this specific link instead of release_stream, e.g. quay.io/...
+        required: false
+        type: string
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: wpb-27900
+
+      - name: Set up ORAS
+        uses: oras-project/setup-oras@v1
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: '4.2.3'
+
+      - name: Login to Quay
+        run: |
+          echo "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_PASSWORD }}" | oras login -u "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_USERNAME }}" --password-stdin quay.io
+          echo "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_PASSWORD }}" | helm registry login -u "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_USERNAME }}" --password-stdin quay.io
+
+      - name: Download online bundle
+        id: download-bundle
+        env:
+          RELEASE_STREAM: ${{ inputs.release_stream }}
+          OCI_LINK: ${{ inputs.oci_link }}
+        run: |
+          # TODO: REMOVE THIS AFTER FIXING INPUTS FOR PUSH EVENTS
+          # This is a temporary default for local/push testing
+          RELEASE_STREAM=${RELEASE_STREAM:-dev-gov}
+          OCI_LINK="quay.io/wire/bundles/dev/gov:2026.8.0-rc.20260731.113937"
+
+          # TODO: only if RELEASE_STREAM is configured
+
+          TEMP_DIR=$(mktemp -d)
+          echo "TEMP_DIR=$TEMP_DIR" >> $GITHUB_OUTPUT
+
+          if [ -n "$OCI_LINK" ]; then
+            # If OCI_LINK is provided, use it directly
+            oras pull "$OCI_LINK" -o "$TEMP_DIR"
+          else
+            # Otherwise, get the OCI link from the release stream's build.json
+            oras pull "quay.io/wire/release-streams/bundles:$RELEASE_STREAM" -o "$TEMP_DIR"
+
+            BUILD_JSON=$(cat "$TEMP_DIR/build.json")
+            echo "$BUILD_JSON"
+
+            REPOSITORY=$(echo "$BUILD_JSON" | jq -r '.repository')
+            TAG=$(echo "$BUILD_JSON" | jq -r '.tag')
+
+            oras pull "$REPOSITORY:$TAG" -o "$TEMP_DIR"
+          fi
+
+          # Extract the bundle
+          tar -xzf "$TEMP_DIR"/*.tgz -C "$TEMP_DIR"
+
+      - name: Process bundle
+        run: |
+          ls "${{ steps.download-bundle.outputs.TEMP_DIR }}"
+
+      - name: pull helm
+        run: |
+          helm pull oci://quay.io/wire/charts/stable/wire-server:5.14.0-pre.3
+

default.nix

@@ -2,10 +2,29 @@
 
 let
   sources = import ./nix/sources.nix;
-  pkgs = import sources.nixpkgs {
+  # for injecting old gnupg dependancy
+  oldpkgs = import sources.oldpkgs {
     inherit system;
     config = { };
+  };
+  # extract the module for injecting
+  #gnupg1orig = oldpkgs.gnupg1orig;
+
+  pkgs = import sources.nixpkgs {
+    inherit system;
+    config = {
+      # there is a unfree package in current nixpkgs version that will refuse to evaluate
+      # so allowUnfree has to be set
+      # The package in question (vault-1.16.2) is not being used
+      allowUnfree = true;
+    };
+    # layering is important here, the lowest takes precedance in case of overlaps
     overlays = [
+      # custom overlay for injections 
+      # (self: super: {
+      #   gnupg1orig = gnupg1orig;
+      # })
+      # main overlay
       (import ./nix/overlay.nix)
     ];
   };
@@ -26,13 +45,13 @@ rec {
   env = pkgs.buildEnv {
     name = "wire-server-deploy";
     paths = with pkgs; [
-      ansible_2_15
-      pythonForAnsible
-      jmespath
+      customAnsible
       apacheHttpd
       awscli2
       gnumake
-      gnupg
+      gnupg1
+      # injected dependacy gnupg1orig
+      # gnupg1orig
 
       kubernetes-tools
 
@@ -45,18 +64,21 @@ rec {
       skopeo
       sops
       opentofu
-      yq
+      yq-go  # Use yq-go (v4+) explicitly instead of python-yq for consistent YAML processing
       create-container-dump
       list-helm-containers
       mirror-apt-jammy
       generate-gpg1-key
+      create-build-entry
       # Linting
       shellcheck
 
       # general utilities for bash operations
       jq
       gnused
       curl
+      gawk
+      oras
 
       niv
       nix-prefetch-docker
@@ -106,4 +128,4 @@ rec {
       ];
     };
   };
-}
+}

nix/overlay.nix

@@ -2,8 +2,14 @@ self:
 let helm-mapkubeapis = self.callPackage ./pkgs/helm-mapkubeapis.nix { };
 in
 super: {
-  pythonForAnsible = (self.python3.withPackages (_: self.ansible.requiredPythonModules ++ [
-    super.python3Packages.boto
+  customAnsible = (self.python3.withPackages (_: self.ansible.requiredPythonModules ++ [
+    # due to ansible package from nixpkgs missing some dependancies to run kubespray playbook
+    # we are making our own custom ansible package and python interpreter, current ansible-core is 2.16.5
+    super.python3Packages.ansible-core
+
+    # DEPENDENCIES
+    super.python3Packages.jmespath
+    super.python3Packages.botocore
     super.python3Packages.boto3
     super.python3Packages.cryptography
     super.python3Packages.six
@@ -33,11 +39,11 @@ super: {
       # or whenever this derivation is built again without having the result in the binary cache.
       # The public part of the key is shipped with the offline bundle
       # ($aptly_root/public/gpg).
-      # The private key (Github secret) was last replaced on 2024-07-12 and is valid for two years.
+      # The private key (Github secret) was last replaced on 2026-07-15 and is valid for two years.
 
       install -Dm755 ${./scripts/generate-gpg1-key.sh} $out/bin/generate-gpg1-key
       # we *--set* PATH here, to ensure we don't pick wrong gpgs
-      wrapProgram $out/bin/generate-gpg1-key --set PATH '${super.lib.makeBinPath (with self; [ bash coreutils gnupg1orig ])}'
+      wrapProgram $out/bin/generate-gpg1-key --set PATH '${super.lib.makeBinPath (with self; [ bash coreutils ])}'
     '';
   mirror-apt-jammy = super.runCommandNoCC "mirror-apt-jammy"
     {
@@ -46,7 +52,7 @@ super: {
     ''
       install -Dm755 ${./scripts/mirror-apt-jammy.sh} $out/bin/mirror-apt-jammy
       # we need to *--set* PATH here, otherwise aptly will pick the wrong gpg
-      wrapProgram $out/bin/mirror-apt-jammy --set PATH '${super.lib.makeBinPath (with self; [ aptly bash coreutils curl gnupg1orig gnused gnutar ])}'
+      wrapProgram $out/bin/mirror-apt-jammy --set PATH '${super.lib.makeBinPath (with self; [ aptly bash coreutils curl gnused gnutar ])}'
     '';
 
   create-container-dump = super.runCommandNoCC "create-container-dump"
@@ -58,7 +64,6 @@ super: {
         wrapProgram $out/bin/create-container-dump --prefix PATH : '${super.lib.makeBinPath [ self.skopeo ]}'
     '';
 
-
   list-helm-containers = super.runCommandNoCC "list-helm-containers"
     {
       nativeBuildInputs = [ super.makeWrapper ];
@@ -67,4 +72,14 @@ super: {
       install -Dm755 ${./scripts/list-helm-containers.sh} $out/bin/list-helm-containers
       wrapProgram $out/bin/list-helm-containers --prefix PATH : '${super.lib.makeBinPath [ self.kubernetes-helm ]}'
     '';
-}
+
+  create-build-entry = super.runCommandNoCC "create-build-entry"
+    {
+      nativeBuildInputs = [ super.makeWrapper ];
+    }
+    ''
+      install -Dm755 ${./scripts/create-build-entry.sh} $out/bin/create-build-entry
+      wrapProgram $out/bin/create-build-entry --prefix PATH : '${super.lib.makeBinPath (with self; [ bash jq ])}'
+    '';
+
+}

nix/scripts/create-build-entry.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -eou pipefail
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <docker-image-with-tag> <directory>"
+  exit 1
+fi
+
+IMAGE_WITH_TAG=$1
+DIRECTORY=$2
+
+IMAGE=$(echo "$IMAGE_WITH_TAG" | cut -d':' -f1)
+TAG=$(echo "$IMAGE_WITH_TAG" | cut -d':' -f2)
+
+JSON_FILE="$DIRECTORY/images.json"
+
+if [ ! -d "$DIRECTORY" ]; then
+  mkdir -p "$DIRECTORY"
+fi
+
+append_image_entry() {
+  local image=$1
+  local tag=$2
+  local json_file=$3
+
+  if [ -f "$json_file" ]; then
+    existing_content=$(jq '.' "$json_file")
+
+    new_entry=$(jq -n --arg image "$image" --arg tag "$tag" '{$image: $tag}')
+    updated_content=$(echo "$existing_content" | jq --argjson new_entry "$new_entry" '. += [$new_entry]')
+  else
+    updated_content=$(jq -n --arg image "$image" --arg tag "$tag" '[{$image: $tag}]')
+  fi
+
+  echo "$updated_content" | jq '.' > "$json_file"
+}
+
+append_image_entry "$IMAGE" "$TAG" "$JSON_FILE"