diff --git a/.github/workflows/offline-min.yml b/.github/workflows/offline-min.yml index b1d979e4a..0091a83f8 100644 --- a/.github/workflows/offline-min.yml +++ b/.github/workflows/offline-min.yml @@ -1,14 +1,40 @@ +# Offline Build Workflow +# +# This workflow builds offline deployment artifacts for different profiles: +# - default: Production deployment (includes external charts, ansible, terraform) +# - build-wiab-staging: Wire-in-a-box (wiab-stag) a production like deployment (includes external charts, ansible, terraform) +# - wiab-dev: Wire-in-a-box dev deployment (includes databases-ephemeral) +# - min: Minimal deployment +# +# Build Optimization via PR Labels: +# - No label: No builds run (must add label to trigger builds) +# - 'build-min': Builds only min profile +# - 'build-all': Explicitly builds all profiles (useful for workflow changes) +# +# Push to master/develop: Always builds all profiles regardless of labels +# on: push: - branches: [5.14*] + branches: ["**"] + tags: [v*] paths-ignore: - - '*.md' - - '**/*.md' + - "*.md" + - "**/*.md" + pull_request: + types: [synchronize, reopened, labeled] + branches: ["**"] + paths-ignore: + - "*.md" + - "**/*.md" jobs: - offline: - name: Prepare min offline package - # Useful to skip expensive CI when writing docs - if: "!contains(github.event.head_commit.message, 'skip ci')" + + # Build min profile + build-min: + name: Build min profile + if: | + (github.event_name == 'push' && github.ref == 'refs/heads/master') || + contains(github.event.pull_request.labels.*.name, 'build-all') || + contains(github.event.pull_request.labels.*.name, 'build-min') runs-on: group: wire-server-deploy steps: @@ -24,28 +50,28 @@ jobs: - name: Install nix environment run: nix-env -f default.nix -iA env + - name: Login to Quay + run: | + echo "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_PASSWORD }}" | oras login -u "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_USERNAME }}" --password-stdin quay.io + echo "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_PASSWORD }}" | helm registry login -u "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_USERNAME }}" --password-stdin quay.io + - name: Get upload name id: upload_name - run: | - # FIXME: Tag with a nice release name using the github tag... - # SOURCE_TAG=${GITHUB_REF#refs/tags/} - echo ::set-output name=UPLOAD_NAME::$GITHUB_SHA - # echo ::set-output name=UPLOAD_NAME::${SOURCE_TAG:-$GITHUB_SHA} + run: echo "UPLOAD_NAME=$GITHUB_SHA" >> $GITHUB_OUTPUT - name: Process the min profile build - run: ./offline/min-build/build.sh + run: ./offline/min-build/build_oci.sh env: - GPG_PRIVATE_KEY: '${{ secrets.GPG_PRIVATE_KEY }}' - DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}' + DOCKER_LOGIN: "${{ secrets.DOCKER_LOGIN }}" - name: Copy min build assets tarball to S3 run: | - # Upload tarball for each profile by specifying their OUTPUT_TAR path aws s3 cp offline/min-build/output/assets.tgz s3://public.wire.com/artifacts/wire-server-deploy-static-min-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz echo "Uploaded to: https://s3-$AWS_REGION.amazonaws.com/public.wire.com/artifacts/wire-server-deploy-static-min-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz" - # remove the archives from the build to optimize the space on the server - rm -rf offline/min-build/output/* env: - AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}' - AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}' + AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" AWS_REGION: "eu-west-1" + + - name: Cleanup min build assets + run: rm -rf offline/min-build/output/ diff --git a/.github/workflows/package-min-bundle.yml b/.github/workflows/package-min-bundle.yml new file mode 100644 index 000000000..a2175dbf8 --- /dev/null +++ b/.github/workflows/package-min-bundle.yml @@ -0,0 +1,86 @@ +name: Package min bundle + +on: + push: + branches: + - wpb-27900 + workflow_dispatch: + inputs: + release_stream: + description: pull the latest bundle from this release stream + type: string + required: false + default: dev-gov + oci_link: + description: pull this specific link instead of release_stream, e.g. quay.io/... + required: false + type: string + +jobs: + update: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: wpb-27900 + + - name: Set up ORAS + uses: oras-project/setup-oras@v1 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: '4.2.3' + + - name: Login to Quay + run: | + echo "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_PASSWORD }}" | oras login -u "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_USERNAME }}" --password-stdin quay.io + echo "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_PASSWORD }}" | helm registry login -u "${{ secrets.QUAY_WIRE_CI_GOV_READONLY_USERNAME }}" --password-stdin quay.io + + - name: Download online bundle + id: download-bundle + env: + RELEASE_STREAM: ${{ inputs.release_stream }} + OCI_LINK: ${{ inputs.oci_link }} + run: | + # TODO: REMOVE THIS AFTER FIXING INPUTS FOR PUSH EVENTS + # This is a temporary default for local/push testing + RELEASE_STREAM=${RELEASE_STREAM:-dev-gov} + OCI_LINK="quay.io/wire/bundles/dev/gov:2026.8.0-rc.20260731.113937" + + # TODO: only if RELEASE_STREAM is configured + + TEMP_DIR=$(mktemp -d) + echo "TEMP_DIR=$TEMP_DIR" >> $GITHUB_OUTPUT + + if [ -n "$OCI_LINK" ]; then + # If OCI_LINK is provided, use it directly + oras pull "$OCI_LINK" -o "$TEMP_DIR" + else + # Otherwise, get the OCI link from the release stream's build.json + oras pull "quay.io/wire/release-streams/bundles:$RELEASE_STREAM" -o "$TEMP_DIR" + + BUILD_JSON=$(cat "$TEMP_DIR/build.json") + echo "$BUILD_JSON" + + REPOSITORY=$(echo "$BUILD_JSON" | jq -r '.repository') + TAG=$(echo "$BUILD_JSON" | jq -r '.tag') + + oras pull "$REPOSITORY:$TAG" -o "$TEMP_DIR" + fi + + # Extract the bundle + tar -xzf "$TEMP_DIR"/*.tgz -C "$TEMP_DIR" + + - name: Process bundle + run: | + ls "${{ steps.download-bundle.outputs.TEMP_DIR }}" + + - name: pull helm + run: | + helm pull oci://quay.io/wire/charts/stable/wire-server:5.14.0-pre.3 + diff --git a/default.nix b/default.nix index 8df06855f..2c5b98872 100644 --- a/default.nix +++ b/default.nix @@ -2,10 +2,29 @@ let sources = import ./nix/sources.nix; - pkgs = import sources.nixpkgs { + # for injecting old gnupg dependancy + oldpkgs = import sources.oldpkgs { inherit system; config = { }; + }; + # extract the module for injecting + #gnupg1orig = oldpkgs.gnupg1orig; + + pkgs = import sources.nixpkgs { + inherit system; + config = { + # there is a unfree package in current nixpkgs version that will refuse to evaluate + # so allowUnfree has to be set + # The package in question (vault-1.16.2) is not being used + allowUnfree = true; + }; + # layering is important here, the lowest takes precedance in case of overlaps overlays = [ + # custom overlay for injections + # (self: super: { + # gnupg1orig = gnupg1orig; + # }) + # main overlay (import ./nix/overlay.nix) ]; }; @@ -26,13 +45,13 @@ rec { env = pkgs.buildEnv { name = "wire-server-deploy"; paths = with pkgs; [ - ansible_2_15 - pythonForAnsible - jmespath + customAnsible apacheHttpd awscli2 gnumake - gnupg + gnupg1 + # injected dependacy gnupg1orig + # gnupg1orig kubernetes-tools @@ -45,11 +64,12 @@ rec { skopeo sops opentofu - yq + yq-go # Use yq-go (v4+) explicitly instead of python-yq for consistent YAML processing create-container-dump list-helm-containers mirror-apt-jammy generate-gpg1-key + create-build-entry # Linting shellcheck @@ -57,6 +77,8 @@ rec { jq gnused curl + gawk + oras niv nix-prefetch-docker @@ -106,4 +128,4 @@ rec { ]; }; }; -} +} \ No newline at end of file diff --git a/nix/overlay.nix b/nix/overlay.nix index 8d53f080e..469251789 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -2,8 +2,14 @@ self: let helm-mapkubeapis = self.callPackage ./pkgs/helm-mapkubeapis.nix { }; in super: { - pythonForAnsible = (self.python3.withPackages (_: self.ansible.requiredPythonModules ++ [ - super.python3Packages.boto + customAnsible = (self.python3.withPackages (_: self.ansible.requiredPythonModules ++ [ + # due to ansible package from nixpkgs missing some dependancies to run kubespray playbook + # we are making our own custom ansible package and python interpreter, current ansible-core is 2.16.5 + super.python3Packages.ansible-core + + # DEPENDENCIES + super.python3Packages.jmespath + super.python3Packages.botocore super.python3Packages.boto3 super.python3Packages.cryptography super.python3Packages.six @@ -33,11 +39,11 @@ super: { # or whenever this derivation is built again without having the result in the binary cache. # The public part of the key is shipped with the offline bundle # ($aptly_root/public/gpg). - # The private key (Github secret) was last replaced on 2024-07-12 and is valid for two years. + # The private key (Github secret) was last replaced on 2026-07-15 and is valid for two years. install -Dm755 ${./scripts/generate-gpg1-key.sh} $out/bin/generate-gpg1-key # we *--set* PATH here, to ensure we don't pick wrong gpgs - wrapProgram $out/bin/generate-gpg1-key --set PATH '${super.lib.makeBinPath (with self; [ bash coreutils gnupg1orig ])}' + wrapProgram $out/bin/generate-gpg1-key --set PATH '${super.lib.makeBinPath (with self; [ bash coreutils ])}' ''; mirror-apt-jammy = super.runCommandNoCC "mirror-apt-jammy" { @@ -46,7 +52,7 @@ super: { '' install -Dm755 ${./scripts/mirror-apt-jammy.sh} $out/bin/mirror-apt-jammy # we need to *--set* PATH here, otherwise aptly will pick the wrong gpg - wrapProgram $out/bin/mirror-apt-jammy --set PATH '${super.lib.makeBinPath (with self; [ aptly bash coreutils curl gnupg1orig gnused gnutar ])}' + wrapProgram $out/bin/mirror-apt-jammy --set PATH '${super.lib.makeBinPath (with self; [ aptly bash coreutils curl gnused gnutar ])}' ''; create-container-dump = super.runCommandNoCC "create-container-dump" @@ -58,7 +64,6 @@ super: { wrapProgram $out/bin/create-container-dump --prefix PATH : '${super.lib.makeBinPath [ self.skopeo ]}' ''; - list-helm-containers = super.runCommandNoCC "list-helm-containers" { nativeBuildInputs = [ super.makeWrapper ]; @@ -67,4 +72,14 @@ super: { install -Dm755 ${./scripts/list-helm-containers.sh} $out/bin/list-helm-containers wrapProgram $out/bin/list-helm-containers --prefix PATH : '${super.lib.makeBinPath [ self.kubernetes-helm ]}' ''; -} + + create-build-entry = super.runCommandNoCC "create-build-entry" + { + nativeBuildInputs = [ super.makeWrapper ]; + } + '' + install -Dm755 ${./scripts/create-build-entry.sh} $out/bin/create-build-entry + wrapProgram $out/bin/create-build-entry --prefix PATH : '${super.lib.makeBinPath (with self; [ bash jq ])}' + ''; + +} \ No newline at end of file diff --git a/nix/scripts/create-build-entry.sh b/nix/scripts/create-build-entry.sh new file mode 100755 index 000000000..e24cba1ae --- /dev/null +++ b/nix/scripts/create-build-entry.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -eou pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +IMAGE_WITH_TAG=$1 +DIRECTORY=$2 + +IMAGE=$(echo "$IMAGE_WITH_TAG" | cut -d':' -f1) +TAG=$(echo "$IMAGE_WITH_TAG" | cut -d':' -f2) + +JSON_FILE="$DIRECTORY/images.json" + +if [ ! -d "$DIRECTORY" ]; then + mkdir -p "$DIRECTORY" +fi + +append_image_entry() { + local image=$1 + local tag=$2 + local json_file=$3 + + if [ -f "$json_file" ]; then + existing_content=$(jq '.' "$json_file") + + new_entry=$(jq -n --arg image "$image" --arg tag "$tag" '{$image: $tag}') + updated_content=$(echo "$existing_content" | jq --argjson new_entry "$new_entry" '. += [$new_entry]') + else + updated_content=$(jq -n --arg image "$image" --arg tag "$tag" '[{$image: $tag}]') + fi + + echo "$updated_content" | jq '.' > "$json_file" +} + +append_image_entry "$IMAGE" "$TAG" "$JSON_FILE" diff --git a/nix/scripts/create-container-dump.sh b/nix/scripts/create-container-dump.sh index a098d0d79..cecf792eb 100644 --- a/nix/scripts/create-container-dump.sh +++ b/nix/scripts/create-container-dump.sh @@ -8,33 +8,77 @@ if [[ ! $# -eq 1 ]]; then exit 1 fi +export HTTP_TIMEOUT=600 # Timeout in seconds (default is typically 90) +export REGISTRY_TIMEOUT=600 # Registry specific timeout + +output_dir=$1 mkdir -p $1 + # Download all the docker images into $1, and append its name to an index.txt # If this errors out for you, copy default-policy.json from the skopeo repo to # /etc/containers/policy.json while IFS= read -r image; do - # sanitize the image file name, replace slashes with underscores, suffix with .tar - image_filename=$(sed -r "s/[:\/]/_/g" <<< $image) - image_path=$(realpath $1)/${image_filename}.tar - if [[ -e $image_path ]];then - echo "Skipping $image_filename…" + +# sanitize the image file name, replace slashes with underscores, suffix with .tar + image_filename=$(sed -r "s/[:\/]/_/g" <<< "$image") + image_path="$(realpath "$1")/${image_filename}.tar" + + if [[ -s "$image_path" ]]; then + echo "Skipping $image_filename…" + continue + fi + + echo "Fetching $image_filename…" + + # All of these images should be publicly fetchable, especially given we + # ship public tarballs containing these images. + # ci.sh already honors DOCKER_LOGIN, so do the same here, otherwise + # fallback to unauthorized fetching. + + # If an image has both a tag and digest, remove the tag. Return the original if there is no match. + image_trimmed=$(echo "$image" | sed -E 's/(.+)(:.+(@.+))/\1\3/') + + tmp_path="${image_path}.tmp" + rm -f "$tmp_path" + + success=false + + for attempt in {1..5}; do + echo "Attempt $attempt/5 for $image_trimmed" + + if [[ -n "${DOCKER_LOGIN:-}" && "$image" =~ quay.io/wire ]]; then + skopeo copy --insecure-policy \ + --src-creds "$DOCKER_LOGIN" \ + --retry-times 10 \ + "docker://$image_trimmed" \ + "docker-archive:${tmp_path}" \ + --additional-tag "$image" || rc=$? else - echo "Fetching $image_filename…" - - # All of these images should be publicly fetchable, especially given we - # ship public tarballs containing these images. - # ci.sh already honors DOCKER_LOGIN, so do the same here, otherwise - # fallback to unauthorized fetching. - - # If an image has both a tag and digest, remove the tag. Return the original if there is no match. - image_trimmed=$(echo "$image" | sed -E 's/(.+)(:.+(@.+))/\1\3/') - if [[ -n "${DOCKER_LOGIN:-}" && "$image" =~ quay.io/wire ]];then - skopeo copy --insecure-policy --src-creds "$DOCKER_LOGIN" \ - docker://$image_trimmed docker-archive:${image_path} --additional-tag $image - else - skopeo copy --insecure-policy \ - docker://$image_trimmed docker-archive:${image_path} --additional-tag $image - fi - echo "${image_filename}.tar" >> $(realpath "$1")/index.txt + skopeo copy --insecure-policy \ + --retry-times 10 \ + "docker://$image_trimmed" \ + "docker-archive:${tmp_path}" \ + --additional-tag "$image" || rc=$? + fi + + rc=$? + + if [[ $rc -eq 0 && -s "$tmp_path" ]]; then + mv "$tmp_path" "$image_path" + success=true + break fi + + echo "Fetch failed for $image_trimmed with rc=$rc; retrying…" + rm -f "$tmp_path" + sleep $((attempt * 20)) + done + + if [[ "$success" != true ]]; then + echo "ERROR: failed to fetch $image after retries" >&2 + exit 1 + fi + + echo "${image_filename}.tar" >> "$(realpath "$1")/index.txt" + create-build-entry "$image" "$output_dir" done diff --git a/nix/scripts/list-helm-containers.sh b/nix/scripts/list-helm-containers.sh index c96f5526e..dd51e73fa 100644 --- a/nix/scripts/list-helm-containers.sh +++ b/nix/scripts/list-helm-containers.sh @@ -5,10 +5,17 @@ # those. # In cases where no container image tag has been specified, it'll use `latest`. # The list is sorted and deduplicated, then printed to stdout. -set -eou pipefail +set -euo pipefail VALUES_DIR="" HELM_IMAGE_TREE_FILE="" +VALUES_TYPE="" + +# Extract images using yq-go (v4+) syntax +# Note: This requires yq-go to be in PATH (see default.nix) +extract_images() { + yq eval '.. | select(has("image")) | .image' "$1" 2>/dev/null || true +} # Parse the arguments for arg in "$@" @@ -20,6 +27,9 @@ do HELM_IMAGE_TREE_FILE=*) HELM_IMAGE_TREE_FILE="${arg#*=}" ;; + VALUES_TYPE=*) + VALUES_TYPE="${arg#*=}" + ;; *) echo "Unknown argument: $arg" >&2 exit 1 @@ -27,12 +37,27 @@ do esac done -if [[ -z "$VALUES_DIR" || -z "$HELM_IMAGE_TREE_FILE" ]]; then - echo "Error: Both VALUES_DIR and HELM_IMAGE_TREE_FILE must be provided." >&2 - echo "Usage: $0 VALUES_DIR= HELM_IMAGE_TREE_FILE=" >&2 +if [[ -z "$VALUES_DIR" || -z "$HELM_IMAGE_TREE_FILE" || -z "$VALUES_TYPE" ]]; then + echo "Error: VALUES_DIR, HELM_IMAGE_TREE_FILE and VALUES_TYPE must be provided." >&2 + echo "Usage: $0 VALUES_DIR= HELM_IMAGE_TREE_FILE= [VALUES_TYPE=]" >&2 exit 1 fi +# create a dependency tree between helm chart and images +append_chart_entry() { + local chart=$1 + local images=$2 + local json_file=$3 + + if [ ! -s "$json_file" ]; then + echo '[]' > "$json_file" + fi + + existing_content=$(jq '.' "$json_file") + new_entry=$(jq -n --arg chart "$chart" --argjson images "$images" '{"chart": $chart, "images": $images}') + updated_content=$(echo "$existing_content" | jq --argjson new_entry "$new_entry" '. += [$new_entry]') + echo "$updated_content" | jq '.' > "$json_file" +} # Some of these images don't contain a "latest" tag. We don't to download /ALL/ # of them, but only :latest in that case - it's bad enough there's no proper @@ -55,20 +80,72 @@ function optionally_complain() { images="" # For each helm chart passed in from stdin, use the example values to # render the charts, and assemble the list of images this would fetch. +chart_count=0 while IFS= read -r chart; do - echo "Running helm template on chart ${chart}…" >&2 - current_images=$(helm template --debug "${chart}" \ - --set federate.dtls.tls.key=emptyString \ - --set federate.dtls.tls.crt=emptyString \ - $( [[ -f "${VALUES_DIR}"/$(basename "${chart}")/prod-values.example.yaml ]] && echo "-f ${VALUES_DIR}/$(basename "${chart}")/prod-values.example.yaml" ) \ - $( [[ -f "${VALUES_DIR}"/$(basename "${chart}")/prod-secrets.example.yaml ]] && echo "-f ${VALUES_DIR}/$(basename "${chart}")/prod-secrets.example.yaml" ) \ - | yq -r '..|.image? | select(.)' | optionally_complain | sort -u) + chart_count=$((chart_count + 1)) + echo "[$chart_count] Running helm template on chart ${chart}…" >&2 + set +e # Temporarily disable exit on error + # Determine values file to use (prod first, then demo as fallback) + values_file="" + if [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-values.example.yaml ]]; then + values_file="${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-values.example.yaml" + elif [[ -f "${VALUES_DIR}"/$(basename "${chart}")/demo-values.example.yaml ]]; then + values_file="${VALUES_DIR}/$(basename "${chart}")/demo-values.example.yaml" + echo "Using demo values for $(basename $chart) (no ${VALUES_TYPE} values found)" >&2 + fi + + # Determine secrets file to use + secrets_file="" + if [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-secrets.example.yaml ]]; then + secrets_file="${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-secrets.example.yaml" + elif [[ -f "${VALUES_DIR}"/$(basename "${chart}")/demo-secrets.example.yaml ]]; then + secrets_file="${VALUES_DIR}/$(basename "${chart}")/demo-secrets.example.yaml" + fi + + # Save helm output to temp file to check exit code before parsing + # This prevents yq from attempting to parse helm error messages + temp_helm_output=$(mktemp) + helm template "${chart}" \ + $( [[ -n "$values_file" ]] && echo "-f $values_file" ) \ + $( [[ -n "$secrets_file" ]] && echo "-f $secrets_file" ) \ + > "$temp_helm_output" 2>&1 + + helm_exit_code=$? + + # Extract images using version-appropriate yq syntax + if [[ $helm_exit_code -eq 0 ]]; then + raw_images=$(extract_images "$temp_helm_output" | grep -v "^null$" | grep -v "^---$" | grep -v "^$" || true) + else + raw_images="" + fi + + set -e # Re-enable exit on error + + if [[ $helm_exit_code -ne 0 ]]; then + echo "ERROR: Failed to process chart $(basename $chart)" >&2 + echo "Chart path: $chart" >&2 + echo "Values file: ${values_file:-none}" >&2 + echo "Secrets file: ${secrets_file:-none}" >&2 + echo "Helm error output:" >&2 + cat "$temp_helm_output" >&2 + echo "Try running: helm template $chart $([ -n "$values_file" ] && echo "-f $values_file") $([ -n "$secrets_file" ] && echo "-f $secrets_file")" >&2 + raw_images="" + fi + + rm -f "$temp_helm_output" + + # Process extracted images + if [[ -n "$raw_images" ]]; then + current_images=$(echo "$raw_images" | grep -v "^$" | optionally_complain | sort -u) + else + current_images="" + fi images+="$current_images\n" if [[ -n "$current_images" ]]; then - basename "${chart}" >> "${HELM_IMAGE_TREE_FILE}" - echo -e "$current_images\n" >> "${HELM_IMAGE_TREE_FILE}" - #echo -e "\n" >> "${HELM_IMAGE_TREE_FILE}" + current_images=$(echo "$current_images" | awk NF) + image_array=$(jq -Rn --arg images "$current_images" '$images | split("\n")') + append_chart_entry "$(basename $chart)" "$image_array" "${HELM_IMAGE_TREE_FILE}" fi done -echo -e "$images" | grep . | sort -u +echo -e "$images" | grep . | sort -u || true diff --git a/nix/sources.json b/nix/sources.json index 3068042f5..a6a54e2c8 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -5,10 +5,10 @@ "homepage": "https://github.com/nmattia/niv", "owner": "nmattia", "repo": "niv", - "rev": "9cb7ef336bb71fd1ca84fc7f2dff15ef4b033f2a", - "sha256": "1ajyqr8zka1zlb25jx1v4xys3zqmdy3prbm1vxlid6ah27a8qnzh", + "rev": "1ad9d90fea53b0e794cdbabf5515b39e539bc148", + "sha256": "1abhp43jpr8iaixyflr86zzj8ps0v8xzax301ilsvp07w3h23dwz", "type": "tarball", - "url": "https://github.com/nmattia/niv/archive/9cb7ef336bb71fd1ca84fc7f2dff15ef4b033f2a.tar.gz", + "url": "https://github.com/nmattia/niv/archive/1ad9d90fea53b0e794cdbabf5515b39e539bc148.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs": { @@ -17,10 +17,22 @@ "homepage": "https://github.com/NixOS/nixpkgs", "owner": "NixOS", "repo": "nixpkgs", - "rev": "057f9aecfb71c4437d2b27d3323df7f93c010b7e", - "sha256": "1ndiv385w1qyb3b18vw13991fzb9wg4cl21wglk89grsfsnra41k", + "rev": "7901e285e8b54a2f0e51ce6183980738447130e5", + "sha256": "1gdwfkqy7zich82dzxzjcsi613zm7p8cigj2fzqsagn63f5g0pnr", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/057f9aecfb71c4437d2b27d3323df7f93c010b7e.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/7901e285e8b54a2f0e51ce6183980738447130e5.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "oldpkgs": { + "branch": "master", + "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to", + "homepage": "https://github.com/NixOS/nixpkgs", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "7901e285e8b54a2f0e51ce6183980738447130e5", + "sha256": "1gdwfkqy7zich82dzxzjcsi613zm7p8cigj2fzqsagn63f5g0pnr", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/7901e285e8b54a2f0e51ce6183980738447130e5.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } diff --git a/offline/min-build/build_oci.sh b/offline/min-build/build_oci.sh new file mode 100755 index 000000000..f7af325da --- /dev/null +++ b/offline/min-build/build_oci.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# this directory will be created to store all the output files +OUTPUT_DIR="$SCRIPT_DIR/output" +# ROOT_DIR points to dir where ansible,bin, values etc can be located +# expected structure to be: /wire-server-deploy/offline/default-build/build.sh +ROOT_DIR="${SCRIPT_DIR}/../../" + +mkdir -p "${OUTPUT_DIR}"/containers-{helm,other,system,adminhost} "${OUTPUT_DIR}"/binaries "${OUTPUT_DIR}"/versions + +# Define the output tar file +OUTPUT_TAR="${OUTPUT_DIR}/assets.tgz" + +TASKS_DIR="${SCRIPT_DIR}/../tasks" + +# for optmization purposes, if these tarballs are already processed by previous profiles check wire-server-deploy/.github/workflows/offline.yml, one can copy those artifacts from previous profiles to your profile by using +#cp $SCRIPT_DIR/..//output/containers-helm.tar "${OUTPUT_DIR}"/ +# one need to comment the tasks below for which one wants to optimize the build + +# Any of the tasks can be skipped by commenting them out +# however, mind the dependencies between them and how they are grouped + +# Processing helm charts +# -------------------------- + +# pulling the charts, charts to be skipped are passed as arguments HELM_CHART_EXCLUDE_LIST +# "${TASKS_DIR}"/proc_pull_charts.sh OUTPUT_DIR="${OUTPUT_DIR}" HELM_CHART_EXCLUDE_LIST="inbucket,wire-server-enterprise,k8ssandra-operator,k8ssandra-test-cluster,elasticsearch-ephemeral,elasticsearch-curator,rabbitmq,demo-smtp,fake-aws,fake-aws-s3,postgresql,keycloakx,openebs,nginx-ingress-controller,kibana,restund,fluent-bit,aws-ingress,databases-ephemeral,redis-cluster,calling-test" + +"${TASKS_DIR}"/proc_pull_oci_charts.sh OUTPUT_DIR="${OUTPUT_DIR}" CHART_LINK="quay.io/wire/bundles/dev/gov:2026.8.0-rc.20260812.114832" + +# copy local copy of values from root directory to output directory +cp -r "${ROOT_DIR}"/values "${OUTPUT_DIR}"/ + +# removing the values/$chart directories in values directory if not required +"${SCRIPT_DIR}"/pre_clean_values_1.sh VALUES_DIR="${OUTPUT_DIR}/values" HELM_CHART_EXCLUDE_LIST="inbucket,wire-server-enterprise,k8ssandra-operator,k8ssandra-test-cluster,elasticsearch-ephemeral,elasticsearch-curator,rabbitmq,demo-smtp,fake-aws,fake-aws-s3,postgresql,keycloakx,openebs,nginx-ingress-controller,kibana,restund,fluent-bit,aws-ingress,databases-ephemeral,redis-cluster,calling-test" + +# all basic chart pre-processing tasks +"${TASKS_DIR}"/pre_chart_process_0.sh "${OUTPUT_DIR}" + +# all extra pre chart processing tasks for this profile should come here +# pre_chart_process_1.sh +# pre_chart_process_2.sh + +# processing the charts +# here we also filter the images post processing the helm charts +# pass the image names to be filtered as arguments as regex #IMAGE_EXCLUDE_LIST='brig|galley' +"${TASKS_DIR}"/process_charts.sh OUTPUT_DIR="${OUTPUT_DIR}" #IMAGE_EXCLUDE_LIST="" + +# all basic chart pre-processing tasks +"${TASKS_DIR}"/post_chart_process_0.sh "${OUTPUT_DIR}" + +# all extra post chart processing tasks for this profile should come here +# post_chart_process_1.sh +# post_chart_process_2.sh + +# -------------------------- + +# Following tasks are independent from each other +# -------------------------- + +# Just need to ship the zauth container in containers-adminhost +"${TASKS_DIR}"/build_adminhost_containers.sh "${OUTPUT_DIR}" --zauth + +# -------------------------- + +# List of directories and files to include in the tar archive +ITEMS_TO_ARCHIVE=( + "containers-adminhost" + "containers-helm.tar" + "charts" + "values" + "versions" +) + +# Function to check if an item exists +check_item_exists() { + local item=$1 + if [[ ! -e "$item" ]]; then + echo "Error: $item does not exist." + exit 1 + fi +} + +cd "$OUTPUT_DIR" || { echo "Error: Cannot change to directory $OUTPUT_DIR"; exit 1; } + +for item in "${ITEMS_TO_ARCHIVE[@]}"; do + check_item_exists "$item" +done + +# Create the tar archive with relative paths +tar czf "$OUTPUT_TAR" "${ITEMS_TO_ARCHIVE[@]}" + +# Dumping details of versions for the build and packed +echo "Dump of versions/helm_image_tree.json" +cat "${OUTPUT_DIR}/versions/helm_image_tree.json" + +echo "Dump of manifest.yaml used" +cat "${OUTPUT_DIR}/manifest.yaml" + diff --git a/offline/tasks/proc_pull_oci_charts.sh b/offline/tasks/proc_pull_oci_charts.sh new file mode 100755 index 000000000..07ec80dcd --- /dev/null +++ b/offline/tasks/proc_pull_oci_charts.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +set -x -euo pipefail + +OUTPUT_DIR="" +CHART_LINK="" + +# Parse the arguments +for arg in "$@" +do + case $arg in + OUTPUT_DIR=*) + OUTPUT_DIR="${arg#*=}" + ;; + CHART_LINK=*) + CHART_LINK="${arg#*=}" + ;; + *) + echo "Unknown argument: $arg" >&2 + exit 1 + ;; + esac +done + +# Check if OUTPUT_DIR is set +if [[ -z "$OUTPUT_DIR" ]] || [[ -z "$CHART_LINK" ]] ; then + echo "usage: $0 OUTPUT_DIR=\"output-dir\" CHART_LINK=\CHART_LINK\"" >&2 + exit 1 +fi + +echo "Pulling Helm charts in $OUTPUT_DIR using $CHART_LINK" + +# pull_charts() accepts charts in format +# +# on stdin +pull_charts() { + echo "Pulling charts into ${OUTPUT_DIR}/charts ..." + mkdir -p "${OUTPUT_DIR}"/charts + + home=$(mktemp -d) + export HELM_CACHE_HOME="$home" + export HELM_DATA_HOME="$home" + export HELM_CONFIG_HOME="$home" + + declare -A repos + # needed to handle associative array lookup + set +u + + while IFS=$'\n' read -r line + do + echo "$line" + IFS=$' ' read -r -a parts <<< "$line" + name=${parts[0]} + repo=${parts[1]} + version=${parts[2]} + + (cd "${OUTPUT_DIR}"/charts; helm pull --untar "$repo:$version" && rm -rf $name:$version) + done + echo "Pulling charts done." +} + +manifest_chart_release () { + + manifest_file="$1" + yq -r ' + .artifacts[] + | select(.kind == "helm-chart") + | [.name, .repository, .tag] + | join(" ") + ' "${manifest_file}" +} + +pull_oci_artifact(){ + oras pull ${CHART_LINK} --output ${OUTPUT_DIR} + + ARTIFACT=$(oras manifest fetch "${CHART_LINK}" \ + | jq -r '.layers[0].annotations["org.opencontainers.image.title"]') + + ls -lh "${OUTPUT_DIR}/${ARTIFACT}" + + tar -xzf "${OUTPUT_DIR}/${ARTIFACT}" -C "$OUTPUT_DIR" +} + +pull_oci_artifact + +manifest_chart_release "${OUTPUT_DIR}/manifest.yaml" | pull_charts diff --git a/offline/tasks/process_charts.sh b/offline/tasks/process_charts.sh index f7ef6d700..13db03614 100755 --- a/offline/tasks/process_charts.sh +++ b/offline/tasks/process_charts.sh @@ -1,10 +1,13 @@ #!/usr/bin/env bash -set -euo pipefail +set -exuo pipefail OUTPUT_DIR="" # Default exclude list IMAGE_EXCLUDE_LIST="" +# Default values type will expect to use prod values +VALUES_TYPE="prod" + # Parse the arguments for arg in "$@" do @@ -12,6 +15,9 @@ do OUTPUT_DIR=*) OUTPUT_DIR="${arg#*=}" ;; + VALUES_TYPE=*) + VALUES_TYPE="${arg#*=}" + ;; IMAGE_EXCLUDE_LIST=*) IMAGE_EXCLUDE_LIST="${arg#*=}" ;; @@ -24,13 +30,13 @@ done # Check if OUTPUT_DIR is set if [[ -z "$OUTPUT_DIR" ]]; then - echo "usage: $0 OUTPUT_DIR=\"output-dir\" [IMAGE_EXCLUDE_LIST=\"image1\|image2...\"]" >&2 + echo "usage: $0 OUTPUT_DIR=\"output-dir\" [IMAGE_EXCLUDE_LIST=\"image1\|image2...\"] [VALUES_TYPE=\"prod\"]" >&2 exit 1 fi -echo "Processing Helm charts in ${OUTPUT_DIR}" +echo "Processing Helm charts in ${OUTPUT_DIR} with VALUES_TYPE=${VALUES_TYPE}" -HELM_IMAGE_TREE_FILE="${OUTPUT_DIR}/versions/helm_image_tree.txt" +HELM_IMAGE_TREE_FILE="${OUTPUT_DIR}/versions/helm_image_tree.json" touch "${HELM_IMAGE_TREE_FILE}" # Check if IMAGE_EXCLUDE_LIST is set, otherwise use a default pattern that matches nothing @@ -42,13 +48,21 @@ echo "Excluding images matching the pattern: $EXCLUDE_PATTERN" # containers (e.g. `quay.io_wire_galley-integration_4.22.0`.) for chartPath in "${OUTPUT_DIR}"/charts/*; do echo "$chartPath" -done | list-helm-containers VALUES_DIR="${OUTPUT_DIR}"/values HELM_IMAGE_TREE_FILE="$HELM_IMAGE_TREE_FILE" | grep -v "\-integration:" > "${OUTPUT_DIR}"/images +done | list-helm-containers VALUES_DIR="${OUTPUT_DIR}"/values HELM_IMAGE_TREE_FILE="$HELM_IMAGE_TREE_FILE" VALUES_TYPE="$VALUES_TYPE" | grep -v "\-integration:" > "${OUTPUT_DIR}"/images # Omit integration test # containers (e.g. `quay.io_wire_galley-integration_4.22.0`.) sed -i '/-integration/d' "${HELM_IMAGE_TREE_FILE}" +# Replace docker.io/bitnami with docker.io/bitnamilegacy and log updated images +# https://github.com/bitnami/charts/issues/35164 +echo "Replacing bitnami with bitnamilegacy..." +sed -i 's|bitnami/|bitnamilegacy/|g' "${OUTPUT_DIR}"/images +sed -i 's|bitnami/|bitnamilegacy/|g' "${HELM_IMAGE_TREE_FILE}" +echo "Updated images:" +grep "bitnamilegacy" "${OUTPUT_DIR}"/images || echo "No bitnami images found" + grep -vE "$EXCLUDE_PATTERN" "${OUTPUT_DIR}"/images | create-container-dump "${OUTPUT_DIR}"/containers-helm tar cf "${OUTPUT_DIR}"/containers-helm.tar -C "${OUTPUT_DIR}" containers-helm -cp "${OUTPUT_DIR}"/containers-helm/index.txt "${OUTPUT_DIR}"/versions/containers-helm.txt +mv "${OUTPUT_DIR}/containers-helm/images.json" "${OUTPUT_DIR}"/versions/containers_helm_images.json \ No newline at end of file