diff --git a/.github/dependabot.yml b/.github/dependabot.yml index aea4db326e..931a5424e8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -108,3 +108,15 @@ updates: - "python" commit-message: prefix: "chore(docs)" + + - package-ecosystem: "docker-compose" + directory: "/tests/multi-runner-orchestration" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + labels: + - "dependencies" + - "docker-compose" + commit-message: + prefix: "chore(docker-compose)" diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index b8945d4b34..de902585c4 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -52,8 +52,6 @@ jobs: - default - ephemeral - multi-runner - - multi-runner-v2 - - multi-runner-scale-set - migration-test - termination-watcher services: diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index 9a47a8e311..a288497c79 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -2,10 +2,12 @@ name: "Smoke Tests" on: pull_request: - paths: - - lambdas/** + paths: - "**/*.tf" - "**/*.hcl" + - "lambdas/**" + - "tests/ministack/**" + - "tests/multi-runner-orchestration/**" - ".github/workflows/smoke-tests.yml" workflow_dispatch: @@ -16,33 +18,14 @@ concurrency: permissions: contents: read -env: - AWS_ACCESS_KEY_ID: "000000000000" - AWS_DEFAULT_REGION: eu-west-1 - AWS_EC2_METADATA_DISABLED: "true" - AWS_ENDPOINT_URL: http://localhost:4566 - AWS_REGION: eu-west-1 - AWS_SECRET_ACCESS_KEY: test-only - TF_IN_AUTOMATION: "true" - TF_INPUT: "false" - jobs: - control_plane_smoke: - name: Run webhook and pool lifecycle smoke test against MiniStack + ministack_smoke: + name: Run webhook and scale-set smoke tests against MiniStack runs-on: ubuntu-latest - timeout-minutes: 30 - services: - ministack: - image: ghcr.io/ministackorg/ministack:1.5.13@sha256:ce3c906f2866ff953ce4c56f06b1fa3e453bc32e41c00de17b5f5a8672c5a42c - ports: - - 4566:4566 - options: --add-host=host.docker.internal:host-gateway - env: - MINISTACK_ACCOUNT_ID: "000000000000" - MINISTACK_REGION: eu-west-1 + timeout-minutes: 120 steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 with: egress-policy: audit @@ -51,116 +34,13 @@ jobs: with: persist-credentials: false - - name: Setup Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version-file: lambdas/.nvmrc - package-manager-cache: false - - - name: Setup Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: latest - terraform_wrapper: false - - - name: Install Lambda dependencies - working-directory: lambdas - run: yarn install --frozen-lockfile - - - name: Build smoke-test Lambda distributions - working-directory: lambdas + - name: Docker Compose Build and Run run: | - yarn workspace @aws-github-runner/webhook dist - yarn workspace @aws-github-runner/control-plane dist - - - name: Start MockServer - id: mockserver - uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 - with: - version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' - port: '1080' - startup-timeout: '60' - - - name: Run webhook and pool lifecycle smoke test - env: - MINISTACK_GITHUB_MOCK_HOST: host.docker.internal - MINISTACK_GITHUB_MOCK_PORT: "1080" - MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} - run: sh tests/ministack/run-smoke.sh - - scale_set_integration_smoke: - name: Run scale-set ECS smoke test against MiniStack and MockServer - runs-on: ubuntu-latest - timeout-minutes: 30 - services: - ministack: - image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0 - ports: - - 4566:4566 - # MiniStack launches nested service containers for the integration smoke test. - # This digest-pinned, non-fork workflow is the narrowly scoped Docker-daemon exception. - options: >- - --add-host=host.docker.internal:host-gateway - --volume /var/run/docker.sock:/var/run/docker.sock - env: - MINISTACK_ACCOUNT_ID: "000000000000" - MINISTACK_REGION: eu-west-1 - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 - with: - egress-policy: audit - - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Setup Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version-file: lambdas/.nvmrc - package-manager-cache: false - - - name: Setup Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: latest - terraform_wrapper: false - - - name: Install Lambda dependencies - working-directory: lambdas - run: yarn install --frozen-lockfile - - - name: Build smoke-test Lambda distributions - working-directory: lambdas - run: | - yarn workspace @aws-github-runner/webhook dist - yarn workspace @aws-github-runner/control-plane dist - - - name: Start MockServer - id: mockserver - uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 - with: - version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' - port: '1080' - startup-timeout: '60' - - - name: Connect MockServer to MiniStack network - shell: bash - run: | - set -euo pipefail - ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')" - network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')" - docker network connect --alias mockserver "$network" mockserver - - - name: Mark repository as safe - shell: sh - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Run scale-set ECS/MockServer smoke test - env: - MINISTACK_GITHUB_MOCK_HOST: mockserver - MINISTACK_GITHUB_MOCK_PORT: "1080" - MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} - run: sh tests/ministack/run-scale-set-integration.sh + compose_file=tests/multi-runner-orchestration/docker-compose.yml + docker compose --parallel 1 -f "$compose_file" pull + docker compose --parallel 1 -f "$compose_file" build + docker compose --parallel 1 -f "$compose_file" run --no-TTY --rm smoke-tests + + - name: Print MiniStack logs on failure + if: failure() + run: docker compose -f tests/multi-runner-orchestration/docker-compose.yml logs --no-color --timestamps --tail=all ministack || true diff --git a/.gitignore b/.gitignore index 276fe10733..a215e2b9d5 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ secrets.auto.tfvars node_modules/ site/ + +lambda_output \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md index d207f23c70..74eb855661 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -373,10 +373,6 @@ The scale-set lane itself defaults to `runner.group_name = "Default"`, name, runner owner, and GitHub App SSM references in the lane; credential values are not placed in the controller manifest. -The [multi-runner scale-set example](multi-runner-scale-set.md) shows how these -provider-specific settings coexist with webhook lanes in the same v2 -`multi_runner_config` map. - ### macOS Runners This feature is in early stage and should be considered experimental. The module supports macOS-based GitHub Actions self-hosted runners on AWS EC2 Mac instances (`mac1.metal`, `mac2.metal`, `mac2-m2.metal`). macOS runners require dedicated hosts due to Apple's licensing requirements and have longer boot times (6–20 minutes). Set `runner_os = "osx"` and `use_dedicated_host = true` to enable. See the full [macOS Runners documentation](mac-runners.md) for details. diff --git a/docs/examples/index.md b/docs/examples/index.md index 50aff55389..aee1d868b0 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -5,8 +5,6 @@ Examples are located in the [examples](https://github.com/github-aws-runners/ter - _[Default](default.md)_: The default example of the module - _[Ephemeral](ephemeral.md)_: Example usages of ephemeral runners based on the default example. - _[Multi Runner](multi-runner.md)_ : Example usage of creating a multi runner which creates multiple runners/ configurations with a single deployment. The examples including: "arm64", "windows", and "ubuntu" runners. -- _[Multi Runner v2](multi-runner-v2.md)_ : Example usage of the experimental v2 multi-runner configuration interface with shared defaults and per-lane overrides. -- _[Multi Runner scale-set](multi-runner-scale-set.md)_ : Example usage of a v2 deployment combining webhook lanes with an experimental GitHub Actions scale-set lane. - _[Permissions boundary](permissions-boundary.md)_: Example usages of permissions boundaries. - _[Prebuilt Images](prebuilt.md)_: Example usages of deploying runners with a custom prebuilt image. - _[Termination watcher](termination-watcher.md)_: Example usages of termination watcher. diff --git a/examples/multi-runner-orchestration/.terraform.lock.hcl b/examples/multi-runner-orchestration/.terraform.lock.hcl new file mode 100644 index 0000000000..9112f6be79 --- /dev/null +++ b/examples/multi-runner-orchestration/.terraform.lock.hcl @@ -0,0 +1,74 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.71.0" + constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0" + hashes = [ + "h1:HNddcXLjW2P3WWN1JFIqeg8o3ouCwit4fD14Emx6+rc=", + "h1:aLOQts7nCLMR0kb/jCdP56wlQQoGqhKLctKgtQjDopc=", + "h1:sZCupscga0Grk22+04FNY1RYDeOcVbCTd6ff8+hsF94=", + "zh:09aa14408c5ab26915fb2a469b5eb67c06393a2054d78592ba4c32f7a882e918", + "zh:1d1138128dc4866f6c16b1ef3b871b3fd9a3dc648e9056e06e210d1817aa052b", + "zh:2c17a79821a0e06fbf1834a12e92227db29c2001fef7ef0b347af2e6dad83f14", + "zh:55616730a485c9c7283546a84f1ac5f8341152c5572338a54fc7af5f69ebfb92", + "zh:7e6461ec2976b5961a1d4f3c08c68fae0645d7f5e0a404f7647baa3b88d337d8", + "zh:7ec935166b2251d075fea980012f5267078785399585e8b443dca484fc57df0f", + "zh:885b3290cd48738cbf9ee8a6d47a4452889843f8f53b8cf67fc8c9b16e553721", + "zh:8930fb33c6c965f7cc82230cb7fbb0cfff9bb30becdfce5e0c2242512d849894", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a702f9a4dc7340182a8daf5b6ee562f14657fe14752f7d54dc799cf2fd91a868", + "zh:b624cb320d787fbfa6dc94be5a1fd52683d1ebd11b42646262b221a694e6ffb8", + "zh:bcbaded68b553afeea93604ec22c254b98f3951862bdeb68ae89c179c7dda117", + "zh:cb57bbcaf0fa3a70ea29b60d183d3a0dd020f041b18da5d1725d2acd17fa7d59", + "zh:d4ca164b1b6039557d8f5f5f911c4aaad1a187e9b3758f6885c95f8954f7de58", + "zh:eb0c705337cef7b96086e8b84702ba965eeeace7a4aa3235b2c8aad5f50f2fca", + "zh:f2787f40ce873d8c1308c92adec9b6adbd223506fe9d79bba447729e422f9146", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.3.2" + constraints = "~> 3.0, ~> 3.2" + hashes = [ + "h1:/Wbz7DHFO92KAqyDBMxGA8nwOJYECM8iLGhR7+wVWhQ=", + "h1:IQ1qrkht1sC1nibUR+AJ3ulryyhVDHfCHZhoJi0sg2Y=", + "h1:vpRtoWIdmRPuUP+2DgvgeWNkhA9HB7h6w4xVnaS2J/U=", + "zh:10ec43b8b7b18d5639238c7fb9e111f6a4b038523dd66c7a426bf27b25fa4c08", + "zh:60beb9cc2ad5b871c710860cee75b42850cc6acd43db0d77cb5e00fda7288b55", + "zh:62538582d0a4a2f10ad8a8d9a6c3cd3f05af6c6d91c6641ffc78d4f0e8e69b27", + "zh:64a8f9ce7852d9efc5b464c12306c946366d59f5e2757def97969c9fd64bd1d6", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:92a374fb736a52f465283326d0a5bf4f495132eb99be209dfb4c75ec803fe8db", + "zh:98da9c42785d27a50f0604758bcb61a30f6278b9f2acd92bb3b2046e0e71916c", + "zh:b0f7896fae554729cdf4a24ac06359a050cff5817e6cd8597cba8a4ae01a7409", + "zh:bc8179ee35d67c72fb03012e7023b9f9816f033a7ec4109c001dd6d29752e812", + "zh:d23a598f713bfb6098bc003571d7de90b5a33b78f9be240488252fe5f3c2a60d", + "zh:d2855b922ea345dbd89ea287e4c6c4757e38bc0aaffeb2b79aa0b8004f9c53ff", + "zh:d3a60422bc6a2f9244d076c5222c07060c826ef91bdbaf4634cb752b86057473", + "zh:faa01928c25d2a6ecd9c7eb8b88134cb08de55a6b11ca6c703ac0092845344ba", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.9.1" + constraints = "~> 3.0" + hashes = [ + "h1:PlW+UZ4EElQF3NQwf41KQwavFujab3Czc51zu9dyVM8=", + "h1:g40qr7yDmIpaur4SsK5BcOda3HSo1RJ6zHVMqN4EJ+0=", + "h1:nozfr4CZq73d4HjubKJundX/8A+Mj282oS/hyNfjUPU=", + "zh:05f4734c1f0be840b711b3eff259ebc5fca436784c728955b1678078466f48d7", + "zh:0b91bf19371d012434eba1deeb6aab77158def9b39601dcbd94450b3974a2a26", + "zh:0ee6eacd47ec00183d55d726a4b6c4ce951a199f944bf22f1aa58392ebdfa7a2", + "zh:19388a4074b76a89a43a6c8328d7ae8ee2e7de3d346af51e80d3e6d3d12925f1", + "zh:23e74d48c5e2ac2e823fd527f49fee9db37d32a1990c9e3bf126ead697b843eb", + "zh:3cabf7fbd096c520064aae3aba61aba670af83ab91291a71fa1b1332929c2b7f", + "zh:5c0a3b8af0be60be4eca12ddee385cfa8babc1ec8e98cdf9de2f2274c73eabfa", + "zh:60b4f8a8ef18f52bf8e19215229dae408bee732825964092db7c989fd2de4097", + "zh:7359015acfedcbd6366f2329c854cf8d3c8ca5cd0faa89d2d37db358d6eba6c5", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b38758402f0e13a1071162da28994023cd2ac676e54af350c9ffd8dfa73fa7b", + "zh:7c7fbb8895eb75bb4de1f933e98553bd99c8d048c89a925ddba490aa5a67f7dc", + "zh:8c2b8c6a7ccdec16b73e2fb9f3700ea097f58c592571e4c5de60c93d2301732c", + ] +} diff --git a/examples/multi-runner-v2/.terraform.lock.hcl.tofu b/examples/multi-runner-orchestration/.terraform.lock.hcl.tofu similarity index 100% rename from examples/multi-runner-v2/.terraform.lock.hcl.tofu rename to examples/multi-runner-orchestration/.terraform.lock.hcl.tofu diff --git a/examples/multi-runner-orchestration/README.md b/examples/multi-runner-orchestration/README.md new file mode 100644 index 0000000000..ce53db3f48 --- /dev/null +++ b/examples/multi-runner-orchestration/README.md @@ -0,0 +1,47 @@ +# Multi-runner orchestration example + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.6 | +| [aws](#requirement\_aws) | >= 6.33 | +| [null](#requirement\_null) | ~> 3.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [base](#module\_base) | ../base | n/a | +| [runners](#module\_runners) | ../../modules/multi-runner | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_region](#input\_aws\_region) | AWS Region where the runner control plane and compute provider resources are deployed. | `string` | `"eu-west-1"` | no | +| [compute\_provider](#input\_compute\_provider) | Provider-specific settings for compute providers. | 
object({
    aws = object({
      ec2 = object({
        instance_types = list(string)
        ami = object({
          filter = optional(map(list(string)), { state = ["available"] })
          owners = optional(list(string), ["amazon"])
          id_ssm_parameter = optional(object({
            arn = string
          }), null)
          kms_key = optional(object({
            arn = string
          }), null)
        })
      })
    })
  })
| n/a | yes | +| [environment](#input\_environment) | Name prefix for the example resources. | `string` | n/a | yes | +| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | 
object({
    url                = optional(string, null)
    ssl_verify         = optional(bool, true)
    runner_owner       = optional(string, null)
    registration_level = optional(string, "organization")
  })
| `{}` | no | +| [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. | 
object({
    id              = string
    key_base64      = string
    installation_id = optional(string, null)
    webhook_secret  = string
  })
| n/a | yes | +| [runners\_lambda\_zip](#input\_runners\_lambda\_zip) | Local ZIP file for the runner-control Lambda. | `string` | n/a | yes | +| [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | 
object({
    name              = string
    runner_group_name = optional(string, "Default")
    min_runners       = optional(number, 0)
    container = optional(object({
      image = optional(string, null)
    }), {})
    ecs = optional(object({
      task = optional(object({
        architecture = optional(string, "x86_64")
      }), {})
    }), {})
  })
| n/a | yes | +| [webhook\_lambda\_zip](#input\_webhook\_lambda\_zip) | Local ZIP file for the webhook Lambda. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [webhook\_endpoint](#output\_webhook\_endpoint) | n/a | +| [webhook\_secret](#output\_webhook\_secret) | n/a | + diff --git a/examples/multi-runner-orchestration/main.tf b/examples/multi-runner-orchestration/main.tf new file mode 100644 index 0000000000..4dbe26d9ac --- /dev/null +++ b/examples/multi-runner-orchestration/main.tf @@ -0,0 +1,203 @@ +module "base" { + source = "../base" + + prefix = var.environment + aws_region = var.aws_region +} + +module "runners" { + source = "../../modules/multi-runner" + + prefix = var.environment + aws_region = var.aws_region + + experimental_features = ["multi-runner-v2"] + + global_config = { + tags = { + Example = var.environment + Project = "MiniStack" + } + runner = { + os = "linux" + architecture = "x64" + } + } + + global_config_github = { + app = { + key_base64 = var.github_app.key_base64 + id = var.github_app.id + installation_id = var.github_app.installation_id + webhook_secret = var.github_app.webhook_secret + } + enterprise_server = { + url = var.github.url + ssl_verify = var.github.ssl_verify + } + runner_owner = var.github.runner_owner + runner_registration_level = var.github.registration_level + + } + + global_config_lambda = { + architecture = "x86_64" + } + + global_config_observability = { + logs = { + level = "debug" + } + } + + global_config_orchestration_provider = { + webhook = { + runner = { + ephemeral = true + jit_config_enabled = true + # The smoke test keeps the ephemeral resources alive until each provider's + # scale-down phase, so it needs capacity for standard, dynamic, and pool runners. + maximum_count = 3 + boot_time_in_minutes = 0 + } + lambda = { + artifact = { + zip = var.runners_lambda_zip + } + scale = { + up = { + job_queued_check_enabled = true + } + down = { + # Smoke scenarios create and remove runners immediately; do not + # wait for the Linux five-minute minimum runtime before checking + # the GitHub runner state. + minimum_running_time_in_minutes = 0 + } + } + pool = { + config = [{ + schedule_expression = "cron(0 0 1 1 ? 2099)" + schedule_expression_timezone = "UTC" + size = 1 + }] + runner_owner = "test-owner" + } + webhook = { + artifact = { + zip = var.webhook_lambda_zip + } + } + } + } + scale_set = { + grouping = { + strategy = "runner_config" + } + container = var.scale_set.container + network = { + vpc_id = module.base.vpc.vpc_id + subnet_ids = module.base.vpc.private_subnets + } + ecs = { + task = { + cpu_architecture = var.scale_set.ecs.task.architecture + } + } + } + } + + global_config_storage_provider = { + aws = { + ssm = { + paths = { + root = "/github-action-runners/${var.environment}" + } + housekeeper = { + lambda = { + artifact = { + zip = var.runners_lambda_zip + } + } + } + } + } + } + + global_config_compute_provider = { + aws = { + ec2 = { + vpc_id = module.base.vpc.vpc_id + subnet_ids = module.base.vpc.private_subnets + ssm_enabled = true + runner_binaries = { + enabled = false + } + } + } + } + + multi_runner_config = { + ec2 = { + runner = { + os = "linux" + architecture = "x64" + name_prefix = "ec2-" + extra_labels = ["ec2"] + group_name = var.scale_set.runner_group_name + } + orchestration_provider = { + webhook = { + github = { + organization_runners = true + } + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "x64", "ec2"]] + bidirectionalLabelMatch = true + dynamic_labels_enabled = true + awsDynamicLabelsPolicy = { + restricted_keys = { + "instance-type" = { allowed = ["m5.*"] } + } + } + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = var.compute_provider.aws.ec2.instance_types + ami = var.compute_provider.aws.ec2.ami + } + } + } + } + ec2_scalet_set = { + runner = { + os = "linux" + architecture = "x64" + name_prefix = "ec2_scalet_set-" + extra_labels = ["self-hosted", "linux", "x64", "ec2", "scale-set"] + group_name = var.scale_set.runner_group_name + } + orchestration_provider = { + scale_set = { + name = var.scale_set.name + runner = { + min_runners = var.scale_set.min_runners + max_runners = 10 + boot_time_in_minutes = 10 + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = var.compute_provider.aws.ec2.instance_types + ami = var.compute_provider.aws.ec2.ami + } + } + } + } + } +} diff --git a/examples/multi-runner-scale-set/outputs.tf b/examples/multi-runner-orchestration/outputs.tf similarity index 72% rename from examples/multi-runner-scale-set/outputs.tf rename to examples/multi-runner-orchestration/outputs.tf index 1feaf2e671..2e9144910e 100644 --- a/examples/multi-runner-scale-set/outputs.tf +++ b/examples/multi-runner-orchestration/outputs.tf @@ -1,8 +1,7 @@ output "webhook_endpoint" { value = module.runners.webhook.endpoint } - output "webhook_secret" { sensitive = true - value = random_id.random.hex -} + value = var.github_app.webhook_secret +} \ No newline at end of file diff --git a/examples/multi-runner-orchestration/providers.tf b/examples/multi-runner-orchestration/providers.tf new file mode 100644 index 0000000000..f24f950b27 --- /dev/null +++ b/examples/multi-runner-orchestration/providers.tf @@ -0,0 +1,9 @@ +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + Example = var.environment + } + } +} diff --git a/examples/multi-runner-orchestration/variables.tf b/examples/multi-runner-orchestration/variables.tf new file mode 100644 index 0000000000..f1ed23bbbf --- /dev/null +++ b/examples/multi-runner-orchestration/variables.tf @@ -0,0 +1,85 @@ +variable "aws_region" { + description = "AWS Region where the runner control plane and compute provider resources are deployed." + type = string + default = "eu-west-1" +} + +variable "environment" { + description = "Name prefix for the example resources." + type = string +} + +variable "github_app" { + description = "GitHub App ID, base64-encoded private key, and installation ID." + + type = object({ + id = string + key_base64 = string + installation_id = optional(string, null) + webhook_secret = string + }) + sensitive = true +} + +variable "github" { + description = "Optional GitHub endpoint and scale-set ownership settings." + + type = object({ + url = optional(string, null) + ssl_verify = optional(bool, true) + runner_owner = optional(string, null) + registration_level = optional(string, "organization") + }) + + default = {} +} + +variable "runners_lambda_zip" { + description = "Local ZIP file for the runner-control Lambda." + type = string +} + +variable "webhook_lambda_zip" { + description = "Local ZIP file for the webhook Lambda." + type = string +} + +variable "compute_provider" { + description = "Provider-specific settings for compute providers." + + type = object({ + aws = object({ + ec2 = object({ + instance_types = list(string) + ami = object({ + filter = optional(map(list(string)), { state = ["available"] }) + owners = optional(list(string), ["amazon"]) + id_ssm_parameter = optional(object({ + arn = string + }), null) + kms_key = optional(object({ + arn = string + }), null) + }) + }) + }) + }) +} + +variable "scale_set" { + description = "GitHub Actions scale-set configuration." + + type = object({ + name = string + runner_group_name = optional(string, "Default") + min_runners = optional(number, 0) + container = optional(object({ + image = optional(string, null) + }), {}) + ecs = optional(object({ + task = optional(object({ + architecture = optional(string, "x86_64") + }), {}) + }), {}) + }) +} \ No newline at end of file diff --git a/examples/multi-runner-scale-set/versions.tf b/examples/multi-runner-orchestration/versions.tf similarity index 76% rename from examples/multi-runner-scale-set/versions.tf rename to examples/multi-runner-orchestration/versions.tf index 6af69ab915..6883c62423 100644 --- a/examples/multi-runner-scale-set/versions.tf +++ b/examples/multi-runner-orchestration/versions.tf @@ -4,14 +4,15 @@ terraform { source = "hashicorp/aws" version = ">= 6.33" } - local = { - source = "hashicorp/local" - version = "~> 2.0" + null = { + source = "hashicorp/null" + version = "~> 3.0" } random = { source = "hashicorp/random" version = "~> 3.0" } } + required_version = ">= 1.5.6" } diff --git a/examples/multi-runner-scale-set/.terraform.lock.hcl b/examples/multi-runner-scale-set/.terraform.lock.hcl deleted file mode 100644 index c96d2b19bf..0000000000 --- a/examples/multi-runner-scale-set/.terraform.lock.hcl +++ /dev/null @@ -1,93 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "6.63.0" - constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0" - hashes = [ - "h1:9cre7jh1lSs/9igpgAcENMUAUlYW3HCtkav3up4oit0=", - "h1:dRlYHkc+r6fgzF57WC7Zjcmb6sF/6TTGDEgwGK+LAZY=", - "zh:005d56736afd17d963998c405cee6f434dbc23a415109f9435ff1542879ae611", - "zh:026ef126321a86ad7080b5d858e2527f96f5289678cbcd8856296e229c43339d", - "zh:06e0b58b2d1eddb5137fc86bee7ad2d07953c0bc3f57cccfc5ae0d2456068a3a", - "zh:07221735d61ababed84734e5ffcfc5bd59d01f29f029166ba5f2175895dceed1", - "zh:1a72db00583112bdb8c19b213a78a3f5de754fffc08f07e061f4e326289fab7d", - "zh:32968e74a53b03e97a084dc7050c22ef661fb5b3ea8a44f5a63e47bc45ad0e7c", - "zh:4b357dfe4b820e3e4acd2881cff8288b2186491e63416751f0d12692ba478ceb", - "zh:81e30884d7de686265e7d87bb92527e802878c65a378470ede2a1e9f4e40ccc9", - "zh:82e137297f6a5a08b9ce2138f7aabea245ad99495d9d9eff502f752d6ca90dbd", - "zh:8eb83b67099f0ea9df238a979dff933ff50ce06a2e3ff05a48556a10f10dd204", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:d0ba30886cbe41850fee689f51ef9088578f323cfd21817bb409951d43c465eb", - "zh:dd48e7089784454bc03d713e9057f5ca0ea1613bd402125054a51894957b7925", - "zh:f250fa81e54cf60fcb0e9c0fc4ac043f1ecc2ac24967f628b3609364fcab3d04", - "zh:f38fc09fc25a8d2cf89a4d4cd6a5ef7cb1aad72798dbdcad58b8876b6a551a54", - "zh:f7c7380fdf126e1901f2084588dbfd724c76cb131ccfa795a541219111103c06", - ] -} - -provider "registry.terraform.io/hashicorp/local" { - version = "2.9.0" - constraints = "~> 2.0" - hashes = [ - "h1:9rBZCMNpxKwMlRbWH2QpwD3kqUCAejdOZQ/aiiDObXQ=", - "h1:m24fjcInWvTVZ1XSo2MaNuKPe+X/gfG8SIi09rA7a7M=", - "zh:0baa4566cf77f1ff52f4293d1c8536202dd23edc197c3196413a28343c3ac3a0", - "zh:16b5559c3c07088ddad11a9bb9e9c0799999363c2958e9a5be2bcbbf2cd9ca64", - "zh:197c79015a10d1cce904a8ea722cbc750c42aeae2da53f44a6a0751d9fd1aa90", - "zh:29d0b03e5343a80677ebfeb2e2c31cbe4b1f65e736e53417454a4277fec2544c", - "zh:4896bfa6cf1d2fd562b47ef2e87f47862ae92a04f8ad5d764380f0c6653473b8", - "zh:531f8529cbca49f681883e57761a05a8398afaef6d1ab0d205d26bf12f4428e8", - "zh:6aaf5011d83161c86d2bfb80c0923ec934e578288758da2f37acb7aec129004b", - "zh:7430275253d3d3c40aa6179e0ec0d63212874dbbc06c5a51b9d07ec590f9756c", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:be17dc611e95e26cdf6cad79dfccf1064f0e32032a2efeb939a9bbe7fb1cbfe9", - "zh:f0e3b0aa644202e1d79d2000dca91f6019425da71e9800fa23f27e51c034f195", - "zh:f62bae4519e4ead49182ddc8afe8cf61e2a4c3ba3973b0fbba967736a2696aa3", - "zh:fcafa360a5b0b96244f26f4e3a6d642b716a376557142c2442ff2fb12d11da18", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.3.1" - constraints = "~> 3.0, ~> 3.2" - hashes = [ - "h1:TuxJq10DVnRP7c5HBZPyyvQGcckNVfijyU1eXEu5e4M=", - "h1:m5FqidbIgh+E9OigiZh8/xbkvpUQFSj3hZo/jqNLCLQ=", - "zh:08c59776542ea16e5a8545752787b17ff412922182b4cfabe16139197be8ac44", - "zh:123109cc7e5ed6d515787fbc212f2a3fd5e75647bb24ab7c801ccd4d4ed42451", - "zh:14b3fa4372754b54844b41d5dbd4671a292d8d6828b90169061feb4d7b15dd05", - "zh:56a4daaa3212f57b764bf3d1f333141c6610c5f21abb240e0111221f7c7fa4d4", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7e888a026dbacd2474a42264227ae35f639780f0f0c613529d10a95cd61988b3", - "zh:85a53646267e87d600df7124e4767ffde9bba3b6356d45d961618bdd68131cc7", - "zh:8ffa0e9c7c39b2ab0905b472465d6e35ef0b776b3f6273bb34c150340b61bff1", - "zh:9846510a1841530d4403f4818e233f91e3b3bade7441047599fbf800742f65be", - "zh:afa98d44860875f037c6def0a7e6ff208e042712ba771f620482b143cd336891", - "zh:bdca130d9ef27488ae0b13bc8fd8019e8bbdd4f2ceff29da066bd333165d68c5", - "zh:cb3b94cbca88210dd0d1f11e2b8a89333f48c3857faf8f70f589072ce7c28610", - "zh:f0c0ba87925fe32f84b80f7513b1efb1b0866f51f899ba825e95ad59ff09b018", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.9.0" - constraints = "~> 3.0" - hashes = [ - "h1:OO+IuvQJSPmWdN8AyyIEvPJbLvDQpgX/zbktoa9KsJE=", - "h1:UlBuNVuCGJ39tTv2c5gz2NRZnQbXfbIWbTzWcth5o74=", - "zh:161ad0bd9a75768c82f53fb6e7172a9d8be2d4889b012645a34795031aaf1bf1", - "zh:19dc9a5b17729725ccfc4f45b0500af0ee5bc6b6b160c7adb8f2bf617d2c80ea", - "zh:269eda8fe42daa7974d5a34d166c3ba9defe80cde86c01e4dadcfdf2e1f05e5f", - "zh:373f7c65566f8f2cc7f45d698654feb9d988996957e1266a69ca00c52d6d16d0", - "zh:5599d16804c41c83009ec621b6d6b6f74e102f5827678a4750f8809055546b61", - "zh:583be0440469a22bff70dcfa56593b01566860b29607437264adb51060cf46fc", - "zh:5f211d8ec3f2e1f414870d9584bfe26e6995560ef81c748f8447a48164767398", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7b547fd16216761ef86efc3ed516ac5ac0c5c42b7c7eb24a08cef2d93f69ed5e", - "zh:7e7c0679daf2a382151d05068c8c3f0dae6b7b7dccf818827b73dd08638df2ef", - "zh:8089dec888a8038b9b4fb23b3df7e1057293dbc5b60b42cc47ff690d69d4b61b", - "zh:c51f15a031edfd6f23ce8ced3446ca7f8d8d647e2499890d7d5d10d5016d7257", - "zh:c94784f005708890dc6895afd53636ec00ec1e430b15d41e5aebfb1d4b39bd04", - ] -} diff --git a/examples/multi-runner-scale-set/.terraform.lock.hcl.tofu b/examples/multi-runner-scale-set/.terraform.lock.hcl.tofu deleted file mode 100644 index 577519c08d..0000000000 --- a/examples/multi-runner-scale-set/.terraform.lock.hcl.tofu +++ /dev/null @@ -1,150 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/hashicorp/aws" { - version = "6.63.0" - constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0" - hashes = [ - "h1:1jhQJPHOPu2mzDG/ke3tK8PNcEqQHA4vhF05WWlM/yg=", - "h1:3+pvT0KN/bkJ6TBuExj+gxptEozhnpo80Ztblwq85eo=", - "h1:5aTequ87wZS7Mh4dEIayDGKcFdaFgHtw74NtqY5Idi0=", - "h1:AMRlrrM3z1SmrslOtotqKq02zapxLKtXaSN9Jbs0Oho=", - "h1:OTjECFWTDxsjcUfOKCNBp75Z5lGrW/KplRDsjTZYT2g=", - "h1:b8LORLOKMOOl+nK1M2UhCjELSjjziClJuAv6hYuySHs=", - "h1:bUfTX1giRLOyfDbBvsDbwR3tJmsTFRWcOTQdj2npDWA=", - "h1:dzs4kwx+itVGAH7yEOyeoWcE3LNRMnWtlt4ROgyAa0M=", - "h1:lnjou+SiwpYJ+j9PXWozXPHSPlhxIZb0RqpsSEBzfGw=", - "h1:pqzUeHAQj9NctgkwaynaF2aB+3QiZXcoslzMGjT743w=", - "h1:qTXEWOWxA6sfUpC29UXrsbHnNzWH7+j1RTUVG4YCm+U=", - "h1:qdHKOKt/ISn9RLjUe22OZBpN3F7H2DFeHJL/CSc2x8E=", - "h1:tpNzIZBzzUW7/kLU3BhYf3jhdO5uNwYfNmgC9B8kvMM=", - "h1:uVVlFgjg6GyxJLbCsTO1+R5fTNbZ73mLpVpSd0mMrFk=", - "h1:xGJsV5IFf7c11cXzJrsY40hiJCghp4odT0eJyTyAUYY=", - "zh:039a03e920e55f14a691feb67216a2d142bfee603128e15f9c5138f9ecd85016", - "zh:14e060b7f46ca7b0fa009b91aef419c58cbdff854de96e9a1d853166f8d902fd", - "zh:18803e8fe2c291c8db5526c71b3287ff7c81453f10ca6d8e69cdf9c535b00783", - "zh:1b83fce6e31a6095e932d80a7c3f47ac04252653a2de2b98ec6204563310fcba", - "zh:2add7bc976ceebb1a94d84598762c9b9cf281ca52ec83deeb4e95e90aa200a12", - "zh:2f22cd5372408f11937fa5513a7b960d3cebc334c5ec65fc5322c3bac1c1f664", - "zh:41c5e857dacfd83b7ca12a435204957ff6ca8830b9efefd0d381ad4d63b19779", - "zh:4eace6246e46999782d219bc4f50f83d19ef9156bacf5ca1528da12da4918015", - "zh:5e1c1281c3f929399e2ed3dbdce03426fd57a9ec55cd36e04acf1712aa5954ba", - "zh:608272b1f5d75ead123c9d933aa1fed7dc832cedd1506019046b4c8fdcc91dce", - "zh:6b3680f8a2f7be2c171953aba89d639fb2624b9cf52ec304e16434874566601d", - "zh:99aa1006f2141f3341a02020e1c91abfb02280e57c77e0415c98b8d900353d88", - "zh:9ad235bef34a89a8dd9943f9fa9f05cc729bb52a4e0dc926a31bb13cb0ae2418", - "zh:e0e3ac361e04748a4ca0c1cdbb6abab2aa817f4ad67e1692817d16e370161d59", - "zh:f60962c982a41fde956e796425e7194b4311741c179c060c1c8b5e16a557d635", - ] -} - -provider "registry.opentofu.org/hashicorp/local" { - version = "2.9.0" - constraints = "~> 2.0" - hashes = [ - "h1:1dtKYW/5a1qob3yneL6WzOlnSGfYtJ6a2XeejCk9yb4=", - "h1:5NseXq5wU8O20ersTtV4ocrLYFFtgFr7n0pRLO1W2Rw=", - "h1:5d22ZPPK4iiygPbwRz/PJF5Es/0axVpMlPRpCR0Padw=", - "h1:AnwyolirmIlBMjH6+tV8bKkvT+5axJNYxi2y2IguiX4=", - "h1:PBp+HeseY021Fw3sLznCG27idgwPoff4cBuNmKgPL2w=", - "h1:VDxIhe4GbzdOCdmt7mQaqdwERQW6GSI7Roonts42Gr0=", - "h1:ZO6eWWnf8LjjV1q/JNeL9WLtZ6fwIttOnyN5LjCNSEo=", - "h1:dPIAf8oUAz+vW2E0iZunMvpuPddRZIztRsPSY1u+VnY=", - "h1:fwTDVG9AhFVKQZIb1EXkHv4FqzsZNlLWgkyPGDmZZEE=", - "h1:kDc465XPC7/6XFCjrMC4mTqhA9ef0FHKuJ3ZgfGNfeg=", - "h1:kGbjxrI2P8MHeyVtE1U3Q1TbyF71ExnHxtkrE+Aj6UU=", - "h1:kcoK6Afbsj54u9zaEqpecWAFKytqjBijtguCNwV3d4M=", - "h1:rxomJjDwOo+YZ+WIPc25FqEgsz9orh/2MCyUcZmFjvw=", - "h1:t0CMn/Rkwquw8l2yQ+O4ApzbMZfY2UazbsDnZygzACA=", - "h1:tJwgm2BS4xCGlElCDQEFXQoefY9Y4t0JdSKTtsPBbBo=", - "zh:13ef7ecd1e397ec5b20ea588508dd3e3b8d6c50d809ae76b079abf9dd8d02e4b", - "zh:2190c9325980076489ce02b0f5dd2c0b91fc8711cefa99e714d8619a32827ad1", - "zh:2a0cfc5600730093705071707e4a4e4e953e7d9091859e0f66b46daa1060dd5d", - "zh:2ff53eac1af43ab9a2248a0e53c963d46e19cf04bc4c3f323591cfcebb218252", - "zh:4ebc3dee700f60af9da29970052fd02fa947813162b224716862dc9d7f1f7542", - "zh:5fe6dab84ceeaa8eb3f1567c5f05578333370c472240ca5c5bfc25e92d4d5586", - "zh:66bbec16367bbf440045502c9779b11f4ac5b022c8d8d17afe12d431950838b5", - "zh:7641e5c2e4b529e869cde29ab5b1de2fd1091489eb745b19ac2709bd7f4dfd84", - "zh:855bfba0756d17ce07595ff57d7cf664443d1495127cb88fb063362734b8b22a", - "zh:aaec10f237921d60c581d1b7a66f0a8a8019d9802dc04af11b5b981f6682e01d", - "zh:e460835a38ffa1e74f6929904bfd14ef473d217fd537b7ce834abe5ce5e2ce07", - "zh:ecc4295215db0e4aea3c9329611c31e09a853e1ae207d56742403bd4f5516703", - "zh:ee6d9fae63a612072e00402894e14826af7a3351c235b9c5b423b7629a77ca29", - "zh:f2b5c8db74aa7ebcf7cd423672358437d42401675069ef67b01ff910054e49d5", - "zh:f5aff74d3eb96d4592c7bca5cd3ea89b469e84efbf382944bd0f844a57059c09", - ] -} - -provider "registry.opentofu.org/hashicorp/null" { - version = "3.3.1" - constraints = "~> 3.0, ~> 3.2" - hashes = [ - "h1:2wld81FnmHW0WVgy081sIfokCr2+NuatS8yjeLEet7Y=", - "h1:AClQjJ6X22V4qcRgcYSxiXCMmp2pz0G8WVQC7wAx66o=", - "h1:AY3XQbuviNd2X5VhHYEbhNta1m/CG3JD2BKFKhCt1Y4=", - "h1:CUOZUd7H11lsU+4tISlnYIiP5BqnX8IDwFCVqfLJyAg=", - "h1:JIfV0nA/pLWnIFGscvTfuavQCn2NeHxJBeb6UUg/joA=", - "h1:RejAh+nyCwqDGExGln2Kb4Ro5LyHak0eJe0P9g8CHPc=", - "h1:SHOuTZjYymsmy4asuRq6NC3yW+zdVZOOt4f5nrb+EPM=", - "h1:WwPat/gT4gO8GvvKNdSkkXWVD65JppLJfqKOt9HhOqQ=", - "h1:Z3hXVLrOyaRiiLmmL5UCOdcRMguwjN1x5TYNdmBDgls=", - "h1:dd78Ad5HdfPzPts7A9qIxfitXhAriV/qza38fr2ukjk=", - "h1:dyVb++KwDdybzLTE6bf7GZiVQ31iWsgKPWmhTQ8G42k=", - "h1:gD8ZH6WWe+5gg5+y8SpLWGPUDzSxcQ3HKP8IDM/wW3I=", - "h1:juXCww0zRQKFTDZoKqYR0+Sn1lu99oeL6pr0Jh6LWx0=", - "h1:kFAySmtsshyNV7IhIrEdASzVcvwy68eeZCVC66P7yNk=", - "h1:nS5azDopRisB2NInwDx3Hrfg2FdVt8Gw0gTQzC0rd70=", - "zh:164eb061d84e01759f391265865fb31828083d0a06b25f7af7e094cbdb18c799", - "zh:1bb9b669a82b52c0cba2860c71e9ee6699ef302f28cb8ed06f572d39bc6c7c4f", - "zh:1ea9b31a8f29302122c1e8d673693f3ac270336dae560af803cd1117265a469a", - "zh:238bd463cb0154fb935dc331da40c0a9cbe5db9cee615ae5f35ccad5eed7dc41", - "zh:30ef2b7384cf7e20f33fe75754b54cf669d59816f3ad4fc73bfb2b26fb6735e9", - "zh:35b5cded16e4b57c207d03ee0979b14baf486fa520e6edb7a2eecf18f1b85471", - "zh:3dc840d13a50cd215c7540573f27e2b61f739ba90aee5b7c3846079aa0ab5534", - "zh:3f9309a18db608f975d5691fcb47a6e14d77199156a52e9c39dcafe3737f2b07", - "zh:44263a219f7dbd1848b545d080110b4f7d0495e77b71cd3c7a0b5ec52a09accb", - "zh:4dec54aa5f445eeea035bbd4839bcded5e47ecd07cba0e70c5a09e9272cb592f", - "zh:5e8fb319d7c6d6c4566a18b9d0c91580b4901a96acd7fdc476bfc79f074368e2", - "zh:b0e8b6d41834b57fcfbb5ca00da52ccb757e1a95b6a2d546c0dae8bfbeca1cdf", - "zh:bbde4c3a1dcc1718027a61a4cdf661619d17af1b58df1038fe27bcf43c3dc29b", - "zh:c4140fff9f692baf29236557f706f9515f93229413438527d764023a82301da3", - "zh:f8e9d83184e4bbeb97c6f0d569833007c48ba5a7ff334def201df4991d03a962", - ] -} - -provider "registry.opentofu.org/hashicorp/random" { - version = "3.9.0" - constraints = "~> 3.0" - hashes = [ - "h1:8EQU5KSxezcjo/phRSe69rDOI0lk4pSaggj7FsskYp8=", - "h1:Lw9im2VBBJQ3RyAbHPQ0rcvcmmcZWm3x+kIOpN+Tv9s=", - "h1:U8KXqGCoNI9/guYbTvzgdtVk3fRthoG0UXwm1JoEpIs=", - "h1:YXaVd4p6qXPPVaxIBaIDNXmBwT02ZqDn0qD+tYpw8sA=", - "h1:cOpc03fphEt/G9Rfc4jLL/fW0D7tgvlXqiDKPF4vuww=", - "h1:g09RR7T1xWkeGrZwWvWMT9ncJrFGr1k3CBD585UmO7w=", - "h1:gGDdPPibmw2EWROx+sh1RGLjR5+nPwZyrf6/N9jXfeM=", - "h1:haE7/nXCOhXKP4oXeEnER3t5CaVQWqujz4nBnpeTUv4=", - "h1:ieSVpfZS2lKuMr05ph0QsOVpCzg7uk3cgKBaXR+Ikug=", - "h1:ig2s1IS9IzehorRjvVAnKIsUUj8fkgyxct1L/kswcc4=", - "h1:j3lS+ZEERFnoab8t1ppDrScGVP/cgWbzlCrEYKTCXYw=", - "h1:lxezrKmOiQIySHAM+os8qLVq7hqufDr8h3Hpzvsk+78=", - "h1:lzRqBJAG+NETxHbEZUJ/YP3RMEjZBinTX7VmgH3lw60=", - "h1:tdSNWK5ApqUsgbdYieyeYLTu6nIZUV3hR1oFqUfAuGo=", - "h1:xedet8yH/zI2CfdxsGlK0nlFWc/Bp61yrWsEa3fHB8g=", - "zh:03f1114cc20b8913523735ab76e0f0a2b16ce13c92923a53304bf85f07fc0dbc", - "zh:105b678ee72322a3067f105d7e05e940f6143238f377f6e87ff4ec909246ac2a", - "zh:55f3bbf13ea18cbace61a706566a80f25f33fe2b1780b6f3d7b582af2a05b6d2", - "zh:63adf996db48f082f7a6351eb485e219cd88795fc71e6ec60a837263ab0d2cb1", - "zh:7e99550738a4e3cc68b8a467714b0d69371025fe95e3326d5323d026d55653e9", - "zh:8342b54af3a18a37e075eeae61be57f4de2ba71b35d95c5075d402dd2c1f289d", - "zh:83ee18e32ac9dd5fc91298554b7c4cfa4c3a1db50f4c797945637cc93c0844ae", - "zh:993ecc0adbf6bd535a59fbc9b735d8c33950e6f6eb5e621d750da9b71d65d80a", - "zh:ad722bc59d4edbf1415e827fc007c0efe6e0e9462d5568bae20b34be1058a261", - "zh:ae9448e1f87b2f9a6c5197a0e9862162ec6b137cb3a3835e11522995d8939e7c", - "zh:bc9cdd3aac784f759125c6627f6f6416e8726a1c184eb9cf3e55b9edbc94c627", - "zh:c8e35b89572ba1c40a9b20022e033a3395fb8d42e7604d50c900f193ba10382e", - "zh:e2deaa8a9975ef81d9f62baed12c41286918b0a10908e0e031f13f69a3b730a1", - "zh:ee39707557210a0ab1098aa357d2cdfe502e5a312d0dbdffb09d08facc4d3fc5", - "zh:f81afe4eb63e8aa9e0ea71be6c990f0dc69cb360e7191c0742a991f4a5081b64", - ] -} diff --git a/examples/multi-runner-scale-set/README.md b/examples/multi-runner-scale-set/README.md deleted file mode 100644 index a2ca69cff4..0000000000 --- a/examples/multi-runner-scale-set/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# Multi-runner scale-set example - -This example demonstrates the experimental multi-runner v2 interface. Shared -defaults are configured with `global_config*` variables, while -each runner lane uses `multi_runner_config` for its matcher, -runner lifecycle, and compute-provider settings. - -The example creates four lanes from one deployment: - -- Linux ARM64 Amazon Linux runners. -- Ephemeral Linux x64 Amazon Linux runners with job retry enabled. -- Linux x64 runners managed by a GitHub Actions scale set. -- Windows x64 Server Core 2022 runners. - -The v2 interface keeps provider-owned settings inside the selected provider -configuration. For example, VPC and subnet settings are under -`global_config_compute_provider.aws.ec2`, while the per-lane -instance types and AMI filter are under each lane's compute provider block. - -The scale-set lane uses `orchestration_provider.scale_set`. Its controller -network is configured under the global scale-set block and its GitHub -installation ID is provided by `var.github_app`. - -Configure the GitHub App variables before applying: - -```bash -terraform init -terraform apply \ - -var='github_app={id="123456",key_base64="...",installation_id="123456789"}' \ - -var='github={runner_owner="example",registration_level="organization"}' \ - -var='scale_set={name="linux-scale-set",container={image="ghcr.io/github-aws-runners/terraform-aws-github-runner-scale-set-service@sha256:"}}' -``` - -The `github_app` value is sensitive and should be supplied through a secure -variable source in real deployments rather than committed to configuration. -The GitHub App must be installed for the configured GitHub account. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5.6 | -| [aws](#requirement\_aws) | >= 6.33 | -| [local](#requirement\_local) | ~> 2.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [random](#provider\_random) | 3.9.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [base](#module\_base) | ../base | n/a | -| [runners](#module\_runners) | ../../modules/multi-runner | n/a | -| [webhook\_github\_app](#module\_webhook\_github\_app) | ../../modules/webhook-github-app | n/a | - -## Resources - -| Name | Type | -|------|------| -| [random_id.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. | 
map(object({
    filter = optional(map(list(string)), { state = ["available"] })
    owners = optional(list(string), ["amazon"])
    id_ssm_parameter = optional(object({
      arn = string
    }), null)
    kms_key = optional(object({
      arn = string
    }), null)
  }))
| `{}` | no | -| [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no | -| [environment](#input\_environment) | Environment name, used as prefix. | `string` | n/a | yes | -| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | 
object({
    url                = optional(string, null)
    ssl_verify         = optional(bool, true)
    runner_owner       = optional(string, null)
    registration_level = optional(string, "organization")
  })
| `{}` | no | -| [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. | 
object({
    id              = string
    key_base64      = string
    installation_id = optional(string, null)
  })
| n/a | yes | -| [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no | -| [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | 
object({
    name              = string
    runner_group_name = optional(string, "Default")
    min_runners       = optional(number, 0)
    container = optional(object({
      image = optional(string, null)
    }), {})
  })
| n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [webhook\_endpoint](#output\_webhook\_endpoint) | n/a | -| [webhook\_secret](#output\_webhook\_secret) | n/a | - diff --git a/examples/multi-runner-scale-set/main.tf b/examples/multi-runner-scale-set/main.tf deleted file mode 100644 index 30d70a3a9b..0000000000 --- a/examples/multi-runner-scale-set/main.tf +++ /dev/null @@ -1,215 +0,0 @@ -locals { - environment = var.environment - aws_region = var.aws_region -} - -resource "random_id" "random" { - byte_length = 20 -} - -module "base" { - source = "../base" - - prefix = local.environment - aws_region = local.aws_region -} - -module "runners" { - source = "../../modules/multi-runner" - - prefix = local.environment - aws_region = local.aws_region - - experimental_features = ["multi-runner-v2"] - - global_config = { - tags = { - Example = local.environment - Project = "ProjectX" - } - runner = { - os = "linux" - architecture = "x64" - extra_labels = ["v2"] - } - } - - global_config_github = { - app = { - key_base64 = var.github_app.key_base64 - id = var.github_app.id - installation_id = var.github_app.installation_id - webhook_secret = random_id.random.hex - } - enterprise_server = { - url = var.github.url - ssl_verify = var.github.ssl_verify - } - runner_owner = var.github.runner_owner - runner_registration_level = var.github.registration_level - } - - global_config_lambda = { - architecture = "arm64" - } - - global_config_orchestration_provider = { - webhook = { - eventbridge = { - enabled = true - accept_events = ["workflow_job"] - } - } - scale_set = { - grouping = { - strategy = "runner_config" - } - container = var.scale_set.container - network = { - vpc_id = module.base.vpc.vpc_id - subnet_ids = module.base.vpc.private_subnets - } - } - } - - global_config_compute_provider = { - aws = { - ec2 = { - vpc_id = module.base.vpc.vpc_id - subnet_ids = module.base.vpc.private_subnets - ssm_enabled = true - runner_binaries = { - enabled = var.runner_binaries_enabled - } - } - } - } - - multi_runner_config = { - linux-arm64 = { - runner = { - architecture = "arm64" - name_prefix = "amazon-arm64-" - extra_labels = ["amazon"] - } - orchestration_provider = { - webhook = { - runner = { - maximum_count = 1 - } - matcherConfig = { - exactMatch = true - labelMatchers = [["self-hosted", "linux", "arm64", "amazon"]] - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["t4g.large", "c6g.large"] - ami = lookup(var.ami, "linux-arm64", null) - } - } - } - } - - linux-x64 = { - runner = { - name_prefix = "amazon-x64-" - extra_labels = ["amazon"] - } - orchestration_provider = { - webhook = { - runner = { - ephemeral = true - maximum_count = 1 - } - matcherConfig = { - labelMatchers = [["self-hosted", "linux", "x64", "amazon"]] - exactMatch = false - priority = 1 - } - queue = { - delay_webhook_event = 0 - } - job_retry = { - enabled = true - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["m5a.large", "m5ad.large"] - ami = lookup(var.ami, "linux-x64", null) - } - } - } - } - - linux-scale-set = { - runner = { - name_prefix = "scale-set-" - extra_labels = ["scale-set"] - group_name = var.scale_set.runner_group_name - } - orchestration_provider = { - scale_set = { - name = var.scale_set.name - runner = { - min_runners = var.scale_set.min_runners - max_runners = 10 - boot_time_in_minutes = 10 - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["m5.large"] - ami = lookup(var.ami, "linux-scale-set", null) - } - } - } - } - - windows-x64 = { - runner = { - os = "windows" - name_prefix = "windows-x64-" - } - orchestration_provider = { - webhook = { - runner = { - boot_time_in_minutes = 20 - maximum_count = 1 - } - matcherConfig = { - exactMatch = true - labelMatchers = [["self-hosted", "windows", "x64", "servercore-2022"]] - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["m5.large", "c5.large"] - ami = lookup(var.ami, "windows-x64", null) - } - } - } - } - } -} - -module "webhook_github_app" { - source = "../../modules/webhook-github-app" - depends_on = [module.runners] - - github_app = { - key_base64 = var.github_app.key_base64 - id = var.github_app.id - webhook_secret = random_id.random.hex - } - webhook_endpoint = module.runners.webhook.endpoint -} \ No newline at end of file diff --git a/examples/multi-runner-scale-set/providers.tf b/examples/multi-runner-scale-set/providers.tf deleted file mode 100644 index eca2fe96a7..0000000000 --- a/examples/multi-runner-scale-set/providers.tf +++ /dev/null @@ -1,9 +0,0 @@ -provider "aws" { - region = local.aws_region - - default_tags { - tags = { - Example = local.environment - } - } -} diff --git a/examples/multi-runner-scale-set/variables.tf b/examples/multi-runner-scale-set/variables.tf deleted file mode 100644 index f0f9f66c4d..0000000000 --- a/examples/multi-runner-scale-set/variables.tf +++ /dev/null @@ -1,72 +0,0 @@ -variable "github_app" { - description = "GitHub App ID, base64-encoded private key, and installation ID." - - type = object({ - id = string - key_base64 = string - installation_id = optional(string, null) - }) - sensitive = true -} - -variable "github" { - description = "Optional GitHub endpoint and scale-set ownership settings." - - type = object({ - url = optional(string, null) - ssl_verify = optional(bool, true) - runner_owner = optional(string, null) - registration_level = optional(string, "organization") - }) - - default = {} -} - -variable "scale_set" { - description = "GitHub Actions scale-set configuration." - - type = object({ - name = string - runner_group_name = optional(string, "Default") - min_runners = optional(number, 0) - container = optional(object({ - image = optional(string, null) - }), {}) - }) -} - -variable "environment" { - description = "Environment name, used as prefix." - - type = string -} - -variable "aws_region" { - description = "AWS region to deploy to." - - type = string - default = "eu-west-1" -} - -variable "runner_binaries_enabled" { - description = "Whether runner binary synchronization is enabled." - - type = bool - default = true -} - -variable "ami" { - description = "Optional AMI configuration keyed by runner lane." - - type = map(object({ - filter = optional(map(list(string)), { state = ["available"] }) - owners = optional(list(string), ["amazon"]) - id_ssm_parameter = optional(object({ - arn = string - }), null) - kms_key = optional(object({ - arn = string - }), null) - })) - default = {} -} \ No newline at end of file diff --git a/examples/multi-runner-v2/.terraform.lock.hcl b/examples/multi-runner-v2/.terraform.lock.hcl deleted file mode 100644 index 62c535d49e..0000000000 --- a/examples/multi-runner-v2/.terraform.lock.hcl +++ /dev/null @@ -1,93 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "6.64.0" - constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0" - hashes = [ - "h1:2fTLxzUDmp/KVIHbIeLTB4bIzWHx8E6Dw+1ALLUi+Yw=", - "h1:wXARLY+IeQ7ufYxCLTPCwToWGMRvOpiOTfJS97iwUzI=", - "zh:07172315d67bc9781240272759cdfc7bd32b7e72384a56862c2c1da3cca99a81", - "zh:154ce7d2659de9a59ddfe96d7cab41a9ddc2cb267a7d4bcdf4e737ff2ffdec06", - "zh:17324d4335a7a7ac01cc23eded530775606680ff53b47cb74a3cb95d1121f836", - "zh:307ab92324ec5a61b124881ab8cac1d9e316f4527dfd0e1b59794c229407eb4e", - "zh:31e25f1903661332e36a95283042dd3ec50b47c186db00663fbd976a11e6a6b2", - "zh:3311d9f3bd12a24886027dbe73859dcd1e67bd0e3046227a338cf2c7ca04d18e", - "zh:37916156a3aac3b29be3acebd15d53145ea4ab5d4aaa825eaebe75481fa00500", - "zh:4158cb8c38b3ac6aa98eb15935ec6bd7c30838d85d2b00acc9812df8382ae908", - "zh:5bfb9499c66d9db5b34dc5c60f426a1ab1baa5457ce2aefebca826a9c3f92fb0", - "zh:6eb29ead5a4aca3b1f35812e7e8c75419180e1928e479b458f206861277736db", - "zh:7a82b6dd0c0cdef8045a4adfbddd36acb86b6b23fcbed8e189c2d71f7dc4a502", - "zh:9556bd792032c3f7e73ea4dd08cec88dc1327f5a4a57d79c30ba844ae2b9a3c0", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:c5234180464cb800c83a41f57462742b802c150ad7d4417626fcd9cb511c01d2", - "zh:cd776b83b1f7b36635957350afe7ce28ba4e4ea3a5e2deb00d13dbd3b35d9d40", - "zh:fb583a7b791c6f915b86573d04f05ddbf7f1a5e4120c5d8a7450a3086c1225c4", - ] -} - -provider "registry.terraform.io/hashicorp/local" { - version = "2.9.0" - constraints = "~> 2.0" - hashes = [ - "h1:9rBZCMNpxKwMlRbWH2QpwD3kqUCAejdOZQ/aiiDObXQ=", - "h1:m24fjcInWvTVZ1XSo2MaNuKPe+X/gfG8SIi09rA7a7M=", - "zh:0baa4566cf77f1ff52f4293d1c8536202dd23edc197c3196413a28343c3ac3a0", - "zh:16b5559c3c07088ddad11a9bb9e9c0799999363c2958e9a5be2bcbbf2cd9ca64", - "zh:197c79015a10d1cce904a8ea722cbc750c42aeae2da53f44a6a0751d9fd1aa90", - "zh:29d0b03e5343a80677ebfeb2e2c31cbe4b1f65e736e53417454a4277fec2544c", - "zh:4896bfa6cf1d2fd562b47ef2e87f47862ae92a04f8ad5d764380f0c6653473b8", - "zh:531f8529cbca49f681883e57761a05a8398afaef6d1ab0d205d26bf12f4428e8", - "zh:6aaf5011d83161c86d2bfb80c0923ec934e578288758da2f37acb7aec129004b", - "zh:7430275253d3d3c40aa6179e0ec0d63212874dbbc06c5a51b9d07ec590f9756c", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:be17dc611e95e26cdf6cad79dfccf1064f0e32032a2efeb939a9bbe7fb1cbfe9", - "zh:f0e3b0aa644202e1d79d2000dca91f6019425da71e9800fa23f27e51c034f195", - "zh:f62bae4519e4ead49182ddc8afe8cf61e2a4c3ba3973b0fbba967736a2696aa3", - "zh:fcafa360a5b0b96244f26f4e3a6d642b716a376557142c2442ff2fb12d11da18", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.3.1" - constraints = "~> 3.0, ~> 3.2" - hashes = [ - "h1:TuxJq10DVnRP7c5HBZPyyvQGcckNVfijyU1eXEu5e4M=", - "h1:m5FqidbIgh+E9OigiZh8/xbkvpUQFSj3hZo/jqNLCLQ=", - "zh:08c59776542ea16e5a8545752787b17ff412922182b4cfabe16139197be8ac44", - "zh:123109cc7e5ed6d515787fbc212f2a3fd5e75647bb24ab7c801ccd4d4ed42451", - "zh:14b3fa4372754b54844b41d5dbd4671a292d8d6828b90169061feb4d7b15dd05", - "zh:56a4daaa3212f57b764bf3d1f333141c6610c5f21abb240e0111221f7c7fa4d4", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7e888a026dbacd2474a42264227ae35f639780f0f0c613529d10a95cd61988b3", - "zh:85a53646267e87d600df7124e4767ffde9bba3b6356d45d961618bdd68131cc7", - "zh:8ffa0e9c7c39b2ab0905b472465d6e35ef0b776b3f6273bb34c150340b61bff1", - "zh:9846510a1841530d4403f4818e233f91e3b3bade7441047599fbf800742f65be", - "zh:afa98d44860875f037c6def0a7e6ff208e042712ba771f620482b143cd336891", - "zh:bdca130d9ef27488ae0b13bc8fd8019e8bbdd4f2ceff29da066bd333165d68c5", - "zh:cb3b94cbca88210dd0d1f11e2b8a89333f48c3857faf8f70f589072ce7c28610", - "zh:f0c0ba87925fe32f84b80f7513b1efb1b0866f51f899ba825e95ad59ff09b018", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.9.0" - constraints = "~> 3.0" - hashes = [ - "h1:OO+IuvQJSPmWdN8AyyIEvPJbLvDQpgX/zbktoa9KsJE=", - "h1:UlBuNVuCGJ39tTv2c5gz2NRZnQbXfbIWbTzWcth5o74=", - "zh:161ad0bd9a75768c82f53fb6e7172a9d8be2d4889b012645a34795031aaf1bf1", - "zh:19dc9a5b17729725ccfc4f45b0500af0ee5bc6b6b160c7adb8f2bf617d2c80ea", - "zh:269eda8fe42daa7974d5a34d166c3ba9defe80cde86c01e4dadcfdf2e1f05e5f", - "zh:373f7c65566f8f2cc7f45d698654feb9d988996957e1266a69ca00c52d6d16d0", - "zh:5599d16804c41c83009ec621b6d6b6f74e102f5827678a4750f8809055546b61", - "zh:583be0440469a22bff70dcfa56593b01566860b29607437264adb51060cf46fc", - "zh:5f211d8ec3f2e1f414870d9584bfe26e6995560ef81c748f8447a48164767398", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7b547fd16216761ef86efc3ed516ac5ac0c5c42b7c7eb24a08cef2d93f69ed5e", - "zh:7e7c0679daf2a382151d05068c8c3f0dae6b7b7dccf818827b73dd08638df2ef", - "zh:8089dec888a8038b9b4fb23b3df7e1057293dbc5b60b42cc47ff690d69d4b61b", - "zh:c51f15a031edfd6f23ce8ced3446ca7f8d8d647e2499890d7d5d10d5016d7257", - "zh:c94784f005708890dc6895afd53636ec00ec1e430b15d41e5aebfb1d4b39bd04", - ] -} diff --git a/examples/multi-runner-v2/README.md b/examples/multi-runner-v2/README.md deleted file mode 100644 index 2755c8fcf5..0000000000 --- a/examples/multi-runner-v2/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Multi-runner v2 example - -This example demonstrates the experimental multi-runner v2 interface. Shared -defaults are configured with `global_config*` variables, while -each runner lane uses `multi_runner_config` for its matcher, -runner lifecycle, and compute-provider settings. - -The example creates three lanes from one deployment: - -- Linux ARM64 Amazon Linux runners. -- Ephemeral Linux x64 Amazon Linux runners with job retry enabled. -- Windows x64 Server Core 2022 runners. - -The v2 interface keeps provider-owned settings inside the selected provider -configuration. For example, VPC and subnet settings are under -`global_config_compute_provider.aws.ec2`, while the per-lane -instance types and AMI configuration are under each lane's compute provider -block. The optional `ami` variable can provide per-lane AMI filters and owners, -which is useful for test environments with locally registered images. - -Configure the GitHub App variables before applying: - -```bash -terraform init -terraform apply \ - -var='github_app={id="123456",key_base64="..."}' -``` - -The `github_app` value is sensitive and should be supplied through a secure -variable source in real deployments rather than committed to configuration. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5.6 | -| [aws](#requirement\_aws) | >= 6.33 | -| [local](#requirement\_local) | ~> 2.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [random](#provider\_random) | 3.9.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [base](#module\_base) | ../base | n/a | -| [runners](#module\_runners) | ../../modules/multi-runner | n/a | -| [webhook\_github\_app](#module\_webhook\_github\_app) | ../../modules/webhook-github-app | n/a | - -## Resources - -| Name | Type | -|------|------| -| [random_id.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. | 
map(object({
    filter = optional(map(list(string)), { state = ["available"] })
    owners = optional(list(string), ["amazon"])
    id_ssm_parameter = optional(object({
      arn = string
    }), null)
    kms_key = optional(object({
      arn = string
    }), null)
  }))
| `{}` | no | -| [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no | -| [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no | -| [github\_app](#input\_github\_app) | GitHub App ID and base64-encoded private key. | 
object({
    id         = string
    key_base64 = string
  })
| n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [webhook\_endpoint](#output\_webhook\_endpoint) | n/a | -| [webhook\_secret](#output\_webhook\_secret) | n/a | - diff --git a/examples/multi-runner-v2/main.tf b/examples/multi-runner-v2/main.tf deleted file mode 100644 index 4f63974f11..0000000000 --- a/examples/multi-runner-v2/main.tf +++ /dev/null @@ -1,180 +0,0 @@ -locals { - environment = var.environment != null ? var.environment : "multi-runner-v2" - aws_region = var.aws_region -} - -resource "random_id" "random" { - byte_length = 20 -} - -module "base" { - source = "../base" - - prefix = local.environment - aws_region = local.aws_region -} - -module "runners" { - source = "../../modules/multi-runner" - - prefix = local.environment - aws_region = local.aws_region - - experimental_features = ["multi-runner-v2"] - - global_config = { - tags = { - Example = local.environment - Project = "ProjectX" - } - runner = { - os = "linux" - architecture = "x64" - extra_labels = ["v2"] - } - } - - global_config_github = { - app = { - key_base64 = var.github_app.key_base64 - id = var.github_app.id - webhook_secret = random_id.random.hex - } - } - - global_config_lambda = { - architecture = "arm64" - } - - global_config_orchestration_provider = { - webhook = { - eventbridge = { - enabled = true - accept_events = ["workflow_job"] - } - } - } - - global_config_compute_provider = { - aws = { - ec2 = { - vpc_id = module.base.vpc.vpc_id - subnet_ids = module.base.vpc.private_subnets - ssm_enabled = true - runner_binaries = { - enabled = true - } - } - } - } - - multi_runner_config = { - linux-arm64 = { - runner = { - architecture = "arm64" - name_prefix = "amazon-arm64-" - extra_labels = ["amazon"] - } - orchestration_provider = { - webhook = { - runner = { - maximum_count = 1 - } - matcherConfig = { - exactMatch = true - labelMatchers = [["self-hosted", "linux", "arm64", "amazon"]] - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["t4g.large", "c6g.large"] - ami = lookup(var.ami, "linux-arm64", null) - } - } - } - } - - linux-x64 = { - runner = { - name_prefix = "amazon-x64-" - extra_labels = ["amazon"] - } - orchestration_provider = { - webhook = { - runner = { - ephemeral = true - maximum_count = 1 - } - matcherConfig = { - labelMatchers = [["self-hosted", "linux", "x64", "amazon"]] - exactMatch = false - priority = 1 - } - queue = { - delay_webhook_event = 0 - } - job_retry = { - enabled = true - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["m5a.large", "m5ad.large"] - ami = lookup(var.ami, "linux-x64", null) - } - } - } - } - - windows-x64 = { - runner = { - os = "windows" - name_prefix = "windows-x64-" - } - orchestration_provider = { - webhook = { - runner = { - boot_time_in_minutes = 20 - maximum_count = 1 - } - matcherConfig = { - exactMatch = true - labelMatchers = [["self-hosted", "windows", "x64", "servercore-2022"]] - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["m5.large", "c5.large"] - ami = lookup(var.ami, "windows-x64", { - filter = { - name = ["Windows_Server-2022-English-Full-ECS_Optimized-*"] - state = ["available"] - } - owners = ["amazon"] - id_ssm_parameter = null - kms_key = null - }) - } - } - } - } - } -} - -module "webhook_github_app" { - source = "../../modules/webhook-github-app" - depends_on = [module.runners] - - github_app = { - key_base64 = var.github_app.key_base64 - id = var.github_app.id - webhook_secret = random_id.random.hex - } - webhook_endpoint = module.runners.webhook.endpoint -} diff --git a/examples/multi-runner-v2/outputs.tf b/examples/multi-runner-v2/outputs.tf deleted file mode 100644 index 1feaf2e671..0000000000 --- a/examples/multi-runner-v2/outputs.tf +++ /dev/null @@ -1,8 +0,0 @@ -output "webhook_endpoint" { - value = module.runners.webhook.endpoint -} - -output "webhook_secret" { - sensitive = true - value = random_id.random.hex -} diff --git a/examples/multi-runner-v2/providers.tf b/examples/multi-runner-v2/providers.tf deleted file mode 100644 index eca2fe96a7..0000000000 --- a/examples/multi-runner-v2/providers.tf +++ /dev/null @@ -1,9 +0,0 @@ -provider "aws" { - region = local.aws_region - - default_tags { - tags = { - Example = local.environment - } - } -} diff --git a/examples/multi-runner-v2/variables.tf b/examples/multi-runner-v2/variables.tf deleted file mode 100644 index fe104758b9..0000000000 --- a/examples/multi-runner-v2/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -variable "github_app" { - description = "GitHub App ID and base64-encoded private key." - - type = object({ - id = string - key_base64 = string - }) - sensitive = true -} - -variable "environment" { - description = "Environment name, used as prefix." - - type = string - default = null -} - -variable "aws_region" { - description = "AWS region to deploy to." - - type = string - default = "eu-west-1" -} - -variable "ami" { - description = "Optional AMI configuration keyed by runner lane." - - type = map(object({ - filter = optional(map(list(string)), { state = ["available"] }) - owners = optional(list(string), ["amazon"]) - id_ssm_parameter = optional(object({ - arn = string - }), null) - kms_key = optional(object({ - arn = string - }), null) - })) - default = {} -} diff --git a/examples/multi-runner-v2/versions.tf b/examples/multi-runner-v2/versions.tf deleted file mode 100644 index 6af69ab915..0000000000 --- a/examples/multi-runner-v2/versions.tf +++ /dev/null @@ -1,17 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 6.33" - } - local = { - source = "hashicorp/local" - version = "~> 2.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - required_version = ">= 1.5.6" -} diff --git a/mkdocs.yaml b/mkdocs.yaml index 4553b6f71f..ba38664afb 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -81,8 +81,6 @@ nav: - Overview: examples/index.md - Default: examples/default.md - Multi Runner: examples/multi-runner.md - - Multi Runner v2: examples/multi-runner-v2.md - - Multi Runner scale-set: examples/multi-runner-scale-set.md - Ephemeral: examples/ephemeral.md - External managed secrets: examples/external-managed-ssm-secrets.md - Custom AMI: examples/prebuilt.md diff --git a/tests/ministack/README.md b/tests/ministack/README.md index 1a00a7e32c..201cc0fda9 100644 --- a/tests/ministack/README.md +++ b/tests/ministack/README.md @@ -1,8 +1,7 @@ # MiniStack example tests The MiniStack workflow runs the `base`, `prebuilt`, `default`, `ephemeral`, -`multi-runner`, `multi-runner-v2`, `multi-runner-scale-set`, and -`termination-watcher` examples directly +`multi-runner`, and `termination-watcher` examples directly with Terraform 1.5.6 and the latest Terraform release, and with OpenTofu 1.11 and the latest OpenTofu release. The examples with input variables get their inputs from their own tfvars files @@ -11,7 +10,7 @@ and uses the configuration checked into the example itself. No override files, setup module, or Terraform fixture configuration is checked in. The helper creates and removes a temporary AMI override for `default` and `ephemeral`, temporary SSM parameters for `multi-runner`, and temporary AMI -fixtures for `multi-runner-v2` and `multi-runner-scale-set`. The migration test +fixtures as needed. The migration test uses its dedicated `run-migration-test.sh` lifecycle script. Start MiniStack, set the AWS endpoint and test credentials, then run: @@ -27,10 +26,6 @@ tests/ministack/run-example.sh apply ephemeral # or tests/ministack/run-example.sh apply multi-runner # or -tests/ministack/run-example.sh apply multi-runner-v2 -# or -tests/ministack/run-example.sh apply multi-runner-scale-set -# or tests/ministack/run-example.sh apply termination-watcher ``` @@ -39,59 +34,5 @@ ZIP fixtures in the paths expected by the modules when they are absent, and removes only the files it created. For `prebuilt`, it seeds AMI metadata through MiniStack's AWS-compatible EC2 API, then removes only the resources it created during cleanup. MiniStack v1.5.11 provides the EC2 image behavior needed by the -`default`, `ephemeral`, `multi-runner`, and `multi-runner-scale-set` examples, -so they are included in the same lifecycle matrix. - -## Webhook and runner lifecycle smoke test - -The smoke test covers two independent lifecycle chains. The webhook chain -sends signed `workflow_job` webhooks through the API Gateway endpoint and -verifies the asynchronous path through EventBridge, the dispatcher Lambda, SQS, -and the scale-up Lambda. It runs scale-up once without a dynamic label and once -with `ghr-ec2-instance-type:m5.large`, checking that the first launch uses a -configured default instance type and the second launch uses exactly `m5.large`. -The scale-up Lambda calls a pinned `mockserver/mockserver` container initialized -from `github-api-expectations.json`; the test uses MockServer's verification API -to confirm the expected GitHub API calls for both jobs. It also checks the -webhook, dispatcher, and scale-up Lambda log groups for each smoke job ID, then -confirms that both MiniStack EC2 runner instances are removed and terminated. - -The second, pool chain then invokes the pool Lambda with a pool size of one and verifies every -expected GitHub API route for pool reconciliation, including the installation, -token, runner-list, and registration-token calls, before confirming that it -creates a second EC2 runner. Installation lookup is mocked for configurations -that do not provide a stored installation ID, but is conditional and is not a -required assertion. The test also verifies the `ghr:Application`, -`ghr:created_by`, `ghr:Type`, and `ghr:Owner` tags used to discover managed -instances. MiniStack v1.5.10 propagates the Terraform launch-template tags to -instances, allowing the scale-down Lambda to discover and remove each runner. -The smoke test invokes scale-down for the webhook and pool-created runners and -verifies the GitHub API calls and EC2 termination. -The pool schedule is configured for a far-future date because the test invokes -the Lambda directly. - -Build the two real Lambda distributions, start MiniStack, and run: - -```sh -(cd lambdas && yarn install --frozen-lockfile) -(cd lambdas && yarn workspace @aws-github-runner/webhook dist) -(cd lambdas && yarn workspace @aws-github-runner/control-plane dist) -sh tests/ministack/run-smoke.sh -``` - -The smoke script generates a temporary RSA key and Terraform variables file, -starts the MockServer container on a temporary port, and removes all temporary -state during cleanup. In CI, the pinned MockServer setup action starts the -server and waits for readiness; the expectations are loaded after checkout. -MiniStack must be able to reach -`host.docker.internal`; -override the hostname with `MINISTACK_GITHUB_MOCK_HOST` when using a different -container runtime. When MiniStack is exposed on a non-default local port, use a -host address reachable from its container for `AWS_ENDPOINT_URL`, for example -`AWS_ENDPOINT_URL=http://:14568`, instead of `127.0.0.1`. - -The workflow also runs `run-scale-set-integration.sh`. It applies the -`multi-runner-scale-set` example and verifies the managed ECS controller, -Fargate task hardening, scale-set environment contract, and reconciler SSM -parameter through MiniStack's AWS-compatible APIs. It does not send webhook -events or exercise webhook scale-up, scale-down, or pool handlers. +`default`, `ephemeral`, and `multi-runner` examples, so they are included in the +same lifecycle matrix. diff --git a/tests/ministack/github-api-expectations.json b/tests/ministack/github-api-expectations.json deleted file mode 100644 index a85e727bd2..0000000000 --- a/tests/ministack/github-api-expectations.json +++ /dev/null @@ -1,77 +0,0 @@ -[ - { - "httpRequest": { - "method": "GET", - "path": "/api/v3/repos/test-owner/test-repo/actions/jobs/123456" - }, - "httpResponse": { - "statusCode": 200, - "headers": { - "Content-Type": ["application/json"], - "X-RateLimit-Limit": ["5000"], - "X-RateLimit-Remaining": ["4999"] - }, - "body": "{\"id\":123456,\"status\":\"queued\",\"name\":\"ministack-smoke\"}" - } - }, - { - "httpRequest": { - "method": "GET", - "path": "/api/v3/repos/test-owner/test-repo/actions/jobs/123457" - }, - "httpResponse": { - "statusCode": 200, - "headers": { - "Content-Type": ["application/json"], - "X-RateLimit-Limit": ["5000"], - "X-RateLimit-Remaining": ["4999"] - }, - "body": "{\"id\":123457,\"status\":\"queued\",\"name\":\"ministack-smoke-dynamic\"}" - } - }, - { - "httpRequest": { - "method": "POST", - "path": "/api/v3/app/installations/123/access_tokens" - }, - "httpResponse": { - "statusCode": 201, - "headers": { - "Content-Type": ["application/json"], - "X-RateLimit-Limit": ["5000"], - "X-RateLimit-Remaining": ["4999"] - }, - "body": "{\"token\":\"ministack-installation-token\",\"expires_at\":\"2099-01-01T00:00:00Z\"}" - } - }, - { - "httpRequest": { - "method": "GET", - "path": "/api/v3/orgs/test-owner/installation" - }, - "httpResponse": { - "statusCode": 200, - "headers": { - "Content-Type": ["application/json"], - "X-RateLimit-Limit": ["5000"], - "X-RateLimit-Remaining": ["4999"] - }, - "body": "{\"id\":123}" - } - }, - { - "httpRequest": { - "method": "POST", - "path": "/api/v3/orgs/test-owner/actions/runners/registration-token" - }, - "httpResponse": { - "statusCode": 201, - "headers": { - "Content-Type": ["application/json"], - "X-RateLimit-Limit": ["5000"], - "X-RateLimit-Remaining": ["4999"] - }, - "body": "{\"token\":\"ministack-registration-token\",\"expires_at\":\"2099-01-01T00:00:00Z\"}" - } - } -] diff --git a/tests/ministack/initializerJson.json b/tests/ministack/initializerJson.json deleted file mode 100644 index 443ddb6201..0000000000 --- a/tests/ministack/initializerJson.json +++ /dev/null @@ -1,138 +0,0 @@ -[ - { - "httpRequest": { - "method": "POST", - "path": "/api/v3/app/installations/456/access_tokens" - }, - "httpResponse": { - "statusCode": 201, - "headers": { "Content-Type": ["application/json"] }, - "body": "{\"token\":\"fake-installation-token\",\"expires_at\":\"2099-01-01T00:00:00Z\"}" - } - }, - { - "httpRequest": { - "method": "POST", - "path": "/api/v3/orgs/example/actions/runners/registration-token" - }, - "httpResponse": { - "statusCode": 201, - "headers": { "Content-Type": ["application/json"] }, - "body": "{\"token\":\"fake-registration-token\"}" - } - }, - { - "httpRequest": { - "method": "POST", - "path": "/api/v3/actions/runner-registration" - }, - "httpResponse": { - "statusCode": 200, - "headers": { "Content-Type": ["application/json"] }, - "body": "{\"url\":\"https://mockserver:1080/tenant/123\",\"token\":\"header.eyJleHAiOjQwNzA5MDg4MDB9.signature\"}" - } - }, - { - "httpRequest": { - "method": "GET", - "path": "/tenant/123/_apis/runtime/runnergroups/" - }, - "httpResponse": { - "statusCode": 200, - "headers": { "Content-Type": ["application/json"] }, - "body": "{\"count\":1,\"value\":[{\"id\":48,\"name\":\"experimental-euw1-sl-cicd-forge-emu\"}]}" - } - }, - { - "httpRequest": { - "method": "GET", - "path": "/tenant/123/_apis/runtime/runnerscalesets" - }, - "httpResponse": { - "statusCode": 200, - "headers": { "Content-Type": ["application/json"] }, - "body": "{\"count\":1,\"value\":[{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}]}" - } - }, - { - "httpRequest": { - "method": "GET", - "path": "/tenant/123/_apis/runtime/runnerscalesets/223" - }, - "httpResponse": { - "statusCode": 200, - "headers": { "Content-Type": ["application/json"] }, - "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}" - } - }, - { - "httpRequest": { - "method": "PATCH", - "path": "/tenant/123/_apis/runtime/runnerscalesets/223" - }, - "httpResponse": { - "statusCode": 200, - "headers": { "Content-Type": ["application/json"] }, - "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"},{\"name\":\"linux\",\"type\":\"user\"},{\"name\":\"scale-set\",\"type\":\"user\"},{\"name\":\"self-hosted\",\"type\":\"user\"},{\"name\":\"x64\",\"type\":\"user\"}],\"runnerSetting\":{}}" - } - }, - { - "httpRequest": { - "method": "POST", - "path": "/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig" - }, - "httpResponseTemplate": { - "templateType": "MUSTACHE", - "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"runner\\\":{\\\"id\\\":321,\\\"name\\\":\\\"{{#jsonPath}}$.name{{/jsonPath}}{{jsonPathResult}}\\\",\\\"runnerScaleSetId\\\":223},\\\"encodedJITConfig\\\":\\\"test-only-jit-configuration\\\"}\"}" - } - }, - { - "httpRequest": { - "method": "POST", - "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions" - }, - "httpResponse": { - "statusCode": 200, - "headers": { "Content-Type": ["application/json"] }, - "body": "{\"sessionId\":\"11111111-1111-1111-1111-111111111111\",\"ownerName\":\"local.medium\",\"runnerScaleSet\":{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48},\"messageQueueUrl\":\"https://mockserver:1080/messages?sessionId=11111111-1111-1111-1111-111111111111&api-version=6.0-preview\",\"messageQueueAccessToken\":\"fake-queue-token\",\"statistics\":{\"totalAvailableJobs\":0,\"totalAcquiredJobs\":0,\"totalAssignedJobs\":0,\"totalRunningJobs\":0,\"totalRegisteredRunners\":0,\"totalBusyRunners\":0,\"totalIdleRunners\":0}}" - } - }, - { - "httpRequest": { - "method": "GET", - "path": "/messages" - }, - "httpResponse": { - "statusCode": 202, - "delay": { "timeUnit": "MILLISECONDS", "value": 1000 } - } - }, - { - "httpRequest": { - "method": "GET", - "path": "/tenant/123/_apis/distributedtask/pools/0/agents" - }, - "httpResponseTemplate": { - "templateType": "MUSTACHE", - "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"count\\\":1,\\\"value\\\":[{\\\"id\\\":321,\\\"name\\\":\\\"{{ request.queryStringParameters.agentName.0 }}\\\",\\\"runnerScaleSetId\\\":223}]}\"}" - } - }, - { - "httpRequest": { - "method": "DELETE", - "path": "/tenant/123/_apis/distributedtask/pools/0/agents/321" - }, - "httpResponse": { - "statusCode": 204 - } - }, - { - "httpRequest": { - "method": "DELETE", - "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111" - }, - "httpResponse": { - "statusCode": 204 - } - } -] diff --git a/tests/ministack/multi-runner-scale-set.tfvars b/tests/ministack/multi-runner-scale-set.tfvars deleted file mode 100644 index 7dd1a0ad39..0000000000 --- a/tests/ministack/multi-runner-scale-set.tfvars +++ /dev/null @@ -1,57 +0,0 @@ -environment = "ministack-scale-set" -aws_region = "eu-west-1" - -github = { - url = "https://mockserver:1080" - ssl_verify = false - runner_owner = "example" - registration_level = "organization" -} - -github_app = { - id = "123" - key_base64 = "ministack-invalid-key" - installation_id = "456" -} - -runner_binaries_enabled = false - -ami = { - "linux-arm64" = { - filter = { - name = ["ministack-scale-set-linux-arm64"] - state = ["available"] - } - owners = ["self"] - } - "linux-x64" = { - filter = { - name = ["ministack-scale-set-linux-x64"] - state = ["available"] - } - owners = ["self"] - } - "linux-scale-set" = { - filter = { - name = ["ministack-scale-set-linux-x64"] - state = ["available"] - } - owners = ["self"] - } - "windows-x64" = { - filter = { - name = ["ministack-scale-set-windows-x64"] - state = ["available"] - } - owners = ["self"] - } -} - -scale_set = { - name = "medium" - runner_group_name = "experimental-euw1-sl-cicd-forge-emu" - min_runners = 1 - container = { - image = "localhost:4566/scale-set-controller:smoke" - } -} diff --git a/tests/ministack/multi-runner-v2.tfvars b/tests/ministack/multi-runner-v2.tfvars deleted file mode 100644 index de54e291cf..0000000000 --- a/tests/ministack/multi-runner-v2.tfvars +++ /dev/null @@ -1,33 +0,0 @@ -environment = "ministack-v2" -aws_region = "eu-west-1" - -github_app = { - id = "0" - key_base64 = "ministack-invalid-key" -} - -runner_binaries_enabled = false - -ami = { - "linux-arm64" = { - filter = { - name = ["ministack-v2-linux-arm64"] - state = ["available"] - } - owners = ["self"] - } - "linux-x64" = { - filter = { - name = ["ministack-v2-linux-x64"] - state = ["available"] - } - owners = ["self"] - } - "windows-x64" = { - filter = { - name = ["ministack-v2-windows-x64"] - state = ["available"] - } - owners = ["self"] - } -} diff --git a/tests/ministack/run-scale-set-integration.sh b/tests/ministack/run-scale-set-integration.sh deleted file mode 100755 index 08e0d156de..0000000000 --- a/tests/ministack/run-scale-set-integration.sh +++ /dev/null @@ -1,600 +0,0 @@ -#!/bin/sh - -set -eu - -export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-000000000000}" -export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-test-only}" -export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-1}" -export AWS_REGION="${AWS_REGION:-eu-west-1}" -export AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://127.0.0.1:4566}" -export AWS_EC2_METADATA_DISABLED="${AWS_EC2_METADATA_DISABLED:-true}" - -script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) -source_root=$(CDPATH='' cd -- "$script_dir/../.." && pwd) -example="multi-runner-scale-set" -base_tfvars="$script_dir/$example.tfvars" -tfvars_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set.XXXXXX") -app_key_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-key.XXXXXX") -log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-logs.XXXXXX") -controller_log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-controller-logs.XXXXXX") -config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-config.XXXXXX") -task_definition_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-task-definition.XXXXXX") -instance_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-instances.XXXXXX") -scale_down_config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-scale-down-config.XXXXXX") -initializer_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-mockserver.XXXXXX") -repository_name="scale-set-controller" -image_reference="localhost:4566/${repository_name}:smoke" -mockserver_host="${MINISTACK_GITHUB_MOCK_HOST:-127.0.0.1}" -mockserver_port="${MINISTACK_GITHUB_MOCK_PORT:-1080}" -mockserver_url="${MINISTACK_GITHUB_MOCK_URL:-http://127.0.0.1:${mockserver_port}}" -controller_mock_url="https://${mockserver_host}:${mockserver_port}" -terraform_state_exists=false - -cleanup() { - cleanup_status=$? - set +e - - if [ "$terraform_state_exists" = true ]; then - "$source_root/tests/ministack/run-example.sh" destroy "$example" "$tfvars_file" >/dev/null 2>&1 - fi - - rm -f "$tfvars_file" "$app_key_file" "$log_file" "$controller_log_file" "$config_file" "$task_definition_file" "$instance_file" "$scale_down_config_file" "$initializer_file" - exit "$cleanup_status" -} -trap cleanup EXIT INT TERM - -require_command() { - if ! command -v "$1" >/dev/null 2>&1; then - echo "$1 is required to run the scale-set MiniStack smoke test." >&2 - exit 69 - fi -} - -for command in aws curl docker openssl python3 terraform; do - require_command "$command" -done - -ministack_aws() { - aws --endpoint-url "$AWS_ENDPOINT_URL" --region "$AWS_DEFAULT_REGION" "$@" -} - -wait_for_http() { - url="$1" - attempts=90 - while ! curl -fsS --max-time 2 "$url" >/dev/null 2>&1; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for $url." >&2 - exit 70 - fi - sleep 1 - done -} - -wait_for_scale_set_runner() { - attempts=90 - while :; do - ministack_aws ec2 describe-instances --output json > "$instance_file" - instance_id=$(python3 - "$instance_file" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as instance_file: - response = json.load(instance_file) - -required_tags = { - "ghr:Application": "github-action-runner", - "ghr:created_by": "scale-set-service", - "ghr:environment": "ministack-scale-set-linux-scale-set", - "ghr:Type": "Org", - "ghr:Owner": "example", - "ghr:scale_set_state": "config-published", - "ghr:github_runner_id": "321", -} -active_states = {"pending", "running", "stopping", "stopped", "shutting-down"} -matches = [] -for reservation in response.get("Reservations", []): - for instance in reservation.get("Instances", []): - if instance.get("State", {}).get("Name") not in active_states: - continue - tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} - if all(tags.get(key) == value for key, value in required_tags.items()) and tags.get("ghr:runner_name", "").startswith("scale-set-"): - matches.append(instance["InstanceId"]) - -if len(matches) == 1: - print(matches[0]) -PY -) - if [ -n "$instance_id" ]; then - printf ' [PASS] MiniStack created and registered scale-set EC2 runner %s\n' "$instance_id" - return 0 - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for the scale-set EC2 runner to reach config-published state." >&2 - cat "$instance_file" >&2 || true - exit 1 - fi - sleep 2 - done -} - -wait_for_no_scale_set_runners() { - attempts=90 - while :; do - ministack_aws ec2 describe-instances --output json > "$instance_file" - active_count=$(python3 - "$instance_file" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as instance_file: - response = json.load(instance_file) - -required_tags = { - "ghr:Application": "github-action-runner", - "ghr:created_by": "scale-set-service", - "ghr:environment": "ministack-scale-set-linux-scale-set", - "ghr:Type": "Org", - "ghr:Owner": "example", -} -active_states = {"pending", "running", "stopping", "stopped", "shutting-down"} -count = 0 -for reservation in response.get("Reservations", []): - for instance in reservation.get("Instances", []): - if instance.get("State", {}).get("Name") not in active_states: - continue - tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} - if all(tags.get(key) == value for key, value in required_tags.items()): - count += 1 -print(count) -PY -) - if [ "$active_count" = "0" ]; then - printf '%s\n' ' [PASS] scale-set EC2 runner was terminated after the minimum changed to zero' - return 0 - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for the scale-set EC2 runner to terminate; active count is $active_count." >&2 - cat "$instance_file" >&2 || true - exit 1 - fi - sleep 2 - done -} - -wait_for_http "$AWS_ENDPOINT_URL/_ministack/health" -attempts=90 -while ! curl -fsS --max-time 2 -X PUT "$mockserver_url/mockserver/status" >/dev/null 2>&1; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for $mockserver_url/mockserver/status." >&2 - exit 70 - fi - sleep 1 -done -curl -fsS -X PUT "$mockserver_url/mockserver/reset" >/dev/null -CONTROLLER_MOCK_URL="$controller_mock_url" python3 - "$source_root/tests/ministack/initializerJson.json" "$initializer_file" <<'PY' -import os -import sys - -source, destination = sys.argv[1:] -with open(source, encoding="utf-8") as source_file: - fixture = source_file.read() -fixture = fixture.replace("https://mockserver:1080", os.environ["CONTROLLER_MOCK_URL"]) -with open(destination, "w", encoding="utf-8") as destination_file: - destination_file.write(fixture) -PY -curl -fsS -X PUT \ - "$mockserver_url/mockserver/expectation" \ - -H 'Content-Type: application/json' \ - --data-binary "@$initializer_file" \ - >/dev/null - -repository_policy=$(python3 - <<'PY' -import json - -print(json.dumps({ - "Version": "2012-10-17", - "Statement": [{ - "Sid": "AllowAccountPull", - "Effect": "Allow", - "Principal": {"AWS": "arn:aws:iam::000000000000:root"}, - "Action": [ - "ecr:BatchCheckLayerAvailability", - "ecr:BatchGetImage", - "ecr:GetDownloadUrlForLayer", - ], - }], -})) -PY -) - -ministack_aws ecr create-repository \ - --repository-name "$repository_name" \ - --image-tag-mutability IMMUTABLE \ - --image-scanning-configuration scanOnPush=false \ - >/dev/null -ministack_aws ecr set-repository-policy \ - --repository-name "$repository_name" \ - --policy-text "$repository_policy" \ - >/dev/null - -docker build \ - --target runtime \ - --file "$source_root/lambdas/services/scale-set/Dockerfile" \ - --tag "$image_reference" \ - "$source_root" - -ministack_aws ecr get-login-password | docker login \ - --username AWS \ - --password-stdin localhost:4566 >/dev/null -docker push "$image_reference" - -ministack_aws ecr describe-images \ - --repository-name "$repository_name" \ - --image-ids imageTag=smoke \ - >/dev/null - -openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$app_key_file" 2>/dev/null -app_key_base64=$(base64 < "$app_key_file" | tr -d '\n') -APP_KEY_BASE64="$app_key_base64" GITHUB_CONFIG_URL="$controller_mock_url" python3 - "$base_tfvars" "$tfvars_file" <<'PY' -import os -import sys - -source, destination = sys.argv[1:] -replacement = os.environ["APP_KEY_BASE64"] -github_config_url = os.environ["GITHUB_CONFIG_URL"] -with open(source, encoding="utf-8") as source_file: - lines = source_file.readlines() -with open(destination, "w", encoding="utf-8") as destination_file: - for line in lines: - if line.lstrip().startswith("key_base64"): - destination_file.write(f' key_base64 = "{replacement}"\n') - elif line.lstrip().startswith("config_url"): - destination_file.write(f' config_url = "{github_config_url}"\n') - else: - destination_file.write(line) -PY -unset app_key_base64 APP_KEY_BASE64 - -terraform_state_exists=true -"$source_root/tests/ministack/run-example.sh" apply "$example" "$tfvars_file" - -config_path="/ministack-scale-set/scale-set-controller/linux-scale-set/linux-scale-set" -ministack_aws ssm get-parameter \ - --name "$config_path" \ - --query 'Parameter.Value' \ - --output text > "$config_file" -EXPECTED_GITHUB_CONFIG_URL="$controller_mock_url/example" python3 - "$config_file" <<'PY' -import json -import os -import sys - -with open(sys.argv[1], encoding="utf-8") as config_file: - config = json.load(config_file) - -assert config["githubConfigUrl"] == os.environ["EXPECTED_GITHUB_CONFIG_URL"] -assert config["forceGhes"] is True -assert config["sslVerify"] is False -assert config["minRunners"] == 1 -assert config["githubApp"]["appIdParameterName"] -assert config["githubApp"]["installationIdParameterName"] -assert config["githubApp"]["privateKeyParameterName"] -print(" [PASS] SSM manifest has the expected MockServer and GitHub App settings") -PY - -task_definition=$(ministack_aws ecs list-task-definitions \ - --family-prefix ministack-scale-set-ss-linux-scale-se- \ - --sort DESC \ - --query 'taskDefinitionArns[0]' \ - --output text) -if [ -z "$task_definition" ] || [ "$task_definition" = "None" ]; then - echo "The scale-set task definition was not registered." >&2 - exit 1 -fi - -actual_image=$(ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].image | [0]' \ - --output text) -if [ "$actual_image" != "$image_reference" ]; then - echo "Expected the ECS task to use $image_reference, got $actual_image." >&2 - exit 1 -fi -printf '%s\n' ' [PASS] ECS task definition uses the image pushed to MiniStack ECR' - -log_driver=$(ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].logConfiguration.logDriver | [0]' \ - --output text) -if [ "$log_driver" != "awslogs" ]; then - echo "Expected the ECS task to request the awslogs driver, got $log_driver." >&2 - exit 1 -fi -printf '%s\n' ' [PASS] ECS task definition requests the awslogs driver' - -log_group=$(ministack_aws logs describe-log-groups \ - --log-group-name-prefix "/aws/ecs/ministack-scale-set-ss-linux-scale-se-" \ - --query 'logGroups[0].logGroupName' \ - --output text) -if [ -z "$log_group" ] || [ "$log_group" = "None" ]; then - echo "The scale-set CloudWatch log group was not created." >&2 - exit 1 -fi -printf '%s\n' ' [PASS] CloudWatch log group was created for the ECS controller' - -task_family=$(ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition.family' \ - --output text) - -# MiniStack exposes ECS credentials through the gateway's container IP. The -# Node.js AWS SDK intentionally rejects that address in -# AWS_CONTAINER_CREDENTIALS_FULL_URI because non-HTTPS full URIs are limited -# to loopback and the real ECS metadata address. Use only synthetic, -# test-scoped credentials in a temporary task-definition revision so the -# smoke test still exercises the pushed image, ECS service, and controller -# lifecycle without changing the production task definition or application. -ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition' \ - --output json > "$task_definition_file" -TASK_DEFINITION_FILE="$task_definition_file" python3 - <<'PY' -import json -import os - -path = os.environ["TASK_DEFINITION_FILE"] -with open(path, encoding="utf-8") as task_definition_file: - task_definition = json.load(task_definition_file) - -for field in ( - "taskDefinitionArn", - "revision", - "status", - "requiresAttributes", - "compatibilities", - "registeredAt", - "registeredBy", -): - task_definition.pop(field, None) - -for container in task_definition["containerDefinitions"]: - if container["name"] != "scale-set-controller": - continue - environment = container.setdefault("environment", []) - environment.extend([ - {"name": "AWS_ACCESS_KEY_ID", "value": "000000000000"}, - {"name": "AWS_SECRET_ACCESS_KEY", "value": "test-only"}, - ]) - -with open(path, "w", encoding="utf-8") as task_definition_file: - json.dump(task_definition, task_definition_file) -PY - -environment_name=$(sed -n 's/^environment[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$base_tfvars") -cluster_name="${environment_name}-scale-set" -task_definition=$(ministack_aws ecs register-task-definition \ - --cli-input-json "file://$task_definition_file" \ - --query 'taskDefinition.taskDefinitionArn' \ - --output text) -ministack_aws ecs update-service \ - --cluster "$cluster_name" \ - --service "$task_family" \ - --task-definition "$task_definition" \ - >/dev/null - -task_revision=$(ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition.revision' \ - --output text) -controller_container="" -attempts=90 -while [ -z "$controller_container" ]; do - controller_container=$(docker ps -a \ - --filter "label=com.amazonaws.ecs.task-definition-family=$task_family" \ - --filter "label=com.amazonaws.ecs.task-definition-version=$task_revision" \ - --filter 'name=scale-set-controller' \ - --format '{{.ID}}' | sed -n '1p') - if [ -n "$controller_container" ]; then - break - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for the MiniStack ECS controller container." >&2 - docker ps -a --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' >&2 || true - exit 1 - fi - sleep 2 -done -printf ' [PASS] MiniStack started ECS controller container %s\n' "$controller_container" - -wait_for_controller_log_event() { - marker="$1" - required_text="${2:-}" - attempts=90 - while :; do - docker logs "$controller_container" > "$controller_log_file" 2>&1 || true - if python3 - "$controller_log_file" "$marker" "$required_text" <<'PY' -import sys - -marker, required = sys.argv[2:] -with open(sys.argv[1], encoding="utf-8") as log_file: - messages = log_file.read().splitlines() -if any(marker in message and (not required or required in message) for message in messages): - raise SystemExit(0) -raise SystemExit(1) -PY - then - printf ' [PASS] CloudWatch logs contain %s\n' "$marker" - return 0 - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for controller log marker '$marker'." >&2 - cat "$controller_log_file" >&2 || true - echo "Controller AWS/ECS metadata environment:" >&2 - docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" \ - | sed -E 's/^(AWS_CONTAINER_AUTHORIZATION_TOKEN|AWS_CONTAINER_CREDENTIALS_FULL_URI)=.*/\1=/' \ - | grep -E '^(AWS_|ECS_)' >&2 || true - echo "Controller network attachments:" >&2 - docker inspect --format '{{json .NetworkSettings.Networks}}' "$controller_container" >&2 || true - echo "Controller credential endpoint probe:" >&2 - docker exec "$controller_container" node -e \ - 'fetch(process.env.AWS_CONTAINER_CREDENTIALS_FULL_URI, {headers: {Authorization: process.env.AWS_CONTAINER_AUTHORIZATION_TOKEN}}).then((response) => { console.error(`status=${response.status}`); process.exit(response.ok ? 0 : 1); }).catch((error) => { console.error(`${error.name}:${error.message}`); process.exit(1); })' \ - >&2 || true - exit 1 - fi - sleep 2 - done -} - -printf '%s\n' ' [INFO] MiniStack 1.5.12 does not emit ECS awslogs streams; validating controller runtime logs instead' -wait_for_controller_log_event 'scale_set_controller_started' -wait_for_controller_log_event 'scale_set_session_created' -wait_for_controller_log_event 'scale_set_reconciled' '"desiredRunners":1' -wait_for_controller_log_event 'scale_set_reconciled' '"status":"converged"' -wait_for_scale_set_runner - -wait_for_mock_route() { - method="$1" - route="$2" - body=$(REQUEST_METHOD="$method" REQUEST_PATH="$route" python3 - <<'PY' -import json -import os - -print(json.dumps({ - "httpRequest": { - "method": os.environ["REQUEST_METHOD"], - "path": os.environ["REQUEST_PATH"], - }, - "times": {"atLeast": 1}, -})) -PY - ) - attempts=45 - while ! curl -fsS --max-time 5 -X PUT \ - http://127.0.0.1:1080/mockserver/verify \ - -H 'Content-Type: application/json' \ - --data "$body" >/dev/null 2>&1; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for MockServer route: $method $route" >&2 - curl -sS --max-time 5 \ - 'http://127.0.0.1:1080/mockserver/retrieve?type=REQUESTS&format=JSON' >&2 || true - exit 1 - fi - sleep 2 - done - printf ' [PASS] MockServer received %s %s\n' "$method" "$route" -} - -wait_for_mock_route POST '/api/v3/app/installations/456/access_tokens' -wait_for_mock_route POST '/api/v3/orgs/example/actions/runners/registration-token' -wait_for_mock_route POST '/api/v3/actions/runner-registration' -wait_for_mock_route GET '/tenant/123/_apis/runtime/runnergroups/' -wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets' -wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets/223' -wait_for_mock_route PATCH '/tenant/123/_apis/runtime/runnerscalesets/223' -wait_for_mock_route POST '/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig' -wait_for_mock_route POST '/tenant/123/_apis/runtime/runnerscalesets/223/sessions' -wait_for_mock_route GET '/messages' - -python3 - "$config_file" "$scale_down_config_file" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as config_file: - reconciler = json.load(config_file) - -assert reconciler["minRunners"] == 1 -reconciler["minRunners"] = 0 -with open(sys.argv[2], "w", encoding="utf-8") as config_file: - json.dump(reconciler, config_file) -PY -ministack_aws ssm put-parameter \ - --name "$config_path" \ - --type String \ - --value "$(cat "$scale_down_config_file")" \ - --overwrite \ - >/dev/null -printf '%s\n' ' [PASS] SSM manifest minimum changed from one runner to zero' - -SCALE_DOWN_CONFIG_FILE="$scale_down_config_file" TASK_DEFINITION_FILE="$task_definition_file" python3 - <<'PY' -import json -import os - -with open(os.environ["SCALE_DOWN_CONFIG_FILE"], encoding="utf-8") as config_file: - reconciler = json.load(config_file) -with open(os.environ["TASK_DEFINITION_FILE"], encoding="utf-8") as task_definition_file: - task_definition = json.load(task_definition_file) - -for container in task_definition["containerDefinitions"]: - if container["name"] != "scale-set-controller": - continue - for environment in container.get("environment", []): - if environment["name"] == "SCALE_SET_CONTROLLER_MANIFEST": - manifest = json.loads(environment["value"]) - assert len(manifest.get("reconcilers", [])) == 1 - manifest["reconcilers"][0]["minRunners"] = reconciler["minRunners"] - environment["value"] = json.dumps(manifest, separators=(",", ":")) - break - else: - raise RuntimeError("scale-set controller task definition has no inline manifest") - -with open(os.environ["TASK_DEFINITION_FILE"], "w", encoding="utf-8") as task_definition_file: - json.dump(task_definition, task_definition_file) -PY -task_definition=$(ministack_aws ecs register-task-definition \ - --cli-input-json "file://$task_definition_file" \ - --query 'taskDefinition.taskDefinitionArn' \ - --output text) -task_revision=$(ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition.revision' \ - --output text) -ministack_aws ecs update-service \ - --cluster "$cluster_name" \ - --service "$task_family" \ - --task-definition "$task_definition" \ - --force-new-deployment \ - >/dev/null - -old_controller_container="$controller_container" -new_controller_container="" -attempts=90 -while [ -z "$new_controller_container" ]; do - for candidate in $(docker ps -a \ - --filter "label=com.amazonaws.ecs.task-definition-family=$task_family" \ - --filter "label=com.amazonaws.ecs.task-definition-version=$task_revision" \ - --filter 'name=scale-set-controller' \ - --format '{{.ID}}'); do - if [ "$candidate" != "$old_controller_container" ]; then - new_controller_container="$candidate" - break - fi - done - if [ -n "$new_controller_container" ]; then - break - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo 'Timed out waiting for the ECS service to deploy the scale-down task.' >&2 - docker ps -a --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' >&2 || true - exit 1 - fi - sleep 2 -done -controller_container="$new_controller_container" -printf ' [PASS] ECS service deployed a fresh controller container %s for scale-down\n' "$controller_container" -wait_for_controller_log_event 'scale_set_controller_started' -wait_for_controller_log_event 'scale_set_session_created' -wait_for_controller_log_event 'scale_set_reconciled' '"desiredRunners":0' -wait_for_controller_log_event 'scale_set_reconciled' '"status":"converged"' -wait_for_no_scale_set_runners - -"$source_root/tests/ministack/run-example.sh" destroy "$example" "$tfvars_file" -terraform_state_exists=false -wait_for_mock_route DELETE '/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111' - -echo 'Scale-set MiniStack ECS/MockServer smoke test passed.' diff --git a/tests/ministack/workflow_job_event.json b/tests/ministack/workflow_job_event.json deleted file mode 100644 index 6200b40fb1..0000000000 --- a/tests/ministack/workflow_job_event.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "action": "queued", - "workflow_job": { - "id": 123456, - "run_id": 654321, - "run_url": "https://github.example.invalid/test-owner/test-repo/actions/runs/654321", - "url": "https://github.example.invalid/test-owner/test-repo/actions/jobs/123456", - "html_url": "https://github.example.invalid/test-owner/test-repo/actions/jobs/123456", - "status": "queued", - "conclusion": null, - "name": "ministack-smoke", - "labels": ["self-hosted", "linux", "x64", "default", "example"] - }, - "repository": { - "id": 1, - "name": "test-repo", - "full_name": "test-owner/test-repo", - "owner": { - "login": "test-owner", - "type": "Organization" - } - }, - "installation": { - "id": 123 - } -} diff --git a/tests/multi-runner-orchestration/Dockerfile b/tests/multi-runner-orchestration/Dockerfile new file mode 100644 index 0000000000..0e1b6f82da --- /dev/null +++ b/tests/multi-runner-orchestration/Dockerfile @@ -0,0 +1,18 @@ +FROM eclipse-temurin:21-jdk AS certificate +RUN keytool -genkeypair \ + -alias wiremock \ + -keyalg RSA \ + -keysize 2048 \ + -validity 3650 \ + -storetype PKCS12 \ + -keystore /tmp/wiremock.p12 \ + -storepass changeit \ + -keypass changeit \ + -dname "CN=wiremock" \ + -ext "SAN=dns:wiremock,dns:localhost,ip:127.0.0.1" \ + -noprompt + +FROM wiremock/wiremock:3.13.2 +COPY --from=certificate /tmp/wiremock.p12 /home/wiremock/wiremock.p12 +COPY mappings /home/wiremock/mappings +ENTRYPOINT ["/docker-entrypoint.sh", "--https-port", "8443", "--https-keystore", "/home/wiremock/wiremock.p12", "--keystore-password", "changeit", "--key-manager-password", "changeit", "--keystore-type", "PKCS12", "--global-response-templating"] diff --git a/tests/multi-runner-orchestration/README.md b/tests/multi-runner-orchestration/README.md new file mode 100644 index 0000000000..9bfe69b528 --- /dev/null +++ b/tests/multi-runner-orchestration/README.md @@ -0,0 +1,105 @@ +# WireMock for `terraform-aws-github-runner` Lambdas + +This standalone WireMock instance contains happy-path stubs for the GitHub REST and GitHub Actions scale-set calls found under `lambdas/`. It accepts both the GitHub Cloud path form (`/repos/...`) and the GHES form (`/api/v3/repos/...`). It is a deterministic test double, not a complete GitHub emulator. + +## Start it + +From the repository root: + +```sh +docker compose -f tests/multi-runner-orchestration/docker-compose.yml up --build --detach +curl --fail http://localhost:8080/__admin/mappings +``` + +The server listens on HTTP `8080` and HTTPS `8443`. The self-signed certificate is only for local/CI testing. Scale-set protocol responses contain the same HTTPS base URL used for its registration handshake. The scale-set test config must set `sslVerify: false` for this test certificate. + +To stop it: + +```sh +docker compose -f tests/multi-runner-orchestration/docker-compose.yml down +``` + +## MiniStack fixtures + +`docker compose --parallel 1 -f tests/multi-runner-orchestration/docker-compose.yml up --build` runs `.ci/build.sh` to build the Lambda distributions, starts MiniStack, seeds the AMI, the `scale-set-controller` ECR repository/policy, builds the controller for `linux/amd64` and `linux/arm64`, and pushes it to MiniStack. The Compose services use `linux/amd64`; CI enables QEMU to build the ARM64 image on its AMD64 runner. + +The controller image build is configured directly in the `docker-buildx` Compose service; it is not a fixture in `fixtures/ministack-fixtures.json`. Compose creates the `multi-runner-orchestration` network and attaches MiniStack's nested ECS containers to it. The AWS CLI uses `ministack:4566` to reach MiniStack. The job resolves MiniStack's network IP for BuildKit's registry push, since BuildKit does not resolve Compose service DNS in this setup; it configures that registry as plain HTTP. + +## Programmatic MiniStack smoke test + +Compose owns the environment lifecycle. Run `terraform-apply` as a blocking step before starting the smoke test container; the smoke command uses `--no-deps` so it cannot begin early or trigger a second apply. The apply service starts its prerequisites, seeds AWS fixtures, builds the Lambda archives and scale-set controller image, then applies `examples/multi-runner-orchestration`. The smoke step runs only after that command exits successfully and returns the smoke container's exit code. The job first registers a scale-set task-definition revision with dummy MiniStack credentials and updates the ECS service: + +```sh +docker compose --parallel 1 -f tests/multi-runner-orchestration/docker-compose.yml build +docker compose -f tests/multi-runner-orchestration/docker-compose.yml run --no-TTY --rm terraform-apply +docker compose -f tests/multi-runner-orchestration/docker-compose.yml run --no-TTY --rm --no-deps smoke-tests +``` + +Stop the long-running MiniStack and WireMock services after the command finishes: + +```sh +docker compose -f tests/multi-runner-orchestration/docker-compose.yml down --remove-orphans +``` + +Name smoke suites `..test.mjs`; the current suites are `webhook.ec2.test.mjs` and `scale-set.ec2.test.mjs`. To cover MicroVM, add `webhook.microvm.test.mjs` and/or `scale-set.microvm.test.mjs`. Webhook suites can reuse `sendWorkflowJob` and shared AWS/CloudWatch helpers from `smoke/support.mjs`; scale-set suites can register the common lifecycle with `registerScaleSetLifecycleTest` from `smoke/scale-set-support.mjs` and provide provider-specific reconciliation assertions. Keep resource checks and cleanup provider-specific. The `*.test.mjs` command discovers new suites automatically. The suite contains four named Node.js test cases: EC2 webhook scale-up and scale-down with standard labels, EC2 webhook scale-up and scale-down with a dynamic instance-type label, EC2 pool runner creation and scale-down, and EC2 scale-set scale-up and scale-down. The webhook and pool cases assert the EC2 runner count before launch, after launch, and after cleanup; the dynamic-label case also verifies the requested `m5.large` override in the scale-up logs. The scale-set case checks zero runners at baseline, one runner after scale-up, and a termination returning the count to zero. All cases assert the corresponding Lambda or controller CloudWatch logs. WireMock API responses are test fixtures, so the tests do not assert those routes. Run the smoke cases with the concise reporter: + +MiniStack does not start the EC2 runner agent, so the webhook and pool tests invoke scale-down twice: the first pass marks the unregistered instance orphaned, and the next pass confirms it is absent from GitHub and terminates it. + +```sh +node --test --test-concurrency=1 \ + --test-reporter=./tests/multi-runner-orchestration/smoke/reporter.mjs \ + tests/multi-runner-orchestration/smoke/*.test.mjs +``` + +The scale-set check registers temporary ECS task-definition revisions with the requested minimum runner count and dummy MiniStack credentials, then updates the local ECS service to exercise reconciliation. The Compose MiniStack container owns the ephemeral AWS state. + +## Route the Lambdas to the mock + +The Compose stack uses the named `multi-runner-orchestration_default` network. MiniStack and its nested ECS containers join that network, so use service DNS names (`ministack` and `github-api`) from containers. For the control-plane and termination-watcher Octokit clients, set `GHES_URL=http://github-api:8080`. Their existing GHES URL handling appends `/api/v3`, which these mappings accept. The webhook itself is inbound to the webhook Lambda; send your signed `workflow_job` fixture to that Lambda's local/API Gateway endpoint. The subsequent GitHub API calls are handled by this mock. + +For the scale-set controller, use a test config with `githubConfigUrl: "https://localhost:8443/acme"` for a host-run test (or `https://github-api:8443/acme` from the Compose network), `forceGhes: true`, and `sslVerify: false`. The mock supplies the scale-set admin handshake, scale-set/session responses, and an empty long-poll response. The controller should then remain healthy and poll until your test stops it. + +Scale-set stubs follow `lambdas/libs/github-actions-scale-set/src/endpoints.ts`, `client.ts`, and `message-session-client.ts`. They accept the tenant-scoped `/{tenant}/_apis/...` paths returned by the registration handshake and return the `/messages?sessionId=...` queue URL used by the MiniStack initializer fixture. The fixture currently uses tenant `123`, runner group `48`, scale set `223`, and runner `321`. + +The mock does not validate GitHub App JWTs, installation permissions, runner token validity, or GitHub's authorization policy. It accepts any request matching a mapping. It also does not implement the runner agent's actual Actions service protocol; the scale-set routes here support the Lambda controller's HTTP client only. + +## GitHub API mappings used by this smoke suite + +The mappings directory contains only the requests exercised by the current MiniStack smoke paths: + +| Flow | Requests covered | +|---|---| +| Webhook and EC2 pool | `POST /app/installations/{id}/access_tokens`; `GET /orgs/{org}/installation` (pool installation lookup); `POST /orgs/{org}/actions/runners/registration-token`; `GET /orgs/{org}/actions/runners`; `GET /orgs/{org}/actions/runner-groups`; `POST /orgs/{org}/actions/runners/generate-jitconfig`; `GET /orgs/{org}/actions/runners/{runner_id}` during orphan cleanup; `GET /repos/{owner}/{repo}/actions/jobs/{job_id}` | +| EC2 scale set | Registration handshake; runner-group and scale-set lookup/update; JIT runner config; message-session create/close and empty queue polling; runner lookup by name and removal | + +Mapping filenames use `...json`; add provider-specific files alongside the matching smoke suite when new paths are exercised. Fixtures for other GitHub API paths can be added in a follow-up. + +## CI usage + +Build and run this image before the Lambda integration step, then set the API URL/environment above in that step. For example, from a GitHub Actions job running on the host: + +```yaml +- name: Start GitHub API mock + run: | + docker build -t local/github-api-mock ./tests/multi-runner-orchestration + docker run -d --name github-api-mock \ + -p 8080:8080 -p 8443:8443 \ + local/github-api-mock + +- name: Run Lambda smoke test + env: + GHES_URL: http://127.0.0.1:8080 + run: + +- name: Stop GitHub API mock + if: always() + run: docker rm -f github-api-mock +``` + +If the Lambdas run in job containers instead of on the host, put them and the mock on the same user-defined Docker network. Use `GHES_URL=http://github-api-mock:8080` for Octokit and `githubConfigUrl=https://github-api-mock:8443/acme` for the scale-set controller. The handshake/session mappings derive the queue URL from the incoming request base URL. + +## Limits and current code routing + +Most Lambdas already support a configurable GHES API base (`GHES_URL` or the scale-set `githubConfigUrl`), so the mock can be used without changing their request code. `gh-agent-syncer` currently constructs a default `Octokit` client, so its release API request still targets `api.github.com` unless that Lambda is given an API base URL override or the test adds a host-level redirect. The mock includes the endpoint and download fixture, but it cannot intercept that hard-coded client by itself. The release fixture is the Linux x64 archive used by the default syncer settings; adjust its mapping for a different OS/architecture. + +Unmatched requests return WireMock's normal 404 with request-match diagnostics. Inspect received calls at `http://localhost:8080/__admin/requests`. diff --git a/tests/multi-runner-orchestration/buildkitd.toml b/tests/multi-runner-orchestration/buildkitd.toml new file mode 100644 index 0000000000..52d6d3b0ba --- /dev/null +++ b/tests/multi-runner-orchestration/buildkitd.toml @@ -0,0 +1,5 @@ +insecure-entitlements = ["network.host"] + +[worker.oci] + max-parallelism = 1 + networkMode = "host" diff --git a/tests/multi-runner-orchestration/docker-compose.yml b/tests/multi-runner-orchestration/docker-compose.yml new file mode 100644 index 0000000000..c5c3c8480d --- /dev/null +++ b/tests/multi-runner-orchestration/docker-compose.yml @@ -0,0 +1,193 @@ +services: + github-api: + build: . + pull_policy: build + platform: linux/amd64 + ports: + - "8080:8080" + - "8443:8443" + ministack: + image: ghcr.io/ministackorg/ministack:v1.5.30@sha256:9019a8ab2531163fc6c3119a277a181faadcd2cde8fd20272dc8ec0ccda154b9 + platform: linux/amd64 + ports: + - "4566:4566" + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + MINISTACK_ACCOUNT_ID: "000000000000" + MINISTACK_REGION: eu-west-1 + DOCKER_NETWORK: multi-runner-orchestration_default + aws-fixtures: + image: amazon/aws-cli:2.27.41@sha256:bc6b7bba44ce38f9604ede49c584824af919047ea03fbcc7c7610671fdef95d8 + platform: linux/amd64 + depends_on: + ministack: + condition: service_started + entrypoint: ["/usr/bin/python"] + command: ["/workspace/tests/multi-runner-orchestration/fixtures/seed_ministack_fixtures.py", "--manifest", "/workspace/tests/multi-runner-orchestration/fixtures/ministack-fixtures.json", "--type", "ami", "--type", "ecr-repository"] + volumes: + - ../..:/workspace:ro + environment: + AWS_ACCESS_KEY_ID: "000000000000" + AWS_SECRET_ACCESS_KEY: test-only + AWS_DEFAULT_REGION: eu-west-1 + AWS_EC2_METADATA_DISABLED: "true" + MINISTACK_ENDPOINT_URL: http://ministack:4566 + lambda-builder: + image: docker:27-cli + platform: linux/amd64 + working_dir: /workspace + entrypoint: ["/bin/sh", "-ec"] + command: + - | + apk add --no-cache bash git + cd /workspace + git config --global --add safe.directory /workspace + OUTPUT_DIR=/workspace/lambda_output bash .ci/build.sh + volumes: + - ../..:/workspace + - /var/run/docker.sock:/var/run/docker.sock + environment: + DOCKER_HOST: unix:///var/run/docker.sock + docker-buildx: + image: docker:27-cli + platform: linux/amd64 + depends_on: + ministack: + condition: service_started + aws-fixtures: + condition: service_completed_successfully + working_dir: /workspace + entrypoint: ["/bin/sh", "-ec"] + command: + - | + apk add --no-cache aws-cli + ecr_password="$$(aws ecr get-login-password --endpoint-url "$${AWS_ENDPOINT_URL}" --region "$${AWS_DEFAULT_REGION}")" + if [ -z "$$ecr_password" ]; then + echo "MiniStack returned an empty ECR login password" >&2 + exit 1 + fi + ministack_ip="$$(getent hosts ministack | awk 'NR == 1 { print $1 }')" + if [ -z "$$ministack_ip" ]; then + echo "Could not resolve MiniStack on the Compose network" >&2 + exit 1 + fi + docker_arch="$$(docker info --format '{{.Architecture}}')" + case "$$docker_arch" in + aarch64|arm64) scale_set_platform=linux/arm64 ;; + x86_64|amd64) scale_set_platform=linux/amd64 ;; + *) echo "Unsupported Docker host architecture: $$docker_arch" >&2; exit 1 ;; + esac + export MINISTACK_REGISTRY="$$ministack_ip:4566" + docker_config="$$(mktemp -d)" + export DOCKER_CONFIG="$$docker_config" + cp /workspace/tests/multi-runner-orchestration/buildkitd.toml "$$DOCKER_CONFIG/buildkitd.toml" + cat >> "$$DOCKER_CONFIG/buildkitd.toml" < "$$DOCKER_CONFIG/config.json" + docker buildx rm runner-smoke-builder >/dev/null 2>&1 || true + trap 'docker buildx rm runner-smoke-builder >/dev/null 2>&1 || true; rm -rf "$$DOCKER_CONFIG"' EXIT + docker buildx create \ + --name runner-smoke-builder \ + --driver docker-container \ + --driver-opt network=multi-runner-orchestration_default \ + --buildkitd-config "$$DOCKER_CONFIG/buildkitd.toml" \ + --use --bootstrap + docker buildx build --builder runner-smoke-builder \ + --network host --allow network.host \ + --platform "$$scale_set_platform" \ + --target runtime \ + --file /workspace/lambdas/services/scale-set/Dockerfile \ + --tag "$${MINISTACK_REGISTRY}/scale-set-controller:latest" \ + --provenance=false --push /workspace + volumes: + - ../..:/workspace:ro + - /var/run/docker.sock:/var/run/docker.sock + environment: + DOCKER_HOST: unix:///var/run/docker.sock + AWS_ACCESS_KEY_ID: "000000000000" + AWS_SECRET_ACCESS_KEY: test-only + AWS_DEFAULT_REGION: eu-west-1 + AWS_EC2_METADATA_DISABLED: "true" + AWS_ENDPOINT_URL: http://ministack:4566 + DOCKER_BUILDKIT: 1 + terraform-apply: + image: hashicorp/terraform:1.17 + depends_on: + github-api: + condition: service_started + ministack: + condition: service_started + aws-fixtures: + condition: service_completed_successfully + lambda-builder: + condition: service_completed_successfully + docker-buildx: + condition: service_completed_successfully + working_dir: /workspace + entrypoint: ["/bin/sh", "-ec"] + command: + - | + host_arch=$$(uname -m) + case "$$host_arch" in + aarch64|arm64) scale_set_arch=ARM64 ;; + x86_64|amd64) scale_set_arch=X86_64 ;; + *) echo "Unsupported host architecture: $$host_arch" >&2; exit 1 ;; + esac + sed "s/architecture = \".*\"/architecture = \"$$scale_set_arch\"/" \ + ./tests/multi-runner-orchestration/fixtures/multi-runner-orchestration.tfvars \ + > /tmp/multi-runner-orchestration.tfvars + terraform -chdir=./examples/multi-runner-orchestration init + terraform -chdir=./examples/multi-runner-orchestration apply -auto-approve \ + -var-file=/tmp/multi-runner-orchestration.tfvars + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - ../..:/workspace + environment: + AWS_ACCESS_KEY_ID: "000000000000" + AWS_SECRET_ACCESS_KEY: test-only + AWS_DEFAULT_REGION: eu-west-1 + AWS_REGION: eu-west-1 + AWS_EC2_METADATA_DISABLED: "true" + AWS_ENDPOINT_URL: http://ministack:4566 + TF_IN_AUTOMATION: "true" + TF_INPUT: "false" + smoke-tests: + image: hashicorp/terraform:1.17 + platform: linux/amd64 + depends_on: + terraform-apply: + condition: service_completed_successfully + working_dir: /workspace + entrypoint: ["/bin/sh", "-ec"] + command: + - | + terraform -chdir=./examples/multi-runner-orchestration init + terraform -chdir=./examples/multi-runner-orchestration output + apk add --no-cache aws-cli nodejs + node tests/multi-runner-orchestration/smoke/prepare-scale-set-credentials.mjs + node --test --test-concurrency=1 \ + --test-reporter=./tests/multi-runner-orchestration/smoke/reporter.mjs \ + tests/multi-runner-orchestration/smoke/*.test.mjs + volumes: + - ../..:/workspace + environment: + AWS_ACCESS_KEY_ID: "000000000000" + AWS_SECRET_ACCESS_KEY: test-only + AWS_DEFAULT_REGION: eu-west-1 + AWS_REGION: eu-west-1 + AWS_EC2_METADATA_DISABLED: "true" + AWS_ENDPOINT_URL: http://ministack:4566 + TF_IN_AUTOMATION: "true" + TF_INPUT: "false" + +networks: + default: + name: multi-runner-orchestration_default diff --git a/tests/multi-runner-orchestration/fixtures/ministack-fixtures.json b/tests/multi-runner-orchestration/fixtures/ministack-fixtures.json new file mode 100644 index 0000000000..7bf7476e55 --- /dev/null +++ b/tests/multi-runner-orchestration/fixtures/ministack-fixtures.json @@ -0,0 +1,38 @@ +{ + "fixtures": [ + { + "type": "ami", + "name": "multi-runner-orchestration", + "description": "MiniStack test-only AMI", + "architecture": "x86_64", + "root_device_name": "/dev/xvda", + "virtualization_type": "hvm", + "image_location": "alpine:3.20" + }, + { + "type": "ecr-repository", + "name": "scale-set-controller", + "image_tag_mutability": "MUTABLE", + "image_scanning_configuration": { + "scanOnPush": false + }, + "policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowAccountPull", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::000000000000:root" + }, + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer" + ] + } + ] + } + } + ] +} diff --git a/tests/multi-runner-orchestration/fixtures/multi-runner-orchestration.tfvars b/tests/multi-runner-orchestration/fixtures/multi-runner-orchestration.tfvars new file mode 100644 index 0000000000..e5025654ad --- /dev/null +++ b/tests/multi-runner-orchestration/fixtures/multi-runner-orchestration.tfvars @@ -0,0 +1,48 @@ +aws_region = "eu-west-1" +environment = "multi-runner-webhook" + +runners_lambda_zip = "../../lambda_output/runners.zip" +webhook_lambda_zip = "../../lambda_output/webhook.zip" + +github = { + url = "https://github-api:8443" + ssl_verify = false + runner_owner = "example" + registration_level = "organization" +} + +scale_set = { + name = "linux-scale-set" + runner_group_name = "experimental-euw1-sl-cicd-forge-emu" + min_runners = 1 + container = { + image = "localhost:4566/scale-set-controller:latest" + } + ecs = { + task = { + architecture = "ARM64" + } + } +} + +github_app = { + id = "123" + installation_id = "123" + key_base64 = "LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRRFM3OUJ1dU9rVTJ2V2EKT3J2aHBWNENON1g3dm5qdXBuWlhSZFNvWGVQNWxKMEs4dHhrcnpIaEVyT3JobGJZd1JoUEJjaFFBVzJtemQ4RgpqNytTVGliL0VFd1ZXZVFWUWZMbTFwMDRyR3FNc1B0WkcvcVFWelNwZTkyRWhYTnZzWFRmS0c1UjNYTE5OdS9uCm9PNmRablJvM2U0TENYd1BaMGdMR3Q3K1hrZk9QQU5LZDZnSW1pS1hiQXAxV1VzK0JrdXI1TTBXNjgvWjd0WlgKYnBTcHV3MWVwc2RwM2FpYzM2V2FmVWloZDNMQUR5UFE5c1JkZ2E2VXBaRFJTUEE3VWlUd3FwTlluY2ZEODFkKwpFWnI3ZlNEcjRqb2s5S0F1SzRQWTBML3lMVGlBVEg1aWFKcWhxT252UDN4MFBOSzd4aEw4SHZpVThON0dqNHlhClQ2eDdheC90QWdNQkFBRUNnZ0VBTGZ0SmdpMXNYOWU0Vk1uTFpZMnRIOG9oc1ZKTTNlY3JJbXl1UmhLTGpQODAKdWF1RjJDdnBzV0RvaWpKRGNjWXNCS1BqemU3MXhuZ1h6d2hPTURnNW4zM2Z2QUxIcWN1aHhlalFSb2V0UldTYQpNZ3RibEVYeVRLVGIvZFN5ZkkxZ1J2b3pERVQ4RHRpVU5QZTd0Rk9lZWpLd3ZyYXZWTk5Fbk9rYlpiTkxMMTAzCndtZS9ObDBjTDhVN1lHaGtLRm0wMmZjTVpaWENiT3puYzQ1UFFXNmVJaUhJWW1sZ056Q2lIenRvZS9UZ3JKekoKS1dsMzU5WUM4VGZZd3AydDQ5ZTVSS3BKN0lRY3A5NUEvVFBHVXVHaEMxRkZEZlh0eUgySlYxRGJOQ3RhVjNRcwp3M3g1LzErZDRoN0dxUjFzTW01V0NMUmduSjhYOU0xRm5IMlAvYzAxK1FLQmdRRDBBcWFPQkJ0ay81WFl4YnowCmM1R25iOXdDa0xGaVp1WElGbnV0d0JTaEprU055Q3NFbDB5UHBrV0NnTk1zQ0xtaUNldGpSUm0zMFZXSC9jeUMKekxxb0FXNU5VTmorK3dmZHcrM1hHODlkVjEzRWxLL1VKNDE0RkR4NFZ4NE4rTWtHaHVkZno2dXdUbFVJYXFDVAo0czIrSko4NDBraUIxRUUycFBoYmd4b2c5UUtCZ1FEZFRTT0QxcEl1b29YR0pBQTZxWUVoMWtOb1g1Q0IzOE9TCk92TThwTFV1djN6cmp1QzdQbm5CZEF5aEdGeG9qcFlQbG1pNFIrWDJraGVPd0ZscTM2V3hDOERHeEN2L0h2ODMKV0lLdTc4c25MQmlLMlBzdk9vdmNZZ1BRVVBkZWV4ODkwcU92Tjh1MzV0NjZuNnliOVlCaWNSd01tVFlJS2Irdwp4ZmEzenBWSUdRS0JnRTZaOS9yeFdiMjhFeElFQ0cwYU9VWWRZMXEwTDEvS1U2U1RrWDJUelZGbjJWZHVybVRKCk5CRmw5eWdseFdCKzR3Vk9lck84azU5QjRISnhFQXJtVmc0K2Y4Q0tGWXgxWXNCT0YvUmQ4ZnRZWkUwcHRFWDgKQTdjMWxaRVNSdzBySWQ1bllHcmRvNm1rWWdIWER5ajBPMkJQdmJaNURvQTcxbEg2N0VIV1JubHRBb0dBVUlTeAppRlY0eUpqL2ZWRTUrQjRXU3Q1dmxqazNGZG41aVdvRGVDd2hkQW5lQk1MdWg1STZzS1Z0dHV0cksrdTlNNXl4CksyT1Zpem9xY2hrUUw2K0IxTUlHK1FzRlN3VFhMU2I2SFhaSC8wK3UrazNxVG92SnVYeThFSDUxRXA1UGhxbnUKUG1GWnNPYkNZVHFaamdFc0hpVXpaRzRXVjdBQSsrdDdVZk5MSTNrQ2dZRUFra1VsV3JxdG1rTFV2TlZ4d0tMZwo5eStWcTRrRGRSR2hta3VhQmNFK0NkOW14bVpDbzFMeHU2bWxJRHlvQzV5TVpLVjlNQndOeW5kQTc4UkIxNEtaCkc4aHRCazROd2FPUWlZYmRLbzJMQ2FVSWdyVlorTk1SNHZvU0NNeThHNG5TeUhRWmJ1b1lXdWU1c3JUSFdEcW8KNElaOVJyRXd5ODR4SHl5RE0xVGN1VG89Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K" + webhook_secret = "ministack-webhook-secret" +} + +compute_provider = { + aws = { + ec2 = { + instance_types = ["m7a.large", "m5.large"] + ami = { + filter = { + name = ["multi-runner-orchestration"] + state = ["available"] + } + owners = ["self"] + } + } + } +} diff --git a/tests/multi-runner-orchestration/fixtures/seed_ministack_fixtures.py b/tests/multi-runner-orchestration/fixtures/seed_ministack_fixtures.py new file mode 100644 index 0000000000..e61c118362 --- /dev/null +++ b/tests/multi-runner-orchestration/fixtures/seed_ministack_fixtures.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python +"""Seed declared fixtures into a running MiniStack instance.""" + +from __future__ import print_function + +import argparse +import json +import os +import subprocess +import sys +import time + +try: + from urllib.request import urlopen +except ImportError: # Python 2.7 in the AWS CLI container image. + from urllib2 import URLError, urlopen +else: + from urllib.error import URLError + +def aws(*args): + endpoint = os.environ.get( + "MINISTACK_ENDPOINT_URL", + os.environ.get("AWS_ENDPOINT_URL", "http://localhost:4566"), + ) + region = os.environ.get("AWS_DEFAULT_REGION", "eu-west-1") + command = ["aws", "--endpoint-url", endpoint, "--region", region] + list(args) + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if process.returncode: + output = (stdout or b"") + (stderr or b"") + if hasattr(output, "decode"): + output = output.decode("utf-8") + sys.stderr.write(output) + raise subprocess.CalledProcessError(process.returncode, command, output) + if hasattr(stdout, "decode"): + stdout = stdout.decode("utf-8") + return stdout.strip() + + +def wait_for_ministack(attempts=60): + endpoint = os.environ.get( + "MINISTACK_ENDPOINT_URL", + os.environ.get("AWS_ENDPOINT_URL", "http://localhost:4566"), + ) + health_url = endpoint.rstrip("/") + "/_ministack/health" + for unused in range(attempts): + try: + response = urlopen(health_url, timeout=2) + response.close() + print("MiniStack is ready at {0}".format(endpoint), file=sys.stderr) + return + except (URLError, IOError): + time.sleep(1) + raise RuntimeError("MiniStack did not become ready at {0}".format(endpoint)) + + +def seed_ami(fixture): + name = fixture["name"] + ami_id = aws( + "ec2", + "describe-images", + "--owners", + "self", + "--filters", + "Name=name,Values={0}".format(name), + "Name=state,Values=available", + "--query", + "Images[0].ImageId", + "--output", + "text", + ) + if ami_id != "None": + print("Fixture AMI already exists: {0} ({1})".format(name, ami_id), file=sys.stderr) + return None + + ami_id = aws( + "ec2", + "register-image", + "--name", + name, + "--description", + fixture.get("description", "MiniStack test-only AMI"), + "--architecture", + fixture["architecture"], + "--root-device-name", + fixture.get("root_device_name", "/dev/xvda"), + "--virtualization-type", + fixture.get("virtualization_type", "hvm"), + "--image-location", + fixture["image_location"], + "--query", + "ImageId", + "--output", + "text", + ) + print("Created fixture AMI: {0} ({1})".format(name, ami_id), file=sys.stderr) + return ami_id + + +def seed_ecr_repository(fixture): + name = fixture["name"] + try: + aws( + "ecr", + "create-repository", + "--repository-name", + name, + "--image-tag-mutability", + fixture.get("image_tag_mutability", "IMMUTABLE"), + "--image-scanning-configuration", + "scanOnPush={0}".format(str(fixture.get("image_scanning_configuration", {}).get("scanOnPush", False)).lower()), + "--output", + "json", + ) + print("Created fixture ECR repository: {0}".format(name), file=sys.stderr) + except subprocess.CalledProcessError as error: + output = error.output or "" + if hasattr(output, "decode"): + output = output.decode("utf-8") + if "RepositoryAlreadyExistsException" not in output: + raise + print("Fixture ECR repository already exists: {0}".format(name), file=sys.stderr) + + policy = fixture.get("policy") + if policy: + aws( + "ecr", + "set-repository-policy", + "--repository-name", + name, + "--policy-text", + json.dumps(policy, separators=(",", ":")), + ) + print("Applied fixture ECR repository policy: {0}".format(name), file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + default_manifest = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ministack-fixtures.json") + parser.add_argument("--manifest", default=default_manifest) + parser.add_argument("--endpoint-url", help="MiniStack endpoint override") + parser.add_argument("--region", help="AWS region override") + parser.add_argument( + "--type", + dest="fixture_types", + action="append", + help="Seed only fixtures of this type; may be repeated", + ) + parser.add_argument( + "--name", + dest="fixture_names", + action="append", + help="Seed only fixtures with this name; may be repeated", + ) + parser.add_argument( + "--created-ids", + action="store_true", + help="Print IDs of fixtures created during this run, for caller cleanup", + ) + args = parser.parse_args() + + if args.endpoint_url: + os.environ["MINISTACK_ENDPOINT_URL"] = args.endpoint_url + if args.region: + os.environ["AWS_DEFAULT_REGION"] = args.region + + wait_for_ministack() + with open(args.manifest, "r") as manifest_file: + manifest = json.load(manifest_file) + fixtures = manifest.get("fixtures", []) + selected = [] + for fixture in fixtures: + fixture_type = fixture.get("type") + if args.fixture_types and fixture_type not in args.fixture_types: + continue + if args.fixture_names and fixture.get("name") not in args.fixture_names: + continue + selected.append(fixture) + + created_ids = [] + for fixture in selected: + fixture_type = fixture.get("type") + if fixture_type == "ami": + created_id = seed_ami(fixture) + if created_id: + created_ids.append(created_id) + elif fixture_type == "ecr-repository": + seed_ecr_repository(fixture) + else: + raise ValueError("Unsupported MiniStack fixture type: {0!r}".format(fixture_type)) + + if args.created_ids: + print("\n".join(created_ids)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/multi-runner-orchestration/mappings/scale-set.ec2.actions-registration-handshake.json b/tests/multi-runner-orchestration/mappings/scale-set.ec2.actions-registration-handshake.json new file mode 100644 index 0000000000..5db902bf74 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/scale-set.ec2.actions-registration-handshake.json @@ -0,0 +1,14 @@ +{ + "name": "Scale set - Actions registration handshake", + "request": { + "method": "POST", + "urlPathPattern": "(?:/api/v3)?/actions/runner-registration" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "body": "{\"url\":\"{{request.baseUrl}}/tenant/123\",\"token\":\"eyJhbGciOiJub25lIn0.eyJleHAiOjQxMDI0NDgwMDB9.mock-signature\"}" + } +} diff --git a/tests/multi-runner-orchestration/mappings/scale-set.ec2.close-message-session.json b/tests/multi-runner-orchestration/mappings/scale-set.ec2.close-message-session.json new file mode 100644 index 0000000000..3f1ad50f49 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/scale-set.ec2.close-message-session.json @@ -0,0 +1,10 @@ +{ + "name": "Scale set - close message session", + "request": { + "method": "DELETE", + "urlPathPattern": "(?:/api/v3)?(?:/tenant/[^/]+)?/_apis/runtime/runnerscalesets/\\d+/sessions/[^/]+" + }, + "response": { + "status": 204 + } +} diff --git a/tests/multi-runner-orchestration/mappings/scale-set.ec2.create-message-session.json b/tests/multi-runner-orchestration/mappings/scale-set.ec2.create-message-session.json new file mode 100644 index 0000000000..43c4dee6e5 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/scale-set.ec2.create-message-session.json @@ -0,0 +1,14 @@ +{ + "name": "Scale set - create message session", + "request": { + "method": "POST", + "urlPathPattern": "(?:/api/v3)?(?:/tenant/[^/]+)?/_apis/runtime/runnerscalesets/\\d+/sessions" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "body": "{\"sessionId\":\"11111111-1111-1111-1111-111111111111\",\"ownerName\":\"local.linux-scale-set\",\"runnerScaleSet\":{\"id\":223,\"name\":\"linux-scale-set\",\"runnerGroupId\":48},\"messageQueueUrl\":\"{{request.baseUrl}}/messages?sessionId=11111111-1111-1111-1111-111111111111&api-version=6.0-preview\",\"messageQueueAccessToken\":\"fake-queue-token\",\"statistics\":{\"totalAvailableJobs\":0,\"totalAcquiredJobs\":0,\"totalAssignedJobs\":0,\"totalRunningJobs\":0,\"totalRegisteredRunners\":0,\"totalBusyRunners\":0,\"totalIdleRunners\":0}}" + } +} diff --git a/tests/multi-runner-orchestration/mappings/scale-set.ec2.generate-jit-runner-config.json b/tests/multi-runner-orchestration/mappings/scale-set.ec2.generate-jit-runner-config.json new file mode 100644 index 0000000000..92c1972253 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/scale-set.ec2.generate-jit-runner-config.json @@ -0,0 +1,14 @@ +{ + "name": "Scale set - generate JIT runner config", + "request": { + "method": "POST", + "urlPathPattern": "(?:/api/v3)?(?:/tenant/[^/]+)?/_apis/runtime/runnerscalesets/\\d+/generatejitconfig" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "body": "{\"runner\":{\"id\":321,\"name\":\"{{jsonPath request.body '$.name'}}\",\"runnerScaleSetId\":223},\"encodedJITConfig\":\"test-only-jit-configuration\"}" + } +} diff --git a/tests/multi-runner-orchestration/mappings/scale-set.ec2.get-scale-set-by-id.json b/tests/multi-runner-orchestration/mappings/scale-set.ec2.get-scale-set-by-id.json new file mode 100644 index 0000000000..93505b73d9 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/scale-set.ec2.get-scale-set-by-id.json @@ -0,0 +1,35 @@ +{ + "name": "Scale set - get scale set by id", + "request": { + "method": "GET", + "urlPathPattern": "(?:/api/v3)?(?:/tenant/[^/]+)?/_apis/runtime/runnerscalesets/\\d+" + }, + "response": { + "status": 200, + "jsonBody": { + "id": 223, + "name": "linux-scale-set", + "runnerGroupId": 48, + "runnerGroupName": "experimental-euw1-sl-cicd-forge-emu", + "labels": [ + { + "name": "linux-scale-set", + "type": "system" + } + ], + "runnerSetting": {}, + "statistics": { + "totalAvailableJobs": 0, + "totalAcquiredJobs": 0, + "totalAssignedJobs": 0, + "totalRunningJobs": 0, + "totalRegisteredRunners": 0, + "totalBusyRunners": 0, + "totalIdleRunners": 0 + } + }, + "headers": { + "Content-Type": "application/json" + } + } +} diff --git a/tests/multi-runner-orchestration/mappings/scale-set.ec2.list-runner-groups.json b/tests/multi-runner-orchestration/mappings/scale-set.ec2.list-runner-groups.json new file mode 100644 index 0000000000..c6d288c70c --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/scale-set.ec2.list-runner-groups.json @@ -0,0 +1,14 @@ +{ + "name": "Scale set - list runner groups", + "request": { + "method": "GET", + "urlPathPattern": "(?:/api/v3)?(?:/tenant/[^/]+)?/_apis/runtime/runnergroups/?" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "body": "{\"count\":1,\"value\":[{\"id\":48,\"name\":\"experimental-euw1-sl-cicd-forge-emu\",\"size\":100,\"isDefaultGroup\":false}]}" + } +} diff --git a/tests/multi-runner-orchestration/mappings/scale-set.ec2.list-runners-by-name.json b/tests/multi-runner-orchestration/mappings/scale-set.ec2.list-runners-by-name.json new file mode 100644 index 0000000000..0e02ccbe7e --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/scale-set.ec2.list-runners-by-name.json @@ -0,0 +1,14 @@ +{ + "name": "Scale set - list runners by name", + "request": { + "method": "GET", + "urlPathPattern": "(?:/api/v3)?(?:/tenant/[^/]+)?/_apis/distributedtask/pools/0/agents" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "body": "{\"count\":1,\"value\":[{\"id\":321,\"name\":\"{{request.query.agentName}}\",\"runnerScaleSetId\":223}]}" + } +} diff --git a/tests/multi-runner-orchestration/mappings/scale-set.ec2.list-scale-sets.json b/tests/multi-runner-orchestration/mappings/scale-set.ec2.list-scale-sets.json new file mode 100644 index 0000000000..5e2ab4c110 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/scale-set.ec2.list-scale-sets.json @@ -0,0 +1,40 @@ +{ + "name": "Scale set - list scale sets", + "request": { + "method": "GET", + "urlPathPattern": "(?:/api/v3)?(?:/tenant/[^/]+)?/_apis/runtime/runnerscalesets" + }, + "response": { + "status": 200, + "jsonBody": { + "count": 1, + "value": [ + { + "id": 223, + "name": "linux-scale-set", + "runnerGroupId": 48, + "runnerGroupName": "experimental-euw1-sl-cicd-forge-emu", + "labels": [ + { + "name": "linux-scale-set", + "type": "system" + } + ], + "runnerSetting": {}, + "statistics": { + "totalAvailableJobs": 0, + "totalAcquiredJobs": 0, + "totalAssignedJobs": 0, + "totalRunningJobs": 0, + "totalRegisteredRunners": 0, + "totalBusyRunners": 0, + "totalIdleRunners": 0 + } + } + ] + }, + "headers": { + "Content-Type": "application/json" + } + } +} diff --git a/tests/multi-runner-orchestration/mappings/scale-set.ec2.poll-empty-message-queue.json b/tests/multi-runner-orchestration/mappings/scale-set.ec2.poll-empty-message-queue.json new file mode 100644 index 0000000000..c47d8ae6a1 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/scale-set.ec2.poll-empty-message-queue.json @@ -0,0 +1,11 @@ +{ + "name": "Scale set - poll message queue empty", + "request": { + "method": "GET", + "urlPathPattern": "(?:/queue/sessions/[^/]+/messages|/messages)" + }, + "response": { + "status": 202, + "fixedDelayMilliseconds": 1000 + } +} diff --git a/tests/multi-runner-orchestration/mappings/scale-set.ec2.remove-runner.json b/tests/multi-runner-orchestration/mappings/scale-set.ec2.remove-runner.json new file mode 100644 index 0000000000..85b4222a37 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/scale-set.ec2.remove-runner.json @@ -0,0 +1,10 @@ +{ + "name": "Scale set - remove runner", + "request": { + "method": "DELETE", + "urlPathPattern": "(?:/api/v3)?(?:/tenant/[^/]+)?/_apis/distributedtask/pools/0/agents/\\d+" + }, + "response": { + "status": 204 + } +} diff --git a/tests/multi-runner-orchestration/mappings/scale-set.ec2.update-scale-set.json b/tests/multi-runner-orchestration/mappings/scale-set.ec2.update-scale-set.json new file mode 100644 index 0000000000..2a1afdb1ff --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/scale-set.ec2.update-scale-set.json @@ -0,0 +1,35 @@ +{ + "name": "Scale set - update scale set", + "request": { + "method": "PATCH", + "urlPathPattern": "(?:/api/v3)?(?:/tenant/[^/]+)?/_apis/runtime/runnerscalesets/\\d+" + }, + "response": { + "status": 200, + "jsonBody": { + "id": 223, + "name": "linux-scale-set", + "runnerGroupId": 48, + "runnerGroupName": "experimental-euw1-sl-cicd-forge-emu", + "labels": [ + { + "name": "linux-scale-set", + "type": "system" + } + ], + "runnerSetting": {}, + "statistics": { + "totalAvailableJobs": 0, + "totalAcquiredJobs": 0, + "totalAssignedJobs": 0, + "totalRunningJobs": 0, + "totalRegisteredRunners": 0, + "totalBusyRunners": 0, + "totalIdleRunners": 0 + } + }, + "headers": { + "Content-Type": "application/json" + } + } +} diff --git a/tests/multi-runner-orchestration/mappings/webhook.ec2.delete-organization-runner.json b/tests/multi-runner-orchestration/mappings/webhook.ec2.delete-organization-runner.json new file mode 100644 index 0000000000..7d16c49cf0 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/webhook.ec2.delete-organization-runner.json @@ -0,0 +1,10 @@ +{ + "name": "Runner - delete org runner", + "request": { + "method": "DELETE", + "urlPathPattern": "(?:/api/v3)?/orgs/[^/]+/actions/runners/\\d+" + }, + "response": { + "status": 204 + } +} diff --git a/tests/multi-runner-orchestration/mappings/webhook.ec2.generate-org-jit-runner-config.json b/tests/multi-runner-orchestration/mappings/webhook.ec2.generate-org-jit-runner-config.json new file mode 100644 index 0000000000..6b08f8fda5 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/webhook.ec2.generate-org-jit-runner-config.json @@ -0,0 +1,20 @@ +{ + "name": "Runner - org JIT config", + "request": { + "method": "POST", + "urlPathPattern": "(?:/api/v3)?/orgs/[^/]+/actions/runners/generate-jitconfig" + }, + "response": { + "status": 200, + "jsonBody": { + "runner": { + "id": 4242, + "name": "wiremock-runner-001" + }, + "encoded_jit_config": "eyJtb2NrIjoiaml0LWNvbmZpZyJ9" + }, + "headers": { + "Content-Type": "application/json" + } + } +} diff --git a/tests/multi-runner-orchestration/mappings/webhook.ec2.get-organization-installation.json b/tests/multi-runner-orchestration/mappings/webhook.ec2.get-organization-installation.json new file mode 100644 index 0000000000..c11b2ce83b --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/webhook.ec2.get-organization-installation.json @@ -0,0 +1,21 @@ +{ + "name": "GitHub App - get organization installation", + "request": { + "method": "GET", + "urlPathPattern": "(?:/api/v3)?/orgs/[^/]+/installation" + }, + "response": { + "status": 200, + "jsonBody": { + "id": 123, + "account": { + "login": "test-owner", + "type": "Organization" + }, + "target_type": "Organization" + }, + "headers": { + "Content-Type": "application/json" + } + } +} diff --git a/tests/multi-runner-orchestration/mappings/webhook.ec2.get-organization-runner.json b/tests/multi-runner-orchestration/mappings/webhook.ec2.get-organization-runner.json new file mode 100644 index 0000000000..8eb38eef9b --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/webhook.ec2.get-organization-runner.json @@ -0,0 +1,13 @@ +{ + "name": "runner get org runner", + "request": { + "method": "GET", + "urlPathPattern": "(?:/api/v3)?/orgs/[^/]+/actions/runners/\\d+" + }, + "response": { + "status": 404, + "jsonBody": { + "message": "Runner not found" + } + } +} diff --git a/tests/multi-runner-orchestration/mappings/webhook.ec2.github-app-installation-token.json b/tests/multi-runner-orchestration/mappings/webhook.ec2.github-app-installation-token.json new file mode 100644 index 0000000000..606fdc125d --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/webhook.ec2.github-app-installation-token.json @@ -0,0 +1,22 @@ +{ + "name": "GitHub App - create installation access token", + "request": { + "method": "POST", + "urlPathPattern": "(?:/api/v3)?/app/installations/\\d+/access_tokens" + }, + "response": { + "status": 201, + "jsonBody": { + "token": "ghs_wiremock_installation_token", + "expires_at": "2099-01-01T00:00:00Z", + "permissions": { + "actions": "write", + "contents": "read", + "metadata": "read" + } + }, + "headers": { + "Content-Type": "application/json" + } + } +} diff --git a/tests/multi-runner-orchestration/mappings/webhook.ec2.list-organization-runner-groups.json b/tests/multi-runner-orchestration/mappings/webhook.ec2.list-organization-runner-groups.json new file mode 100644 index 0000000000..c0b2b5af98 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/webhook.ec2.list-organization-runner-groups.json @@ -0,0 +1,24 @@ +{ + "name": "Runner - org runner groups", + "request": { + "method": "GET", + "urlPathPattern": "(?:/api/v3)?/orgs/[^/]+/actions/runner-groups" + }, + "response": { + "status": 200, + "jsonBody": { + "total_count": 1, + "runner_groups": [ + { + "id": 1, + "name": "experimental-euw1-sl-cicd-forge-emu", + "size": 100, + "isDefaultGroup": false + } + ] + }, + "headers": { + "Content-Type": "application/json" + } + } +} diff --git a/tests/multi-runner-orchestration/mappings/webhook.ec2.list-organization-runners.json b/tests/multi-runner-orchestration/mappings/webhook.ec2.list-organization-runners.json new file mode 100644 index 0000000000..3c75201834 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/webhook.ec2.list-organization-runners.json @@ -0,0 +1,37 @@ +{ + "name": "Runner - list org runners", + "request": { + "method": "GET", + "urlPathPattern": "(?:/api/v3)?/orgs/[^/]+/actions/runners" + }, + "response": { + "status": 200, + "jsonBody": { + "total_count": 1, + "runners": [ + { + "id": 4242, + "name": "wiremock-runner-001", + "os": "Linux", + "status": "online", + "busy": false, + "labels": [ + { + "id": 1, + "name": "self-hosted", + "type": "read-only" + }, + { + "id": 2, + "name": "Linux", + "type": "read-only" + } + ] + } + ] + }, + "headers": { + "Content-Type": "application/json" + } + } +} diff --git a/tests/multi-runner-orchestration/mappings/webhook.ec2.organization-registration-token.json b/tests/multi-runner-orchestration/mappings/webhook.ec2.organization-registration-token.json new file mode 100644 index 0000000000..2f74c75c32 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/webhook.ec2.organization-registration-token.json @@ -0,0 +1,17 @@ +{ + "name": "Runner - org registration token", + "request": { + "method": "POST", + "urlPathPattern": "(?:/api/v3)?/orgs/[^/]+/actions/runners/registration-token" + }, + "response": { + "status": 201, + "jsonBody": { + "token": "runner-registration-token-mock", + "expires_at": "2099-01-01T00:00:00Z" + }, + "headers": { + "Content-Type": "application/json" + } + } +} diff --git a/tests/multi-runner-orchestration/mappings/webhook.ec2.workflow-job-details.json b/tests/multi-runner-orchestration/mappings/webhook.ec2.workflow-job-details.json new file mode 100644 index 0000000000..b031a2ef78 --- /dev/null +++ b/tests/multi-runner-orchestration/mappings/webhook.ec2.workflow-job-details.json @@ -0,0 +1,27 @@ +{ + "name": "Runner - repo workflow job", + "request": { + "method": "GET", + "urlPathPattern": "(?:/api/v3)?/repos/[^/]+/[^/]+/actions/jobs/\\d+" + }, + "response": { + "status": 200, + "jsonBody": { + "id": 5150, + "run_id": 8080, + "status": "queued", + "conclusion": null, + "name": "wiremock-job", + "started_at": null, + "completed_at": null, + "html_url": "https://github.com/acme/repo/actions/runs/8080/job/5150", + "labels": [ + "self-hosted", + "Linux" + ] + }, + "headers": { + "Content-Type": "application/json" + } + } +} diff --git a/tests/multi-runner-orchestration/smoke/prepare-scale-set-credentials.mjs b/tests/multi-runner-orchestration/smoke/prepare-scale-set-credentials.mjs new file mode 100644 index 0000000000..b4bc79f821 --- /dev/null +++ b/tests/multi-runner-orchestration/smoke/prepare-scale-set-credentials.mjs @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { aws } from './support.mjs'; +import { findScaleSetControllerService } from './scale-set-support.mjs'; + +const cluster = process.env.MINISTACK_SCALE_SET_CLUSTER + ?? 'multi-runner-webhook-scale-set'; +const controllerService = await findScaleSetControllerService(cluster); +const taskDefinition = structuredClone(controllerService.taskDefinition); + +for (const field of [ + 'taskDefinitionArn', + 'revision', + 'status', + 'requiresAttributes', + 'compatibilities', + 'registeredAt', + 'registeredBy', +]) { + delete taskDefinition[field]; +} + +const controller = taskDefinition.containerDefinitions?.find( + (definition) => definition.name === 'scale-set-controller', +); +assert.ok(controller, 'Task definition has no scale-set-controller container'); + +const environment = new Map( + (controller.environment ?? []) + .filter(({ name }) => name !== 'AWS_ACCESS_KEY_ID' && name !== 'AWS_SECRET_ACCESS_KEY') + .map(({ name, value }) => [name, value]), +); +environment.set('AWS_ACCESS_KEY_ID', '000000000000'); +environment.set('AWS_SECRET_ACCESS_KEY', 'test-only'); +controller.environment = [...environment].map(([name, value]) => ({ name, value })); + +const temporaryDirectory = await mkdtemp(path.join(tmpdir(), 'scale-set-credentials-')); +try { + const inputFile = path.join(temporaryDirectory, 'task-definition.json'); + await writeFile(inputFile, JSON.stringify(taskDefinition)); + + const registration = await aws( + 'ecs', 'register-task-definition', + '--cli-input-json', `file://${inputFile}`, + ); + const taskDefinitionArn = registration.taskDefinition?.taskDefinitionArn; + assert.ok(taskDefinitionArn, 'MiniStack did not register the credential-enabled task definition'); + + await aws( + 'ecs', 'update-service', + '--cluster', cluster, + '--service', controllerService.service.serviceName, + '--task-definition', taskDefinitionArn, + '--force-new-deployment', + ); + +} finally { + await rm(temporaryDirectory, { recursive: true, force: true }); +} diff --git a/tests/multi-runner-orchestration/smoke/reporter.mjs b/tests/multi-runner-orchestration/smoke/reporter.mjs new file mode 100644 index 0000000000..225ad1a45f --- /dev/null +++ b/tests/multi-runner-orchestration/smoke/reporter.mjs @@ -0,0 +1,27 @@ +function formatFailure(error) { + if (!error) return 'No error details in test event'; + + const messages = []; + const visited = new Set(); + for (let current = error; current && !visited.has(current); current = current.cause) { + visited.add(current); + messages.push(current.stack ?? current.message ?? String(current)); + } + return messages.join('\nCaused by: '); +} + +export default async function* smokeReporter(events) { + for await (const event of events) { + const { name, details } = event.data ?? {}; + const error = details?.error ?? event.data?.error; + const failureText = formatFailure(error); + if (details?.type !== 'test') continue; + + if (event.type === 'test:pass') { + yield `PASS ${name}\n`; + } else if (event.type === 'test:fail') { + yield `FAIL ${name}\n`; + yield ` ${failureText.split('\n').join('\n ')}\n`; + } + } +} diff --git a/tests/multi-runner-orchestration/smoke/scale-set-support.mjs b/tests/multi-runner-orchestration/smoke/scale-set-support.mjs new file mode 100644 index 0000000000..8e4749b2eb --- /dev/null +++ b/tests/multi-runner-orchestration/smoke/scale-set-support.mjs @@ -0,0 +1,201 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { + assertCloudWatchEvent, + aws, + waitFor, +} from './support.mjs'; + +const testTimeout = Number(process.env.MINISTACK_SMOKE_TIMEOUT_MS ?? 7200000); + +export async function findScaleSetControllerService( + clusterName = process.env.MINISTACK_SCALE_SET_CLUSTER ?? 'multi-runner-webhook-scale-set', +) { + const clusters = (await aws('ecs', 'list-clusters')).clusterArns ?? []; + const clusterArn = clusters.find((arn) => arn.includes(clusterName)); + assert.ok(clusterArn, 'Scale-set ECS cluster is missing; ensure Compose terraform-apply completed'); + + const serviceArns = (await aws('ecs', 'list-services', '--cluster', clusterArn)).serviceArns ?? []; + for (const serviceArn of serviceArns) { + const response = await aws( + 'ecs', 'describe-services', + '--cluster', clusterArn, + '--services', serviceArn, + ); + const service = response.services?.[0]; + if (!service?.taskDefinition) continue; + + const taskResponse = await aws('ecs', 'describe-task-definition', '--task-definition', service.taskDefinition); + const taskDefinition = taskResponse.taskDefinition; + const container = taskDefinition.containerDefinitions?.find( + (definition) => definition.name === 'scale-set-controller', + ); + + if (container) { + return { + clusterArn, + service, + taskDefinition, + container, + logGroup: container.logConfiguration?.options?.['awslogs-group'], + }; + } + } + + throw new Error('Scale-set controller ECS service/task definition not found'); +} + +export function prepareTaskDefinition(controller, minimumRunners) { + const taskDefinition = structuredClone(controller.taskDefinition); + const registeredFields = [ + 'taskDefinitionArn', + 'revision', + 'status', + 'requiresAttributes', + 'compatibilities', + 'registeredAt', + 'registeredBy', + ]; + + for (const field of registeredFields) delete taskDefinition[field]; + + const container = taskDefinition.containerDefinitions.find( + (definition) => definition.name === 'scale-set-controller', + ); + const environment = new Map( + (container.environment ?? []).map(({ name, value }) => [name, value]), + ); + environment.set('AWS_ACCESS_KEY_ID', '000000000000'); + environment.set('AWS_SECRET_ACCESS_KEY', 'test-only'); + + const manifest = JSON.parse(environment.get('SCALE_SET_CONTROLLER_MANIFEST')); + assert.equal(manifest.reconcilers?.length, 1, 'Expected one scale-set reconciler'); + manifest.reconcilers[0].minRunners = minimumRunners; + environment.set('SCALE_SET_CONTROLLER_MANIFEST', JSON.stringify(manifest)); + + container.environment = [...environment].map(([name, value]) => ({ name, value })); + return taskDefinition; +} + +export async function waitForTaskRevision(controller, taskDefinitionArn, minimumRunners) { + return waitFor(`controller deployment with minimum ${minimumRunners}`, async () => { + const response = await aws( + 'ecs', 'list-tasks', + '--cluster', controller.clusterArn, + '--service-name', controller.service.serviceName, + ).catch(() => ({})); + const taskArns = response.taskArns ?? []; + if (taskArns.length === 0) return false; + + const taskResponse = await aws( + 'ecs', 'describe-tasks', + '--cluster', controller.clusterArn, + '--tasks', ...taskArns, + ).catch(() => ({})); + + return taskResponse.tasks?.find((task) => ( + task.taskDefinitionArn === taskDefinitionArn && task.lastStatus === 'RUNNING' + )); + }); +} + +export async function deployMinimumRunners(controller, minimumRunners, temporaryDirectory) { + const taskDefinition = prepareTaskDefinition(controller, minimumRunners); + const inputFile = path.join(temporaryDirectory, `task-${minimumRunners}.json`); + await writeFile(inputFile, JSON.stringify(taskDefinition)); + + const registration = await aws( + 'ecs', 'register-task-definition', + '--cli-input-json', `file://${inputFile}`, + ); + const taskDefinitionArn = registration.taskDefinition?.taskDefinitionArn; + assert.ok(taskDefinitionArn, 'MiniStack did not register the scale-set task revision'); + const startTime = Date.now() - 1000; + + await aws( + 'ecs', 'update-service', + '--cluster', controller.clusterArn, + '--service', controller.service.serviceName, + '--task-definition', taskDefinitionArn, + '--force-new-deployment', + ); + const task = await waitForTaskRevision(controller, taskDefinitionArn, minimumRunners); + const streamPrefix = controller.container.logConfiguration?.options?.['awslogs-stream-prefix']; + const taskId = task.taskArn?.split('/').at(-1); + assert.ok(streamPrefix && taskId, 'MiniStack did not report the controller task log stream identity'); + + return { + startTime, + logStreamName: `${streamPrefix}/${controller.container.name}/${taskId}`, + }; +} + +export async function assertReconciliation(controller, minimumRunners, deployment) { + await assertCloudWatchEvent( + controller.logGroup, + 'scale_set_compute_provider_reconcile_started', + { desiredRunners: minimumRunners }, + deployment.startTime, + deployment.logStreamName, + ); + + const message = await assertCloudWatchEvent( + controller.logGroup, + 'scale_set_reconciled', + { desiredRunners: minimumRunners }, + deployment.startTime, + deployment.logStreamName, + ); + + return JSON.parse(message); +} + +export function registerScaleSetLifecycleTest({ + providerName, + clusterName = process.env.MINISTACK_SCALE_SET_CLUSTER ?? 'multi-runner-webhook-scale-set', + verifyBaseline, + verifyScaleUp, + verifyScaleDown, +}) { + for (const [name, verifier] of Object.entries({ verifyBaseline, verifyScaleUp, verifyScaleDown })) { + if (typeof verifier !== 'function') { + throw new TypeError(`${name} must be a function for ${providerName} scale-set smoke coverage`); + } + } + + test(`${providerName} Scale set: baseline at 0, scale up to 1, and scale down to 0 runners`, { + timeout: testTimeout, + }, async () => { + const controller = await findScaleSetControllerService(clusterName); + assert.ok(controller.logGroup, 'Scale-set controller task has no CloudWatch log group'); + + const temporaryDirectory = await mkdtemp(path.join(tmpdir(), 'scale-set-smoke-')); + const scaleSteps = [ + { minimumRunners: 0, verify: verifyBaseline }, + { minimumRunners: 1, verify: verifyScaleUp }, + { minimumRunners: 0, verify: verifyScaleDown }, + ]; + + try { + for (const step of scaleSteps) { + const deployment = await deployMinimumRunners( + controller, + step.minimumRunners, + temporaryDirectory, + ); + const reconciliation = await assertReconciliation( + controller, + step.minimumRunners, + deployment, + ); + + step.verify(reconciliation); + } + } finally { + await rm(temporaryDirectory, { recursive: true, force: true }); + } + }); +} diff --git a/tests/multi-runner-orchestration/smoke/scale-set.ec2.test.mjs b/tests/multi-runner-orchestration/smoke/scale-set.ec2.test.mjs new file mode 100644 index 0000000000..1e21ba1e28 --- /dev/null +++ b/tests/multi-runner-orchestration/smoke/scale-set.ec2.test.mjs @@ -0,0 +1,19 @@ +import assert from 'node:assert/strict'; +import { registerScaleSetLifecycleTest } from './scale-set-support.mjs'; + +registerScaleSetLifecycleTest({ + providerName: 'EC2', + verifyBaseline: (event) => assert.equal( + event.currentRunners, + 0, + 'Scale-set baseline should have no runners', + ), + verifyScaleUp: (event) => { + assert.ok(event.actions?.launched > 0, 'EC2 scale-set scale-up did not launch a runner'); + assert.equal(event.currentRunners, 1, 'EC2 scale-set scale-up did not reach one runner'); + }, + verifyScaleDown: (event) => { + assert.ok(event.actions?.terminated > 0, 'EC2 scale-set scale-down did not terminate a runner'); + assert.equal(event.currentRunners, 0, 'EC2 scale-set scale-down did not return to zero runners'); + }, +}); diff --git a/tests/multi-runner-orchestration/smoke/support.mjs b/tests/multi-runner-orchestration/smoke/support.mjs new file mode 100644 index 0000000000..99d544779a --- /dev/null +++ b/tests/multi-runner-orchestration/smoke/support.mjs @@ -0,0 +1,391 @@ +import { spawn } from 'node:child_process'; +import { createHmac, randomUUID } from 'node:crypto'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { setTimeout as delay } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { request as httpRequest } from 'node:http'; + +const supportDirectory = path.dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = path.resolve(supportDirectory, '../../..'); +const exampleDirectory = path.join(repositoryRoot, 'examples/multi-runner-orchestration'); + +export const awsEndpoint = process.env.AWS_ENDPOINT_URL ?? 'http://localhost:4566'; +export const awsRegion = process.env.AWS_DEFAULT_REGION ?? 'eu-west-1'; + +const awsEnvironment = { + ...process.env, + AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID ?? '000000000000', + AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY ?? 'test-only', + AWS_DEFAULT_REGION: awsRegion, + AWS_REGION: awsRegion, + AWS_ENDPOINT_URL: awsEndpoint, + AWS_EC2_METADATA_DISABLED: 'true', +}; + +export function run(command, args, { cwd = repositoryRoot, silent = false, check = true } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + env: awsEnvironment, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const stdout = []; + const stderr = []; + + child.stdout.on('data', (chunk) => { + stdout.push(chunk); + if (!silent) process.stdout.write(chunk); + }); + child.stderr.on('data', (chunk) => { + stderr.push(chunk); + if (!silent) process.stderr.write(chunk); + }); + child.on('error', reject); + child.on('close', (code) => { + const result = { + code: code ?? 1, + stdout: Buffer.concat(stdout).toString().trim(), + stderr: Buffer.concat(stderr).toString().trim(), + }; + + if (code === 0 || !check) { + resolve(result); + return; + } + + reject(new Error(`${command} ${args.join(' ')} failed (${code}): ${result.stderr}`)); + }); + }); +} + +export async function aws(...args) { + const result = await run( + 'aws', + ['--endpoint-url', awsEndpoint, '--region', awsRegion, ...args, '--output', 'json'], + { silent: true }, + ); + + return result.stdout ? JSON.parse(result.stdout) : {}; +} + +export async function invokeLambda(functionName, payload = {}) { + const temporaryDirectory = await mkdtemp(path.join(tmpdir(), 'runner-smoke-lambda-')); + const responseFile = path.join(temporaryDirectory, 'response.json'); + + try { + const response = await aws( + 'lambda', 'invoke', + '--function-name', functionName, + '--cli-binary-format', 'raw-in-base64-out', + '--payload', JSON.stringify(payload), + responseFile, + ); + + if (response.FunctionError) { + throw new Error(`${functionName} invocation failed: ${response.FunctionError}`); + } + + return response; + } finally { + await rm(temporaryDirectory, { recursive: true, force: true }); + } +} + +export async function assertEc2RunnerCount(expectedCount) { + const environment = process.env.MINISTACK_RUNNER_ENVIRONMENT ?? 'multi-runner-webhook-ec2'; + const description = `${expectedCount} active EC2 runner(s) in ${environment}`; + + try { + return await waitFor(description, async () => { + const response = await aws( + 'ec2', 'describe-instances', + '--filters', + `Name=tag:ghr:environment,Values=${environment}`, + 'Name=instance-state-name,Values=pending,running', + ); + const instances = (response.Reservations ?? []).flatMap((reservation) => reservation.Instances ?? []); + return instances.length === expectedCount ? instances : false; + }); + } catch (error) { + const allInstances = await aws( + 'ec2', 'describe-instances', + '--filters', `Name=tag:ghr:environment,Values=${environment}`, + ).catch((diagnosticError) => ({ diagnosticError: diagnosticError.message })); + const instances = (allInstances.Reservations ?? []) + .flatMap((reservation) => reservation.Instances ?? []) + .map(({ InstanceId, State, StateReason, LaunchTime, Tags }) => ({ + instanceId: InstanceId, + state: State?.Name, + stateReason: StateReason?.Message, + launchTime: LaunchTime, + name: Tags?.find(({ Key }) => Key === 'Name')?.Value, + })); + const logGroups = [ + '/aws/lambda/multi-runner-webhook-webhook', + `/aws/lambda/${environment}-scale-up`, + `/aws/lambda/${environment}-pool`, + ]; + const logs = await Promise.all(logGroups.map(async (logGroup) => ({ + logGroup, + messages: await logMessages(logGroup, Date.now() - 15 * 60_000), + }))); + const logContext = logs + .map(({ logGroup, messages }) => `${logGroup}:\n${messages.slice(-10).join('\n') || 'No recent messages'}`) + .join('\n'); + + throw new Error( + `${error.message}\nEC2 instances with environment ${environment}: ${JSON.stringify(instances.length ? instances : allInstances)}\nRecent Lambda logs:\n${logContext}`, + { cause: error }, + ); + } +} + +export async function assertEc2InstancesTerminated(instances) { + for (const { InstanceId } of instances) { + await waitFor(`EC2 instance ${InstanceId} to reach terminated state`, async () => { + const response = await aws('ec2', 'describe-instances', '--instance-ids', InstanceId); + const instance = (response.Reservations ?? []) + .flatMap((reservation) => reservation.Instances ?? []) + .find(({ InstanceId: id }) => id === InstanceId); + return instance?.State?.Name === 'terminated'; + }); + } +} + +export async function waitFor(description, predicate, attempts = 90) { + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const result = await predicate(); + if (result) return result; + + await delay(2000); + } + + throw new Error(`Timed out waiting for ${description}`); +} + +export async function terraformOutput(name) { + const result = await run( + 'terraform', + [`-chdir=${exampleDirectory}`, 'output', '-raw', name], + { silent: true }, + ); + + return result.stdout; +} + +let nextJobId = 0; + +function createWorkflowJobEvent(jobId, labels) { + const repositoryName = 'test-owner/test-repo'; + const jobUrl = `https://github.example.invalid/${repositoryName}/actions/jobs/${jobId}`; + const workflowRunUrl = `https://github.example.invalid/${repositoryName}/actions/runs/654321`; + + return { + action: 'queued', + installation: { id: 123 }, + repository: { + full_name: repositoryName, + id: 1, + name: 'test-repo', + owner: { login: 'test-owner', type: 'Organization' }, + }, + workflow_job: { + conclusion: null, + html_url: jobUrl, + id: jobId, + labels, + name: `node-smoke-${jobId}`, + run_id: 654321, + run_url: workflowRunUrl, + status: 'queued', + url: jobUrl, + }, + }; +} + +export async function sendWorkflowJob(labels) { + const endpoint = process.env.MINISTACK_WEBHOOK_ENDPOINT + ?? await terraformOutput('webhook_endpoint'); + const secret = process.env.MINISTACK_WEBHOOK_SECRET + ?? await terraformOutput('webhook_secret'); + const jobId = Number(process.env.MINISTACK_SMOKE_JOB_ID ?? Date.now().toString().slice(-8)) + + nextJobId; + nextJobId += 1; + + const eventBody = Buffer.from(JSON.stringify(createWorkflowJobEvent(jobId, labels))); + const signature = createHmac('sha256', secret).update(eventBody).digest('hex'); + const webhookUrl = new URL(endpoint); + const startTime = Date.now() - 1000; + const deliveryId = `node-smoke-${randomUUID()}`; + const response = await postWebhook( + webhookUrl.host, + `${webhookUrl.pathname}${webhookUrl.search}`, + { + 'content-type': 'application/json', + 'x-github-event': 'workflow_job', + 'x-github-delivery': deliveryId, + 'x-github-hook-installation-target-id': '123', + 'x-hub-signature-256': `sha256=${signature}`, + }, + eventBody, + ); + + if (![200, 201].includes(response.status)) { + throw new Error(`Webhook failed with HTTP ${response.status}: ${response.body}`); + } + + return { jobId, deliveryId, startTime }; +} + +export async function assertWebhookJobReceivedAndDispatched(jobId, deliveryId, startTime) { + const webhookLogGroup = '/aws/lambda/multi-runner-webhook-webhook'; + const dispatcherLogGroup = '/aws/lambda/multi-runner-webhook-dispatch-to-runner'; + await assertCloudWatchLogContains( + webhookLogGroup, + `webhook delivery ${deliveryId} accepted for job ${jobId}`, + deliveryId, + startTime, + ); + await assertCloudWatchLogContains( + webhookLogGroup, + `webhook delivery ${deliveryId} parsed workflow job ${jobId}`, + `\"workflowJobId\":${jobId}`, + startTime, + ); + await assertCloudWatchLogContains( + dispatcherLogGroup, + `webhook delivery ${deliveryId} dispatched job ${jobId} to a runner queue`, + `Successfully dispatched job for test-owner/test-repo to the queue(s)`, + startTime, + ); + await assertCloudWatchLogContains( + dispatcherLogGroup, + `webhook delivery ${deliveryId} matched job ${jobId}`, + `Job ID: ${jobId}`, + startTime, + ); +} + +export async function assertEc2InstanceType(instances, expectedTypes, description) { + if (instances.length !== 1) { + throw new Error(`${description}: expected one EC2 instance, found ${instances.length}`); + } + + const [{ InstanceId, InstanceType }] = instances; + if (!expectedTypes.includes(InstanceType)) { + throw new Error( + `${description}: EC2 instance ${InstanceId} has type ${InstanceType}; expected ${expectedTypes.join(' or ')}`, + ); + } +} + +export function postWebhook(host, pathname, headers, body) { + return new Promise((resolve, reject) => { + const endpoint = new URL(awsEndpoint); + const request = httpRequest( + { + hostname: endpoint.hostname, + port: endpoint.port || (endpoint.protocol === 'https:' ? 443 : 80), + path: pathname, + method: 'POST', + headers: { ...headers, host }, + }, + (response) => { + const responseBody = []; + response.setEncoding('utf8'); + response.on('data', (chunk) => responseBody.push(chunk)); + response.on('end', () => { + resolve({ + status: response.statusCode ?? 0, + body: responseBody.join(''), + }); + }); + }, + ); + + request.setTimeout(15000, () => request.destroy(new Error('Timed out sending webhook'))); + request.on('error', reject); + request.end(body); + }); +} + +export async function logMessages(logGroup, startTime, logStreamName) { + const args = [ + '--endpoint-url', awsEndpoint, + '--region', awsRegion, + 'logs', 'filter-log-events', + '--log-group-name', logGroup, + '--start-time', String(startTime), + '--limit', '100', + '--output', 'json', + ]; + if (logStreamName) args.push('--log-stream-names', logStreamName); + + const result = await run( + 'aws', + args, + { silent: true, check: false }, + ); + + if (result.code !== 0) return []; + + try { + const response = JSON.parse(result.stdout); + return response.events?.map((event) => event.message) ?? []; + } catch { + return []; + } +} + +export async function assertCloudWatchLogContains(logGroup, description, text, startTime) { + let recentMessages = []; + try { + await waitFor(description, async () => { + recentMessages = await logMessages(logGroup, startTime); + return recentMessages.find((message) => message.includes(String(text))); + }); + } catch (error) { + const logContext = recentMessages.length + ? recentMessages.slice(-10).join('\n') + : `No CloudWatch messages found in ${logGroup} after ${new Date(startTime).toISOString()}`; + throw new Error(`${error.message}\nRecent CloudWatch messages:\n${logContext}`, { cause: error }); + } + +} + +function eventMatches(message, eventName, expectedFields) { + try { + const event = JSON.parse(message); + return (event.event === eventName || event.message === eventName) + && Object.entries(expectedFields).every(([key, value]) => event[key] === value); + } catch { + return message.includes(eventName) + && Object.entries(expectedFields).every(([key, value]) => ( + message.includes(`"${key}":${JSON.stringify(value)}`) + )); + } +} + +export async function assertCloudWatchEvent( + logGroup, + eventName, + expectedFields, + startTime = Date.now() - 60_000, + logStreamName, +) { + const description = `CloudWatch ${eventName} (${JSON.stringify(expectedFields)}) in ${logGroup}`; + let recentMessages = []; + try { + return await waitFor(description, async () => { + recentMessages = await logMessages(logGroup, startTime, logStreamName); + return recentMessages.find((entry) => eventMatches(entry, eventName, expectedFields)); + }); + } catch (error) { + const logContext = recentMessages.length + ? recentMessages.slice(-10).join('\n') + : `No CloudWatch messages found in ${logGroup} after ${new Date(startTime).toISOString()}`; + throw new Error(`${error.message}\nRecent CloudWatch messages:\n${logContext}`, { cause: error }); + } +} diff --git a/tests/multi-runner-orchestration/smoke/webhook.ec2.test.mjs b/tests/multi-runner-orchestration/smoke/webhook.ec2.test.mjs new file mode 100644 index 0000000000..504637d12c --- /dev/null +++ b/tests/multi-runner-orchestration/smoke/webhook.ec2.test.mjs @@ -0,0 +1,104 @@ +import { test } from 'node:test'; +import { + assertEc2RunnerCount, + assertEc2InstanceType, + assertEc2InstancesTerminated, + assertCloudWatchLogContains, + assertCloudWatchEvent, + assertWebhookJobReceivedAndDispatched, + invokeLambda, + sendWorkflowJob, +} from './support.mjs'; + +const testTimeout = Number(process.env.MINISTACK_SMOKE_TIMEOUT_MS ?? 7200000); +const scaleDownFunction = 'multi-runner-webhook-ec2-scale-down'; +const scaleDownLogGroup = '/aws/lambda/multi-runner-webhook-ec2-scale-down'; +const configuredInstanceTypes = ['m7a.large', 'm5.large']; + +async function scaleDown(description, startTime, instances) { + // MiniStack does not boot the EC2 runner agent to register it with GitHub. + // The first scheduled cleanup marks it orphaned; the next confirms GitHub + // has no such runner and terminates the EC2 instance. + await invokeLambda(scaleDownFunction); + await invokeLambda(scaleDownFunction); + for (const { InstanceId } of instances) { + await assertCloudWatchLogContains( + scaleDownLogGroup, + `${description} runner instance ${InstanceId} was terminated`, + `Runner ${InstanceId} has been terminated.`, + startTime, + ); + } + await assertEc2InstancesTerminated(instances); + await assertEc2RunnerCount(0); +} + +test('EC2 Webhook: standard labels create and terminate a runner', { + timeout: testTimeout, +}, async () => { + await assertEc2RunnerCount(0); + const { jobId, deliveryId, startTime } = await sendWorkflowJob(['self-hosted', 'linux', 'x64', 'ec2']); + await assertWebhookJobReceivedAndDispatched(jobId, deliveryId, startTime); + await assertCloudWatchLogContains( + '/aws/lambda/multi-runner-webhook-ec2-scale-up', + `scale-up attempted runner launch for standard-label job ${jobId}`, + 'Runner creation summary.', + startTime, + ); + await assertCloudWatchEvent( + '/aws/lambda/multi-runner-webhook-ec2-scale-up', + 'Runner creation summary.', + { successfulRunnerCount: 1 }, + startTime, + ); + const instances = await assertEc2RunnerCount(1); + await assertEc2InstanceType(instances, configuredInstanceTypes, `standard-label job ${jobId}`); + const scaleDownStartTime = Date.now() - 1000; + await scaleDown(`standard-label job ${jobId}`, scaleDownStartTime, instances); +}); + +test('EC2 Webhook: dynamic labels apply m5.large and terminate the runner', { + timeout: testTimeout, +}, async () => { + await assertEc2RunnerCount(0); + const dynamicLabel = 'ghr-ec2-instance-type:m5.large'; + const { jobId, deliveryId, startTime } = await sendWorkflowJob([ + 'self-hosted', 'linux', 'x64', 'ec2', dynamicLabel, + ]); + await assertWebhookJobReceivedAndDispatched(jobId, deliveryId, startTime); + await assertCloudWatchLogContains( + '/aws/lambda/multi-runner-webhook-ec2-scale-up', + `scale-up applied dynamic label ${dynamicLabel} for job ${jobId}`, + '"ec2OverrideConfig":{"InstanceType":"m5.large"}', + startTime, + ); + await assertCloudWatchEvent( + '/aws/lambda/multi-runner-webhook-ec2-scale-up', + 'Runner creation summary.', + { successfulRunnerCount: 1 }, + startTime, + ); + const instances = await assertEc2RunnerCount(1); + await assertEc2InstanceType(instances, ['m5.large'], `dynamic-label job ${jobId}`); + const scaleDownStartTime = Date.now() - 1000; + await scaleDown(`dynamic-label job ${jobId}`, scaleDownStartTime, instances); +}); + +test('EC2 Pool: create and terminate one runner', { + timeout: testTimeout, +}, async () => { + await assertEc2RunnerCount(0); + const startTime = Date.now() - 1000; + await invokeLambda('multi-runner-webhook-ec2-pool', { poolSize: 1, type: 'ec2' }); + await assertCloudWatchLogContains( + '/aws/lambda/multi-runner-webhook-ec2-pool', + 'pool handler requested one runner', + 'The pool will be topped up with 1 runners.', + startTime, + ); + const instances = await assertEc2RunnerCount(1); + await assertEc2InstanceType(instances, configuredInstanceTypes, 'pool'); + + const scaleDownStartTime = Date.now() - 1000; + await scaleDown('pool', scaleDownStartTime, instances); +});