diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 2d2c4cf92a2..8bdbddc53d2 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -13,3 +13,34 @@ jobs: - uses: golangci/golangci-lint-action@v9 with: working-directory: src/brats/ + + bosh-monitor-lint: + name: bosh-monitor lint + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v6 + with: + go-version-file: src/bosh-monitor/go.mod + - uses: golangci/golangci-lint-action@v9 + with: + working-directory: src/bosh-monitor/ + + bosh-monitor-test: + name: bosh-monitor test + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v6 + with: + go-version-file: src/bosh-monitor/go.mod + - name: Run tests + working-directory: src/bosh-monitor + run: go test ./... + - name: Build binary + working-directory: src/bosh-monitor + run: go build -o bin/bosh-monitor . diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index b954cdf18bb..e584edc6dbf 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -8,7 +8,6 @@ jobs: matrix: sub_project: - common:parallel - - monitor:parallel - nats_sync:parallel - release:parallel - integration_support:parallel diff --git a/ci/fly-bats.yml b/ci/fly-bats.yml new file mode 100644 index 00000000000..c6bbff5c817 --- /dev/null +++ b/ci/fly-bats.yml @@ -0,0 +1,149 @@ +--- +# Standalone pipeline for running the BOSH Acceptance Test suite (BATs) +# against a specific branch. +# +# Intended to be driven by `bundle exec rake fly:bats` from src/, which +# pushes the current branch and sets this pipeline automatically. +# +# Manual usage: +# fly -t bosh set-pipeline -p bats-local \ +# -c ci/fly-bats.yml \ +# --var bosh_repo=https://github.com/cloudfoundry/bosh.git \ +# --var bosh_branch=my-feature-branch \ +# --var env_name=bats-local \ +# --var stemcell_name=bosh-google-kvm-ubuntu-noble \ +# --var deploy_args="-o bosh-deployment/external-ip-not-recommended.yml" \ +# --var bat_rspec_flags="" +# fly -t bosh unpause-pipeline -p bats-local +# fly -t bosh trigger-job -j bats-local/bats -w + +resources: + - name: bosh + type: git + source: + uri: ((bosh_repo)) + branch: ((bosh_branch)) + + # bosh-ci is the same repo as bosh but filtered to ci/ paths so that + # scripts under ci/bats/ are available at bosh-ci/ in the task workspace. + - name: bosh-ci + type: git + source: + uri: ((bosh_repo)) + branch: ((bosh_branch)) + paths: [ci] + + - name: bosh-cli + type: github-release + source: + owner: cloudfoundry + repository: bosh-cli + access_token: ((github_public_repo_token)) + + - name: stemcell + type: bosh-io-stemcell + source: + name: bosh-google-kvm-ubuntu-noble + + - name: bats + type: git + source: + uri: https://github.com/cloudfoundry/bosh-acceptance-tests.git + branch: master + + - name: bosh-deployment + type: git + source: + uri: https://github.com/cloudfoundry/bosh-deployment + branch: master + + - name: integration-image + type: registry-image + source: + repository: ghcr.io/cloudfoundry/bosh/integration + tag: main + username: ((github_read_write_packages.username)) + password: ((github_read_write_packages.password)) + +jobs: + - name: bats + serial: true + plan: + - do: + - in_parallel: + - get: bosh + - get: bosh-ci + - get: bosh-cli + params: + globs: [bosh-cli-*-linux-amd64] + - get: stemcell + - get: bats + - get: bosh-deployment + - get: integration-image + + - task: make-candidate + image: integration-image + file: bosh-ci/ci/tasks/make-candidate.yml + + - task: compile-bosh-release + config: + platform: linux + # ubuntu-noble-stemcell ships the BOSH agent at /var/vcap/bosh/bin/bosh-agent. + image_resource: + type: registry-image + source: + repository: ghcr.io/cloudfoundry/ubuntu-noble-stemcell + inputs: + - name: release + outputs: + - name: compiled-release + run: + path: /bin/bash + args: + - -exc + - | + # golang-1.26-linux on this branch is a stub (env files only; no binary + # blob in GCS yet). Install Go 1.26.3 and drop a shim at + # /usr/local/bin/go so `go build` works even though $GOROOT/bin is empty. + GO_VERSION=1.26.3 + if ! /usr/local/go/bin/go version 2>/dev/null | grep -q "go${GO_VERSION}"; then + curl -sSfL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ + | tar -C /usr/local -xz + fi + cat > /usr/local/bin/go <<'GOSHIM' + #!/bin/bash + export GOROOT=/usr/local/go + exec /usr/local/go/bin/go "$@" + GOSHIM + chmod +x /usr/local/bin/go + + /var/vcap/bosh/bin/bosh-agent compile \ + --output-directory compiled-release \ + release/*.tgz + + - task: run-bats + image: integration-image + input_mapping: + bosh-release: compiled-release + config: + platform: linux + inputs: + - name: bosh + - name: bosh-ci + - name: bosh-cli + - name: bosh-deployment + - name: stemcell + - name: bats + - name: bosh-release + caches: + - path: cache-dot-bosh-dir + params: + BAT_INFRASTRUCTURE: gcp + GCP_JSON_KEY: ((gcp_json_key)) + GCP_PROJECT_ID: ((gcp_project_id)) + STEMCELL_NAME: ((stemcell_name)) + ENV_NAME: ((env_name)) + DEPLOY_ARGS: ((deploy_args)) + BAT_RSPEC_FLAGS: ((bat_rspec_flags)) + run: + path: bosh/ci/tasks/run-bats-pipeline.sh diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 8fe3f7c922a..52c58d6f360 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -1406,7 +1406,7 @@ jobs: - get: bosh - get: golang-release - get: integration-image - - task: bump-deps + - task: bump-brats-deps file: golang-release/ci/tasks/shared/bump-deps.yml input_mapping: input_repo: bosh @@ -1414,6 +1414,14 @@ jobs: output_repo: bosh-out params: SOURCE_PATH: src/brats/ + - task: bump-bosh-monitor-deps + file: golang-release/ci/tasks/shared/bump-deps.yml + input_mapping: + input_repo: bosh-out + output_mapping: + output_repo: bosh-out + params: + SOURCE_PATH: src/bosh-monitor/ - task: lint-brats file: bosh-ci/ci/tasks/lint-brats.yml image: integration-image @@ -1431,6 +1439,8 @@ jobs: - get: bosh-ci - get: integration-image - get: bosh + - get: golang-release + trigger: true - get: nginx-release trigger: true - get: mariadb-connector-c-resource @@ -1535,6 +1545,22 @@ jobs: credentials_source: static json_key: '((gcp_json_key))' RUBY_VERSION_PATH: src/.ruby-version + - task: bump-golang-package + file: golang-release/ci/tasks/shared/bump-golang-package.yml + input_mapping: + input_repo: bosh-out + output_mapping: + output_repo: bosh-out + params: + GIT_USER_NAME: *git_user_name + GIT_USER_EMAIL: *git_user_email + PACKAGES: '["golang-1.26-linux"]' + PACKAGES_TO_REMOVE: '[]' + PRIVATE_YML: | + blobstore: + options: + credentials_source: static + json_key: '((gcp_json_key))' - task: bump-bosh-blobstore-dav file: bosh-ci/ci/tasks/bump-blobstore-cli.yml image: integration-image diff --git a/ci/tasks/run-bats-pipeline.sh b/ci/tasks/run-bats-pipeline.sh new file mode 100755 index 00000000000..b0fef1516e5 --- /dev/null +++ b/ci/tasks/run-bats-pipeline.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# run-bats-pipeline.sh +# +# Chains together the full BATs pipeline in a single Concourse task: +# terraform apply → deploy-director → prepare-bats-config → run-bats +# terraform destroy ← destroy-director ← (EXIT trap, always runs) +# +# Required env vars (set via run-bats-pipeline.yml params): +# GCP_JSON_KEY – GCP service account JSON (resolved from Concourse creds) +# GCP_PROJECT_ID – GCP project ID +# ENV_NAME – Unique terraform env name, e.g. "bats-local" +# BAT_INFRASTRUCTURE – "gcp" +# STEMCELL_NAME – Stemcell name for bats-config.yml +# DEPLOY_ARGS – Extra ops-file args for bosh create-env +# BAT_RSPEC_FLAGS – Extra flags appended to the BAT run (optional) + +set -eu + +ROOT_DIR="$PWD" +TERRAFORM_DIR="${ROOT_DIR}/bosh-ci/ci/bats/iaas/gcp/terraform" +TERRAFORM_VERSION="1.9.8" + +# ── Shared working directories ─────────────────────────────────────────────── +mkdir -p director-state bats-config environment +# cache-dot-bosh-dir is provided as a Concourse cache volume; create if absent +mkdir -p cache-dot-bosh-dir/.bosh + +# prepare-bats-config.sh expects its terraform metadata at terraform/metadata +# but deploy-director.sh expects it at environment/metadata. +# Symlink terraform/ → environment/ so both scripts find what they need. +ln -sf "${ROOT_DIR}/environment" "${ROOT_DIR}/terraform" + +# ── Install terraform ──────────────────────────────────────────────────────── +if ! command -v terraform &>/dev/null; then + echo "--- Installing terraform ${TERRAFORM_VERSION} ---" + curl -sSL \ + "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" \ + -o /tmp/terraform.zip + unzip -qo /tmp/terraform.zip -d /usr/local/bin terraform + chmod +x /usr/local/bin/terraform +fi + +# ── GCP credentials file (used by the GCS backend and the Google provider) ── +GCP_CREDS_FILE="$(mktemp /tmp/gcp-creds-XXXXXX.json)" +echo "${GCP_JSON_KEY}" > "${GCP_CREDS_FILE}" +chmod 600 "${GCP_CREDS_FILE}" + +# ── Provision GCP environment via terraform ────────────────────────────────── +echo "--- Provisioning GCP environment (env: ${ENV_NAME}) ---" +pushd "${TERRAFORM_DIR}" >/dev/null + +terraform init \ + -input=false \ + -reconfigure \ + -backend-config="bucket=bosh-director-pipeline" \ + -backend-config="prefix=bats-terraform/${ENV_NAME}" \ + -backend-config="credentials=${GCP_CREDS_FILE}" + +terraform apply \ + -input=false \ + -auto-approve \ + -var "project_id=${GCP_PROJECT_ID}" \ + -var "gcp_credentials_json=${GCP_JSON_KEY}" \ + -var "name=${ENV_NAME}" + +# Convert terraform outputs to the flat metadata JSON consumed by director-vars +# and prepare-bats-config.sh. +terraform output -json \ + | jq 'with_entries(.value = .value.value)' \ + > "${ROOT_DIR}/environment/metadata" + +popd >/dev/null + +# ── Teardown trap (always runs on EXIT) ───────────────────────────────────── +function collect_director_diagnostics { + # Only collect when we have a deployed director and bosh-cli is available. + [[ -f director-state/director-creds.yml ]] || return 0 + [[ -f "${ROOT_DIR}/environment/metadata" ]] || return 0 + command -v bosh-cli &>/dev/null || return 0 + + local director_ip + director_ip="$(jq -r '.director_public_ip // empty' "${ROOT_DIR}/environment/metadata")" + [[ -n "${director_ip}" ]] || return 0 + + echo "--- Collecting director diagnostics (IP: ${director_ip}) ---" + + # Extract jumpbox SSH private key from the vars-store. + local jumpbox_key_file + jumpbox_key_file="$(mktemp /tmp/jumpbox-key-XXXXXX)" + bosh-cli interpolate director-state/director-creds.yml \ + --path /jumpbox_ssh/private_key > "${jumpbox_key_file}" 2>/dev/null || { rm -f "${jumpbox_key_file}"; return 0; } + chmod 600 "${jumpbox_key_file}" + + ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -i "${jumpbox_key_file}" \ + "jumpbox@${director_ip}" \ + 'echo "=== monit status ===" && sudo /var/vcap/bosh/bin/monit status; + echo "=== bosh_nats_sync log (last 100 lines) ===" && sudo tail -100 /var/vcap/sys/log/nats/bosh-nats-sync.log 2>/dev/null || true; + echo "=== bosh_nats_sync bpm stdout ===" && sudo cat /var/vcap/sys/log/bpm/nats/bosh_nats_sync.stdout.log 2>/dev/null || true; + echo "=== bosh_nats_sync bpm stderr ===" && sudo cat /var/vcap/sys/log/bpm/nats/bosh_nats_sync.stderr.log 2>/dev/null || true; + echo "=== nats log (last 50 lines) ===" && sudo tail -50 /var/vcap/sys/log/nats/nats.log 2>/dev/null || true; + echo "=== nats bpm stdout (last 50 lines) ===" && sudo tail -50 /var/vcap/sys/log/bpm/nats/nats.stdout.log 2>/dev/null || true; + echo "=== health_monitor log (last 100 lines) ===" && sudo tail -100 /var/vcap/sys/log/health_monitor/health_monitor.log 2>/dev/null || true; + echo "=== health_monitor bpm stdout ===" && sudo tail -50 /var/vcap/sys/log/bpm/health_monitor/health_monitor.stdout.log 2>/dev/null || true; + echo "=== health_monitor bpm stderr ===" && sudo tail -50 /var/vcap/sys/log/bpm/health_monitor/health_monitor.stderr.log 2>/dev/null || true' \ + 2>&1 || echo "(SSH diagnostics failed — VM may not be reachable)" + + rm -f "${jumpbox_key_file}" +} + +function teardown { + local exit_code=$? + set +e + + # Always collect diagnostics – on success this helps correlate logs with + # passing runs; on failure it captures the state at the point of failure. + collect_director_diagnostics + + echo "--- Tearing down BOSH director ---" + if [[ -f director-state/director-state.json ]]; then + # destroy-director.sh expects bosh-cli/bosh-cli-* to exist; restore it + # because deploy-director.sh already moved the original binary away. + cp /usr/local/bin/bosh-cli bosh-cli/bosh-cli-restore 2>/dev/null || true + bosh-ci/ci/bats/tasks/destroy-director.sh || true + fi + + echo "--- Destroying GCP environment (env: ${ENV_NAME}) ---" + pushd "${TERRAFORM_DIR}" >/dev/null + terraform destroy \ + -input=false \ + -auto-approve \ + -var "project_id=${GCP_PROJECT_ID}" \ + -var "gcp_credentials_json=${GCP_JSON_KEY}" \ + -var "name=${ENV_NAME}" || true + popd >/dev/null + + rm -f "${GCP_CREDS_FILE}" + + exit "${exit_code}" +} +trap teardown EXIT + +# ── Deploy BOSH director ───────────────────────────────────────────────────── +echo "--- Deploying BOSH director ---" +# deploy-director.sh moves bosh-cli/bosh-cli-* to /usr/local/bin/bosh-cli. +# After this call bosh-cli is installed system-wide as 'bosh-cli'. +bosh-ci/ci/bats/tasks/deploy-director.sh + +# ── Prepare BATs config ────────────────────────────────────────────────────── +echo "--- Preparing BATs config ---" +bosh-ci/ci/bats/iaas/gcp/prepare-bats-config.sh + +# ── Run BATs ───────────────────────────────────────────────────────────────── +echo "--- Running BATs ---" +# Source the environment file that prepare-bats-config.sh wrote; this exports +# BOSH_ENVIRONMENT, BOSH_CLIENT, BOSH_CLIENT_SECRET, BOSH_CA_CERT, +# BOSH_ALL_PROXY, and the default BAT_RSPEC_FLAGS. +# shellcheck source=/dev/null +source bats-config/bats.env + +# Allow the caller to append extra RSpec flags (e.g. "--tag wip"). +if [[ -n "${BAT_RSPEC_FLAGS:-}" ]]; then + export BAT_RSPEC_FLAGS="${BAT_RSPEC_FLAGS}" +fi + +bats/ci/tasks/run-bats.sh diff --git a/jobs/health_monitor/spec b/jobs/health_monitor/spec index a15493b353a..a40daac7294 100644 --- a/jobs/health_monitor/spec +++ b/jobs/health_monitor/spec @@ -13,7 +13,6 @@ templates: packages: - health_monitor - - director-ruby-3.4 properties: # diff --git a/jobs/health_monitor/templates/bpm.yml b/jobs/health_monitor/templates/bpm.yml index ceb1e28e26b..c95453755d7 100644 --- a/jobs/health_monitor/templates/bpm.yml +++ b/jobs/health_monitor/templates/bpm.yml @@ -1,10 +1,10 @@ <%= health_monitor_config = { "name" => "health_monitor", - "executable" => "/var/vcap/jobs/health_monitor/bin/health_monitor", + "executable" => "/var/vcap/packages/health_monitor/bin/bosh-monitor", + "args" => ["-c", "/var/vcap/jobs/health_monitor/config/health_monitor.yml"], "env" => { - "BUNDLE_GEMFILE" => "/var/vcap/packages/health_monitor/Gemfile", - "GEM_HOME" => "/var/vcap/packages/health_monitor/gem_home/ruby/3.4.0", + "PATH" => "/var/vcap/packages/health_monitor/bin:/bin:/usr/bin:/sbin:/usr/sbin", }, "unsafe" => { "unrestricted_volumes" => [ @@ -13,10 +13,6 @@ health_monitor_config = { # without this we don't know which "version" of a job to use "path" => "/var/vcap/jobs", }, - { - "path" => "/var/vcap/data/jobs/*/*/bin/bosh-monitor", - "allow_executions" => true, - }, { "path" => "/var/vcap/sys/log", "writable" => true, diff --git a/jobs/health_monitor/templates/health_monitor b/jobs/health_monitor/templates/health_monitor index 1bbbbe42dae..1d3947e0d45 100755 --- a/jobs/health_monitor/templates/health_monitor +++ b/jobs/health_monitor/templates/health_monitor @@ -1,4 +1,3 @@ #!/bin/bash -source /var/vcap/packages/director-ruby-3.4/bosh/runtime.env exec /var/vcap/packages/health_monitor/bin/bosh-monitor -c /var/vcap/jobs/health_monitor/config/health_monitor.yml diff --git a/jobs/health_monitor/templates/health_monitor.yml.erb b/jobs/health_monitor/templates/health_monitor.yml.erb index 27a47f0cea6..da3dfb96156 100644 --- a/jobs/health_monitor/templates/health_monitor.yml.erb +++ b/jobs/health_monitor/templates/health_monitor.yml.erb @@ -43,6 +43,7 @@ params = { }, { 'name' => 'event_logger', + 'executable' => 'hm-event-logger', 'events' => [ 'alert', ], @@ -162,6 +163,7 @@ end if p('hm.datadog_enabled') datadog_plugin = { 'name' => 'data_dog', + 'executable' => 'hm-datadog', 'events' => [ 'alert', 'heartbeat', @@ -206,6 +208,7 @@ end if p('hm.consul_event_forwarder_enabled') consul_event_forwarder_plugin = { 'name' => 'consul_event_forwarder', + 'executable' => 'hm-consul', 'events' => [ 'alert', 'heartbeat', diff --git a/packages/golang-1.26-linux/packaging b/packages/golang-1.26-linux/packaging new file mode 100644 index 00000000000..db241eb12ab --- /dev/null +++ b/packages/golang-1.26-linux/packaging @@ -0,0 +1,7 @@ +set -e -x -u + +mkdir "${BOSH_INSTALL_TARGET}/bosh" + +PACKAGE_NAME="$(basename "${BOSH_INSTALL_TARGET}")" +sed "s#\${PACKAGE_NAME}#${PACKAGE_NAME}#" golang-1.26-linux/compile.env.unix > "${BOSH_INSTALL_TARGET}/bosh/compile.env" +sed "s#\${PACKAGE_NAME}#${PACKAGE_NAME}#" golang-1.26-linux/runtime.env.unix > "${BOSH_INSTALL_TARGET}/bosh/runtime.env" diff --git a/packages/golang-1.26-linux/spec b/packages/golang-1.26-linux/spec new file mode 100644 index 00000000000..6ce76af2a55 --- /dev/null +++ b/packages/golang-1.26-linux/spec @@ -0,0 +1,8 @@ +--- +name: golang-1.26-linux + +dependencies: [] + +files: +- golang-1.26-linux/compile.env.unix +- golang-1.26-linux/runtime.env.unix diff --git a/packages/health_monitor/packaging b/packages/health_monitor/packaging index 531ef1c2729..32341d1e665 100644 --- a/packages/health_monitor/packaging +++ b/packages/health_monitor/packaging @@ -1,26 +1,36 @@ set -e -mkdir -p ${BOSH_INSTALL_TARGET}/{bin,gem_home} +# Use the Go toolchain from a vendored golang BOSH package if available. +# When the golang-1.26-linux package is present, source its compile.env +# to set PATH/GOROOT/etc. the same way other BOSH releases do. +for pkg in "${BOSH_PACKAGES_DIR}"/golang-*; do + if [ -f "${pkg}/bosh/compile.env" ]; then + # shellcheck disable=SC1091 + source "${pkg}/bosh/compile.env" + break + fi +done -source /var/vcap/packages/director-ruby-3.4/bosh/compile.env +if ! command -v go >/dev/null 2>&1; then + echo "ERROR: Go toolchain not found." >&2 + echo " Add a golang-* BOSH package as a dependency in packages/health_monitor/spec" >&2 + echo " or ensure 'go' is available on PATH." >&2 + exit 1 +fi -cat > Gemfile <2' -gem 'bosh-monitor' -EOF +echo "Using $(go version)" +mkdir -p "${BOSH_INSTALL_TARGET}/bin" -for gemspec in $( find . -maxdepth 2 -name *.gemspec ); do - gem_name="$( basename "$( dirname "$gemspec" )" )" - gem_spec="$( basename "$gemspec" )" +cd bosh-monitor - pushd "$gem_name" - gem build "$gem_spec" - mv *.gem ../vendor/cache - popd > /dev/null -done +# BOSH compilation VMs have no network access, so dependencies must come from +# the committed vendor/ directory rather than the module proxy. +export GOFLAGS="-mod=vendor" +export GOPROXY=off -bosh_bundle_local +go build -o "${BOSH_INSTALL_TARGET}/bin/bosh-monitor" . -cp Gemfile ${BOSH_INSTALL_TARGET} -cp Gemfile.lock ${BOSH_INSTALL_TARGET} +for plugin_dir in cmd/plugins/hm-*; do + plugin_name="$(basename "${plugin_dir}")" + go build -o "${BOSH_INSTALL_TARGET}/bin/${plugin_name}" "./${plugin_dir}" +done diff --git a/packages/health_monitor/spec b/packages/health_monitor/spec index 89372fd7086..f7fffb7e0cd 100644 --- a/packages/health_monitor/spec +++ b/packages/health_monitor/spec @@ -1,10 +1,8 @@ --- name: health_monitor + dependencies: -- director-ruby-3.4 +- golang-1.26-linux files: - bosh-monitor/**/* -- bosh-common/**/* -- vendor/cache/*.gem -- vendor/cache/extensions/** diff --git a/src/Gemfile b/src/Gemfile index 85143d52307..53e3b68457b 100644 --- a/src/Gemfile +++ b/src/Gemfile @@ -1,7 +1,6 @@ source 'https://rubygems.org' gem 'bosh-director', path: 'bosh-director' -gem 'bosh-monitor', path: 'bosh-monitor' gem 'bosh-nats-sync', path: 'bosh-nats-sync' gem 'bosh-common', path: 'bosh-common' diff --git a/src/Gemfile.lock b/src/Gemfile.lock index 491010ca7ed..f1ff59b17de 100644 --- a/src/Gemfile.lock +++ b/src/Gemfile.lock @@ -37,26 +37,6 @@ PATH tzinfo-data unix-crypt -PATH - remote: bosh-monitor - specs: - bosh-monitor (0.0.0) - async - async-http - bosh-common - cf-uaa-lib - dogapi - io-stream - logging - nats-pure - net-smtp - openssl - ostruct - puma - riemann-client - securerandom - sinatra - PATH remote: bosh-nats-sync specs: @@ -348,7 +328,6 @@ DEPENDENCIES async-rspec bosh-common! bosh-director! - bosh-monitor! bosh-nats-sync! bundle-audit factory_bot diff --git a/src/bosh-monitor/.golangci.yml b/src/bosh-monitor/.golangci.yml new file mode 100644 index 00000000000..74b75fed6ff --- /dev/null +++ b/src/bosh-monitor/.golangci.yml @@ -0,0 +1,24 @@ +version: "2" + +linters: + default: standard + + settings: + errcheck: + exclude-functions: + - (io.Closer).Close + - (io.ReadCloser).Close + - (net.Conn).Close + - (net.Listener).Close + - (*os.File).Close + - (os/exec.Cmd).Wait + + exclusions: + rules: + - path: _test\.go + linters: + - errcheck + +formatters: + enable: + - goimports diff --git a/src/bosh-monitor/README.md b/src/bosh-monitor/README.md deleted file mode 100644 index 498bc1708d9..00000000000 --- a/src/bosh-monitor/README.md +++ /dev/null @@ -1,159 +0,0 @@ -## Synopsis - -BOSH Monitor is a component that listens to and responds to events (Heartbeats & Alerts) on the message bus (NATS). - -The Monitor also includes a few primary components: -- The Agent Monitor maintains a record of known agents (by heartbeat event subscription) -- The Director Monitor maintains a record of known agents (by director HTTP polling). -- The Agent Analyzer that analyzes agent state periodically and generates Alerts. - -The Monitor also supports generic event processing plugins that respond to Heartbeats & Alerts. - -## Heartbeat Events - -The Agent on each VM sends periodic heartbeats to the BOSH Monitor via the message bus (NATS). - -The message syntax is as follows: - -| *Subject* | *Payload* | -|---------------------------------|-----------| -| hm.agent.heartbeat.\ | none | - -## Alert Events - -A BOSH Alert is a specific type of event sent by BOSH components via the message bus. - -Alerts includes the following data: - -- Id -- Severity -- Source (usually deployment/job/index tuple) -- Timestamp -- Description -- Long description (optional) -- Tags (optional) - -## Event Handling Plugins - -Alerts are processed by a number of plugins that register to receive incoming alerts. - -Among the included plugins are: -- Event Logger - Logs all events -- Resurrector - Restarts VMs that have stopped sending heartbeat -- PagerDuty - Sends various events to PagerDuty.com using their API -- DataDog - Sends various events to DataDog.com using their API -- Emailer - Sends configurable Emails on events receipt -- Consul Event Forwarder - Sends heartbeats as events and TTL checks to a consul cluster -- EventLogger - Stores events in Director DB -- JSON Emitter - Sends metrics in JSON format to any binaries in /var/vcap/jobs/*/bin/bosh-monitor - -Plugins should conform to the following interface: - -| *Method* | *Arguments* | *Description* | -|--------------------|-------------|---------------------------------------------------------------------------------------| -| *validate_options* | | Validates the plugin configuration options | -| *run* | | Initializes the plugin process | -| *process* | event | Processes an event (Bosh::Monitor::Events::Heartbeat or Bosh::Monitor::Events::Alert) | - -The event processor handles deduping duplicate events. - -Plugins are notified in the order that they were registered (based on configuration order). - -## Agent Monitor - Heartbeat Event Processing - -The Agent Monitor listens for heartbeat events on the message bus and handles them in the following way: - -- If the Agent is known to the Monitor then the last heartbeat timestamp gets updated. -- If the Agent is unknown to the Monitor then it is recorded with a flag that marks it as a "rogue agent". - -No analysis is performed when a heartbeat is received. The Agent Analyzer process and Director Monitor polling are asynchronous to heartbeat event processing by the Agent Monitor. - -## Director Monitor - Agent Discovery - -The Director Monitor polls the Director periodically via HTTP to get the list of managed VMs. - -The message syntax is as follows: - -| *Method* | *Endpoint* | *Response* | -|--------------------------------------|------------|---------------------------------------------------------------------| -| /deployments/\/vms | GET | JSON including agent ids, job names and indices for all managed VMs | - -- If a new agent is discovered via polling then it is recorded by the Monitor as part of the managed deployment. -- If a "rogue agent" is discovered via polling then its "rogue agent" flag is cleared. - -The Director Monitor does not actively poll the agents themselves, just the Director. The Director Monitor simply remembers the state of the world as reported by polling and event processing so that the difference can be analyzed. - -## Agent Analyzer - -The Agent Analyzer is a periodic process that generates "Agent Missing" alerts. - -If an agent's heartbeat timestamp is not updated within the configured time period, the Agent Analyzer process will generate an "Agent Missing" alert. - -Both known VM agents and rogue agents may send "Agent Missing" alerts, but they have different configurable time periods. - -## Alerts from BOSH Agent - -The Monitor subscribes to Agent alerts of the following format: - -| *Subject* | *Payload* | -|-----------------------------|----------------------------------------------------------------------------------------------| -| hm.agent.alert.\ | JSON containing the following keys: id, service, event, action, description, timestamp, tags | - -BOSH Agent is responsible for mapping any underlying supervisor alert format to the expected JSON payload and sending it to BOSH Monitor. - -The Monitor is responsible for interpreting the JSON payload and mapping it to a sequence of Monitor & Plugin actions, possibly generating new alerts that bypass the message bus. Malformed payloads are ignored. - -Job name and index are not part of alerts from the Agent, those are looked up in the Director. If heartbeat came from a rogue agent, and we have no job name and/or index then we note that fact in the alert description but don't try to be too worried about that (service name and agent id should be enough). We might consider including agent IP address as a part of heartbeat so we can track down rogue agents. - -## Authoring new health monitoring plugins - -There are many existing ways to communicate health alerts to the external world. If you need an additional method then you can create new `bosh-monitor` plugins. - -The following infrastructures are for developing/testing your new plugin into a new single-server BOSH: - -1. Clone bosh repo and install dependencies - - ``` - git clone https://github.com/cloudfoundry/bosh.git - cd bosh - bundle install - ``` - -2. Run the current `bosh-monitor` tests to ensure they currently all pass - - ``` - cd bosh-monitor - rspec - ``` - -3. Create a `plugin.rb` for your plugin extension https://github.com/cloudfoundry/bosh/tree/master/bosh-monitor/lib/bosh/monitor/plugins -4. Create a matching test file for your plugin extension `spec.rb` https://github.com/cloudfoundry/bosh/tree/master/bosh-monitor/spec/unit/bosh/monitor/plugins -5. Write tests and make them pass - - ``` - rspec - ``` - -6. Allow configuration to be passed into the `health_monitor` job template to activate and configure your plugin https://github.com/cloudfoundry/bosh/blob/master/release/jobs/health_monitor/spec and https://github.com/cloudfoundry/bosh/blob/master/release/jobs/health_monitor/templates/health_monitor.yml.erb -7. Write / update tests in https://github.com/cloudfoundry/bosh/blob/master/release/spec/health_monitor.yml.erb_spec.rb and make them pass. -8. Run the rake task to create a bosh release of your modified `bosh` - - ``` - rake release:create_dev_release - ``` - -9. Upload to your director - - ``` - rake release:upload_dev_release - ``` - -10. Construct a deployment manifest to deploy a new bosh https://github.com/cloudfoundry/bosh/blob/master/release/examples/bosh-openstack-solo.yml -11. Deploy - - ``` - bosh deployment path/to/manifest.yml - bosh deploy - ``` - -12. Externally test your plugin -> external thing diff --git a/src/bosh-monitor/bin/bosh-monitor b/src/bosh-monitor/bin/bosh-monitor deleted file mode 100755 index 7881663c4f0..00000000000 --- a/src/bosh-monitor/bin/bosh-monitor +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env ruby - -require 'bosh/monitor' -require 'optparse' - -config_file = nil - -opts = OptionParser.new do |opts| - opts.on('-c', '--config FILE', 'configuration file') do |opt| - config_file = opt - end -end - -opts.parse!(ARGV.dup) - -if config_file.nil? - puts opts - exit 1 -end - -runner = Bosh::Monitor::Runner.new(config_file) - -Signal.trap('INT') do - runner.stop -end - -Signal.trap('EXIT') do - Bosh::Monitor.logger.info('HealthMonitor exiting!') -end - -runner.run diff --git a/src/bosh-monitor/bin/bosh-monitor-console b/src/bosh-monitor/bin/bosh-monitor-console deleted file mode 100755 index 1cea5884006..00000000000 --- a/src/bosh-monitor/bin/bosh-monitor-console +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env ruby - -require 'bosh/monitor' -require 'irb' -require 'irb/completion' - -module Bosh - module Monitor - class Console - include YamlHelper - - def self.start(context) - new.start(context) - end - - def start(_context) - config_file = nil - - opts = OptionParser.new do |opt| - opt.on('-c', '--config [ARG]', 'configuration file') { |c| config_file = c } - end - - opts.parse!(ARGV) - - if config_file.nil? - puts opts - exit 1 - end - - puts "=> Loading #{config_file}" - Bosh::Monitor.config = load_yaml_file(config_file) - - begin - require 'ruby-debug' - puts '=> Debugger enabled' - rescue LoadError - puts '=> ruby-debug not found, debugger disabled' - end - - puts '=> Welcome to BOSH Health Monitor console' - - IRB.start - end - end - end -end - -Bosh::Monitor::Console.start(self) diff --git a/src/bosh-monitor/bosh-monitor.gemspec b/src/bosh-monitor/bosh-monitor.gemspec deleted file mode 100644 index f97203c96c0..00000000000 --- a/src/bosh-monitor/bosh-monitor.gemspec +++ /dev/null @@ -1,47 +0,0 @@ -# coding: utf-8 -require File.expand_path('../lib/bosh/monitor/version', __FILE__) - -Gem::Specification.new do |spec| - spec.name = 'bosh-monitor' - spec.version = Bosh::Monitor::VERSION - spec.platform = Gem::Platform::RUBY - spec.summary = 'BOSH Health Monitor' - spec.description = 'BOSH Health Monitor' - - spec.authors = ['Cloud Foundry'] - spec.email = ['support@cloudfoundry.com'] - spec.homepage = 'https://github.com/cloudfoundry/bosh' - spec.license = 'Apache-2.0' - spec.required_ruby_version = '>= 3.0.0' - - spec.files = Dir['lib/**/*'].select { |f| File.file?(f) } - spec.test_files = Dir['spec/**/*'].select { |f| File.file?(f) } - - spec.bindir = 'bin' - spec.executables = ['bosh-monitor', 'bosh-monitor-console'] - spec.require_paths = ['lib'] - - spec.add_dependency 'async' - spec.add_dependency 'async-http' - spec.add_dependency 'bosh-common' - spec.add_dependency 'cf-uaa-lib' - spec.add_dependency 'io-stream' - spec.add_dependency 'logging' - spec.add_dependency 'nats-pure' - spec.add_dependency 'openssl' - spec.add_dependency 'ostruct' - spec.add_dependency 'puma' - spec.add_dependency 'securerandom' - spec.add_dependency 'sinatra' - - spec.add_dependency 'dogapi' - spec.add_dependency 'net-smtp' - spec.add_dependency 'riemann-client' - - spec.add_development_dependency 'async-rspec' - spec.add_development_dependency 'rack-test' - spec.add_development_dependency 'rspec' - spec.add_development_dependency 'simplecov' - spec.add_development_dependency 'timecop' - spec.add_development_dependency 'webmock' -end diff --git a/src/bosh-monitor/cmd/plugins/hm-consul/consul_test.go b/src/bosh-monitor/cmd/plugins/hm-consul/consul_test.go new file mode 100644 index 00000000000..fd79a96fdb6 --- /dev/null +++ b/src/bosh-monitor/cmd/plugins/hm-consul/consul_test.go @@ -0,0 +1,141 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "github.com/cloudfoundry/bosh/src/bosh-monitor/cmd/plugins/pluginlib" + "github.com/cloudfoundry/bosh/src/bosh-monitor/cmd/plugins/plugintestutil" + "github.com/cloudfoundry/bosh/src/bosh-monitor/pkg/pluginproto" +) + +const cmdTimeout = 3 * time.Second + +// Ruby: "validates options" — missing host/port/protocol → error +func TestConsulMissingOptions(t *testing.T) { + for name, opts := range map[string]map[string]interface{}{ + "missing host": {"port": float64(8500), "protocol": "http"}, + "missing port": {"host": "localhost", "protocol": "http"}, + "missing protocol": {"host": "localhost", "port": float64(8500)}, + } { + opts := opts + stdinR, stdinW := io.Pipe() + stdoutR, stdoutW := io.Pipe() + _ = plugintestutil.CmdSink(stdoutR) + errCh := make(chan error, 1) + go func() { errCh <- pluginlib.RunWithIO(stdinR, stdoutW, runConsul) }() + plugintestutil.SendInit(t, stdinW, opts) + stdinW.Close() + select { + case err := <-errCh: + if err == nil { + t.Errorf("%s: expected startup error, got nil", name) + } + case <-time.After(3 * time.Second): + t.Fatalf("%s: timed out", name) + } + } +} + +// Ruby: alert event with events=true → PUT to /v1/event/fire/