diff --git a/.github/config/hive/clients.yaml b/.github/config/hive/clients.yaml new file mode 100644 index 00000000..46e8af76 --- /dev/null +++ b/.github/config/hive/clients.yaml @@ -0,0 +1,4 @@ +- client: ethlambda + nametag: devnet4 +- client: ethlambda + nametag: devnet5 diff --git a/.github/scripts/check-hive-results.sh b/.github/scripts/check-hive-results.sh new file mode 100644 index 00000000..d1cb3e21 --- /dev/null +++ b/.github/scripts/check-hive-results.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +set -euo pipefail + +results_dir="${1:-src/results}" + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to parse Hive results but was not found in PATH" + exit 1 +fi + +if [ ! -d "${results_dir}" ]; then + echo "Hive results directory '${results_dir}' not found" + exit 1 +fi + +shopt -s nullglob +json_files=("${results_dir}"/*.json) +shopt -u nullglob + +if [ "${#json_files[@]}" -eq 0 ]; then + echo "No Hive JSON result files found in ${results_dir}" + exit 1 +fi + +failures=0 +failed_logs_root="${results_dir}/failed_logs" +rm -rf "${failed_logs_root}" +mkdir -p "${failed_logs_root}" + +for json_file in "${json_files[@]}"; do + if [[ "${json_file}" == *"hive.json" ]]; then + continue + fi + + suite_name="$(jq -r '.name // empty' "${json_file}")" + suite_label="${suite_name:-$(basename "${json_file}" .json)}" + failed_cases="$(jq '[.testCases[]? | select(.summaryResult.pass != true)] | length' "${json_file}")" + + if [ "${failed_cases}" -eq 0 ]; then + continue + fi + + failures=$((failures + failed_cases)) + echo "Detected ${failed_cases} failing Hive test case(s) in ${suite_label}" + + failure_list="$( + jq -r ' + .testCases[]? + | select(.summaryResult.pass != true) + | . as $case + | ($case.summaryResult // {}) as $summary + | ($summary.message // $summary.reason // $summary.error // "") as $message + | "- " + ($case.name // "unknown test") + + (if $message != "" then ": " + $message else "" end) + ' "${json_file}" + )" + + printf '%s\n' "${failure_list}" + + suite_slug="$(printf '%s' "${suite_label}" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9._-]+/-/g; s/^-//; s/-$//')" + suite_dir="${failed_logs_root}/${suite_slug:-suite}" + mkdir -p "${suite_dir}" + + cp "${json_file}" "${suite_dir}/" + printf '%s\n\n%s\n' "Detected ${failed_cases} failing Hive test case(s) in ${suite_label}" "${failure_list}" > "${suite_dir}/failed-tests.txt" + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Hive failures: ${suite_label}" + printf '%s\n' "${failure_list}" + echo + } >> "${GITHUB_STEP_SUMMARY}" + fi +done + +if [ "${failures}" -gt 0 ]; then + echo "Hive reported ${failures} failing test case(s) in total" + exit 1 +fi + +echo "Hive reported no failing test cases." diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c472fd7e..a6fc85bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,5 +72,212 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Run fixture-based tests - uses: ./.github/actions/run-fixture-tests + - name: Get leanSpec fixtures release info + id: fixtures-release + run: | + api_url="https://api.github.com/repos/leanEthereum/leanSpec/releases/latest" + json=$(curl -sL "$api_url") + fixtures_url=$(echo "$json" | python3 -c "import sys,json; j=json.load(sys.stdin); print(next(a.get('browser_download_url') for a in j.get('assets',[]) if a.get('name')=='fixtures-prod-scheme.tar.gz'))") + sha_url=$(echo "$json" | python3 -c "import sys,json; j=json.load(sys.stdin); print(next(a.get('browser_download_url') for a in j.get('assets',[]) if a.get('name')=='fixtures-prod-scheme.tar.gz.sha256'))") + sha=$(curl -sL "$sha_url" | cut -d' ' -f1) + { + echo "url=$fixtures_url" + echo "sha_url=$sha_url" + echo "sha=$sha" + } >> $GITHUB_OUTPUT + + - name: Restore test fixtures cache + id: cache-fixtures + uses: actions/cache/restore@v5 + with: + path: leanSpec/fixtures + key: leanspec-fixtures-${{ steps.fixtures-release.outputs.sha }} + + - name: Download leanSpec fixtures release + id: download-fixtures + if: steps.cache-fixtures.outputs.cache-hit != 'true' + run: | + tmpdir=$(mktemp -d) + trap 'rm -rf "$tmpdir"' EXIT + fixtures_url="${{ steps.fixtures-release.outputs.url }}" + sha_url="${{ steps.fixtures-release.outputs.sha_url }}" + echo "Downloading fixtures from $fixtures_url" + curl -L -f -o "$tmpdir/fixtures-prod-scheme.tar.gz" "$fixtures_url" + curl -L -f -o "$tmpdir/fixtures-prod-scheme.tar.gz.sha256" "$sha_url" + expected=$(cut -d' ' -f1 "$tmpdir/fixtures-prod-scheme.tar.gz.sha256") + actual=$(sha256sum "$tmpdir/fixtures-prod-scheme.tar.gz" | awk '{print $1}') + if [ "$expected" != "$actual" ]; then + echo "SHA256 mismatch: expected $expected, got $actual" + exit 1 + fi + rm -rf leanSpec/fixtures + mkdir -p leanSpec/fixtures + tar -xzf "$tmpdir/fixtures-prod-scheme.tar.gz" -C leanSpec/fixtures --strip-components=1 + + # Save fixtures only when the download actually SUCCEEDED, so a + # cancelled or failed download never persists a partial fixture set, + # while still saving even if the later Rust test step fails. + - name: Save test fixtures cache + if: >- + always() + && steps.cache-fixtures.outputs.cache-hit != 'true' + && steps.download-fixtures.outcome == 'success' + uses: actions/cache/save@v5 + with: + path: leanSpec/fixtures + key: ${{ steps.cache-fixtures.outputs.cache-primary-key }} + + # Ensure make sees fixtures as up-to-date (its timestamp must be + # newer than leanSpec/, which intermediate steps may have modified). + - name: Mark fixtures as up-to-date + run: touch leanSpec/fixtures + + - name: Setup Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: "1.92.0" + + - name: Setup cache + uses: Swatinem/rust-cache@v2 + + - name: Run tests + run: make test + + docker_build: + name: Build Docker + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build ethlambda Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + build-args: | + GIT_COMMIT=${{ github.sha }} + GIT_BRANCH=${{ github.ref_name }} + push: false + tags: | + ghcr.io/lambdaclass/ethlambda:devnet4 + ghcr.io/lambdaclass/ethlambda:devnet5 + outputs: type=docker,dest=/tmp/ethlambda_image.tar + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Upload image artifacts + uses: actions/upload-artifact@v7 + with: + name: ethlambda_image + path: /tmp/ethlambda_image.tar + + run-hive: + name: Hive - ${{ matrix.name }} + runs-on: ubuntu-latest + permissions: + contents: read + needs: docker_build + strategy: + fail-fast: false + matrix: + include: + - name: "Lean RPC Compat" + limit: "rpc-compat" + artifact_prefix: lean_rpc_compat + - name: "Lean Sync" + limit: "sync" + artifact_prefix: lean_sync + - name: "Lean Client Interop" + limit: "client-interop" + artifact_prefix: lean_client_interop + - name: "Lean Validation" + limit: "validation" + artifact_prefix: lean_validation + - name: "Lean Gossip" + limit: "gossip" + artifact_prefix: lean_gossip + - name: "Lean ReqResp" + limit: "reqresp" + artifact_prefix: lean_reqresp + - name: "Lean Fork Choice Spec Tests" + limit: "lean-spec-tests-fork-choice" + artifact_prefix: lean_spec_fork_choice + - name: "Lean State Transition Spec Tests" + limit: "lean-spec-tests-state-transition" + artifact_prefix: lean_spec_state_transition + - name: "Lean Verify Signatures Spec Tests" + limit: "lean-spec-tests-verify-signatures" + artifact_prefix: lean_spec_verify_signatures + steps: + - uses: actions/checkout@v6 + + - name: Download ethlambda image artifact + uses: actions/download-artifact@v6 + with: + name: ethlambda_image + path: /tmp + + - name: Load image + run: docker load --input /tmp/ethlambda_image.tar + + - name: Load hive client config + id: client-config + shell: bash + run: | + { + echo "config<> "$GITHUB_OUTPUT" + + - name: Determine hive flags + id: hive-flags + shell: bash + env: + SIM_LIMIT: ${{ matrix.limit }} + run: | + FLAGS="--sim.parallelism 4 --sim.loglevel 3" + if [[ -n "$SIM_LIMIT" ]]; then + escaped_limit=${SIM_LIMIT//\'/\'\\\'\'} + FLAGS+=" --sim.limit '$escaped_limit'" + fi + echo "flags=$FLAGS" >> "$GITHUB_OUTPUT" + + - name: Run Hive Simulation + uses: ethpandaops/hive-github-action@09050fc75bae8a10b1876a0826ca4114ea9ed6e8 # v0.6.3 + with: + hive_repository: ethereum/hive + hive_version: dde4f59d04ff0ff8b6585670b08cea1b6c8ab65c + simulator: lean + client: ethlambda_devnet4 + client_config: ${{ steps.client-config.outputs.config }} + extra_flags: ${{ steps.hive-flags.outputs.flags }} + + - name: Check Hive Results For Failures + id: verify-hive-results + if: ${{ success() }} + shell: bash + run: bash ./.github/scripts/check-hive-results.sh src/results + + - name: Upload Hive Failure Logs + if: ${{ failure() && steps.verify-hive-results.conclusion == 'failure' }} + uses: actions/upload-artifact@v7 + with: + name: hive_failed_logs_${{ matrix.artifact_prefix }} + path: src/results/failed_logs + if-no-files-found: warn + + - name: Upload Hive Run Artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@v7 + with: + name: hive_run_artifacts_${{ matrix.artifact_prefix }} + path: | + src/hive.log + src/results + if-no-files-found: warn