diff --git a/.taskcluster.yml b/.taskcluster.yml index af4d853c9c..30f8d5bb3f 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -32,8 +32,24 @@ tasks: then: ${event.pull_request.body} taskboot_image: "mozilla/taskboot:0.5.2" + + # The files a push adds, changes or removes. + changed_paths: + $if: 'tasks_for == "github-push"' + then: + $flattenDeep: + $map: { $eval: "event.commits" } + each(c): + - { $eval: "c.added" } + - { $eval: "c.modified" } + - { $eval: "c.removed" } + else: [] in: $let: + # Whether a push changes the bug-fix agent image or what goes into it. + touches_bug_fix_image: + $eval: '" agents/bug-fix/" in (" " + join(changed_paths, " ")) || " libs/agent-tools/" in (" " + join(changed_paths, " ")) || " libs/hackbot-runtime/" in (" " + join(changed_paths, " ")) || " libs/phabricator-client/" in (" " + join(changed_paths, " ")) || " libs/phabricator-proxy/" in (" " + join(changed_paths, " ")) || "uv.lock" in changed_paths || "pyproject.toml" in changed_paths || ".taskcluster.yml" in changed_paths' + default_task_definition: created: { $fromNow: "" } deadline: { $fromNow: "1 hour" } @@ -287,6 +303,179 @@ tasks: name: bugbug hackbot-ui tests description: bugbug hackbot-ui tests + # Agent images use BuildKit features and a non-final target, which taskboot's + # build commands don't support, so call podman (what taskboot uses) directly. + # Its storage is a cache, so unchanged layers (system packages, mostly) are + # reused; layers left over from older builds are pruned after a week. + - $mergeDeep: + - { $eval: default_task_definition } + - taskId: { $eval: as_slugid("bug_fix_image_build") } + workerType: ci + payload: + capabilities: + privileged: true + image: "${taskboot_image}" + cache: + bugbug-bug-fix-image-podman: /var/lib/containers + command: + - "/bin/sh" + - "-cxe" + - "git clone --quiet ${repository} /code && + cd /code && + git -c advice.detachedHead=false checkout ${head_rev} && + podman build --target agent -f agents/bug-fix/Dockerfile -t mozilla/bugbug-hackbot-agent-bug-fix:latest . && + mkdir /images && + podman save --format docker-archive mozilla/bugbug-hackbot-agent-bug-fix:latest | zstd -q -o /images/bugbug-hackbot-agent-bug-fix.tar.zst && + podman image prune --force --filter until=168h" + artifacts: + public/bug-fix-agent: + expires: { $fromNow: "2 weeks" } + path: /images + type: directory + scopes: + - docker-worker:capability:privileged + - docker-worker:cache:bugbug-bug-fix-image-podman + - generic-worker:cache:bugbug-bug-fix-image-podman + metadata: + name: bugbug bug-fix agent image build + description: bugbug bug-fix agent image build + + # Runs inside the image, where the entrypoint has started the display and + # audio services. The tests download Firefox, minidump-stackwalk and Python + # packages, so failures are retried once to ride out network hiccups. + # TASKCLUSTER_ROOT_URL is unset as it is where the agent runs: mach would + # otherwise look Firefox's toolchains up on this cluster. + - $mergeDeep: + - { $eval: default_task_definition } + - taskId: { $eval: as_slugid("bug_fix_image_tests") } + dependencies: + - { $eval: as_slugid("bug_fix_image_build") } + payload: + image: + type: task-image + path: public/bug-fix-agent/bugbug-hackbot-agent-bug-fix.tar.zst + taskId: { $eval: as_slugid("bug_fix_image_build") } + command: + - "/bin/bash" + - "-cx" + - "git clone --quiet ${repository} /tmp/bugbug && + cd /tmp/bugbug && + git -c advice.detachedHead=false checkout ${head_rev} && + /usr/local/bin/python3 -m pip install --quiet --disable-pip-version-check --target /tmp/pydeps pytest==9.1.0 pytest-asyncio==1.4.0 && + cd agents/bug-fix && + unset TASKCLUSTER_ROOT_URL && + export PYTHONPATH=/tmp/pydeps BUG_FIX_IMAGE_INSIDE=1 && + (python -m pytest -v -rs tests/test_image.py -k 'not artifact_build and not android' || + python -m pytest -v -rs --last-failed tests/test_image.py -k 'not artifact_build and not android')" + metadata: + name: bugbug bug-fix agent image tests + description: bugbug bug-fix agent image tests + + # Bootstraps a Firefox checkout, makes an artifact build and runs tests in it. + # Too slow for every pull request, it runs after merges that change the + # image or what goes into it. + - $if: 'tasks_for == "github-push" && head_branch == "refs/heads/master" && touches_bug_fix_image' + then: + $mergeDeep: + - { $eval: default_task_definition } + - taskId: { $eval: as_slugid("bug_fix_image_slow_tests") } + dependencies: + - { $eval: as_slugid("bug_fix_image_build") } + workerType: compute-large + payload: + image: + type: task-image + path: public/bug-fix-agent/bugbug-hackbot-agent-bug-fix.tar.zst + taskId: { $eval: as_slugid("bug_fix_image_build") } + command: + - "/bin/bash" + - "-cx" + - "git clone --quiet ${repository} /tmp/bugbug && + cd /tmp/bugbug && + git -c advice.detachedHead=false checkout ${head_rev} && + /usr/local/bin/python3 -m pip install --quiet --disable-pip-version-check --target /tmp/pydeps pytest==9.1.0 pytest-asyncio==1.4.0 && + cd agents/bug-fix && + unset TASKCLUSTER_ROOT_URL && + export PYTHONPATH=/tmp/pydeps BUG_FIX_IMAGE_INSIDE=1 && + (python -m pytest -s -v -rs tests/test_image.py -k artifact_build || + python -m pytest -s -v -rs --last-failed tests/test_image.py -k artifact_build)" + metadata: + name: bugbug bug-fix agent image slow tests + description: bugbug bug-fix agent image slow tests + + # Like the slow tests, but a full build with the agent's own mozconfig, as + # the agent makes. Too slow for every pull request, it runs after merges + # that change the image or what goes into it; a failure is not retried. + - $if: 'tasks_for == "github-push" && head_branch == "refs/heads/master" && touches_bug_fix_image' + then: + $mergeDeep: + - { $eval: default_task_definition } + - taskId: { $eval: as_slugid("bug_fix_image_full_build_tests") } + dependencies: + - { $eval: as_slugid("bug_fix_image_build") } + deadline: { $fromNow: "8 hours" } + # Its objdir alone takes over 30 GiB, more than other pools' disks hold. + workerType: firefox-build + payload: + maxRunTime: 21600 + env: + BUG_FIX_IMAGE_FULL_BUILD: "1" + image: + type: task-image + path: public/bug-fix-agent/bugbug-hackbot-agent-bug-fix.tar.zst + taskId: { $eval: as_slugid("bug_fix_image_build") } + command: + - "/bin/bash" + - "-cx" + - "git clone --quiet ${repository} /tmp/bugbug && + cd /tmp/bugbug && + git -c advice.detachedHead=false checkout ${head_rev} && + /usr/local/bin/python3 -m pip install --quiet --disable-pip-version-check --target /tmp/pydeps pytest==9.1.0 pytest-asyncio==1.4.0 && + cd agents/bug-fix && + unset TASKCLUSTER_ROOT_URL && + export PYTHONPATH=/tmp/pydeps BUG_FIX_IMAGE_INSIDE=1 && + python -m pytest -s -v -rs tests/test_image.py -k full_build" + metadata: + name: bugbug bug-fix agent image full build tests + description: bugbug bug-fix agent image full build tests + + # Bootstraps a Firefox checkout for Android, makes an artifact build and + # runs tests in it. + # Too slow for every pull request, it runs after merges that change the + # image or what goes into it. + - $if: 'tasks_for == "github-push" && head_branch == "refs/heads/master" && touches_bug_fix_image' + then: + $mergeDeep: + - { $eval: default_task_definition } + - taskId: { $eval: as_slugid("bug_fix_image_android_tests") } + dependencies: + - { $eval: as_slugid("bug_fix_image_build") } + deadline: { $fromNow: "3 hours" } + # The Android SDK and NDK take ~14 GiB, which leaves other pools' disks + # nearly full. + workerType: firefox-build + payload: + maxRunTime: 7200 + image: + type: task-image + path: public/bug-fix-agent/bugbug-hackbot-agent-bug-fix.tar.zst + taskId: { $eval: as_slugid("bug_fix_image_build") } + command: + - "/bin/bash" + - "-cx" + - "git clone --quiet ${repository} /tmp/bugbug && + cd /tmp/bugbug && + git -c advice.detachedHead=false checkout ${head_rev} && + /usr/local/bin/python3 -m pip install --quiet --disable-pip-version-check --target /tmp/pydeps pytest==9.1.0 pytest-asyncio==1.4.0 && + cd agents/bug-fix && + unset TASKCLUSTER_ROOT_URL && + export PYTHONPATH=/tmp/pydeps BUG_FIX_IMAGE_INSIDE=1 && + (python -m pytest -s -v -rs tests/test_image.py -k android || + python -m pytest -s -v -rs --last-failed tests/test_image.py -k android)" + metadata: + name: bugbug bug-fix agent image Android tests + description: bugbug bug-fix agent image Android tests + - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"' then: $mergeDeep: diff --git a/agents/bug-fix/Dockerfile b/agents/bug-fix/Dockerfile index 558c6c5774..b1eb74a5c5 100644 --- a/agents/bug-fix/Dockerfile +++ b/agents/bug-fix/Dockerfile @@ -17,9 +17,15 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,target=/app,rw \ uv sync --locked --no-dev --no-editable --package hackbot-agent-bug-fix +# profiler-cli, the Firefox Profiler's command-line analyzer. It runs with the +# node `mach bootstrap` installs, so this stage only installs it. +FROM node:24-trixie-slim AS profiler-cli +ARG PROFILER_CLI_VERSION=0.10.0 +RUN npm install --global --prefix /opt/profiler-cli --no-fund --no-audit \ + @firefox-devtools/profiler-cli@${PROFILER_CLI_VERSION} + FROM python:3.14 AS base -COPY --from=builder /opt/venv /opt/venv WORKDIR /app ENV PYTHONUNBUFFERED=1 @@ -28,33 +34,103 @@ ENV PATH="/opt/venv/bin:$PATH" FROM base AS agent +# The agent both builds Firefox and runs it, so it needs the build prerequisites +# plus the browser's runtime libraries. `mach bootstrap` installs the *build* +# toolchain into ~/.mozbuild, but runs unprivileged here, so it cannot install +# these system packages itself. RUN apt-get update \ && apt-get install -y --no-install-recommends \ + # ./mach needs mercurial for its vcs checks even on a git checkout. + mercurial git curl zip unzip bzip2 ca-certificates \ + # Debugging and inspecting: crashes, syscalls, JSON from mach and CI, + # profile databases. + gdb strace ripgrep jq sqlite3 \ # Firefox runtime deps. - libgtk-3-0 libasound2t64 \ - ffmpeg \ - fluxbox \ - xvfb \ + libgtk-3-0t64 libdbus-glib-1-2 libx11-xcb1 libxcb1 libxcb-shm0 \ + libxcb-render0 libxcb-glx0 libxcb-shape0 libxtst6 libxt6 libpci3 \ + libasound2t64 libpulse0 libpango-1.0-0 libcairo2 libxcomposite1 \ + libxdamage1 libxfixes3 libxrandr2 libxkbcommon0 libnss3 libnspr4 \ + # Firefox renders through EGL on X11; Mesa's software one under Xvfb. + libegl1 libegl-mesa0 \ + # Fonts. Beyond the basics, the scripts CI's test images cover (Arabic, + # math, Korean, Japanese), so text renders like it does there. + fontconfig fonts-liberation fonts-dejavu-core fonts-kacst-one \ + fonts-stix fonts-unfonts-core fonts-vlgothic \ + # Names libc frames in crash stacks. + libc6-dbg \ + # Codecs, plus a PipeWire server used as a dummy audio sink/source by + # media tests, and pactl to inspect it. + ffmpeg pipewire wireplumber pipewire-pulse pulseaudio-utils \ + # Headless display and session bus, started by docker-entrypoint.sh. + dbus-x11 x11-xserver-utils fluxbox xvfb \ && rm -rf /var/lib/apt/lists/* +# Gecko media tests use the pipewire audiotestsrc plugin as a dummy sound +# source; it ships disabled. See firefox's +# taskcluster/docker/recipes/ubuntu2404-test-system-setup-base.sh. +RUN install -d -o root -g root -m 755 /etc/pipewire \ + && sed -e '/^context.spa-libs = {/,/^}$/ s/#\(audiotestsrc\)/\1/' \ + /usr/share/pipewire/pipewire.conf > /etc/pipewire/pipewire.conf + +# The Firefox tree's agent skills profile with samply and analyze profiles with +# the Firefox Profiler's profiler-cli. samply and node come from `mach +# bootstrap`, profiler-cli from the stage above. +COPY --from=profiler-cli /opt/profiler-cli /opt/profiler-cli + +# The tree's skills run Python helper scripts with `uv run`. +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ + # hackbot.toml lives at the agent root (not inside the package), so copy it into # the working dir; the runtime discovers it there (cwd) at startup. COPY agents/bug-fix/hackbot.toml /app/hackbot.toml +COPY agents/bug-fix/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh RUN useradd --create-home --shell /bin/bash agent \ && mkdir -p /workspace \ - && chown agent:agent /workspace + && chown agent:agent /workspace \ + # Xvfb runs unprivileged and cannot create its own socket directory. + && mkdir -p /tmp/.X11-unix \ + && chmod 1777 /tmp/.X11-unix \ + # Same identity hackbot_runtime commits with, so the agent's own `git commit` + # and `git stash` work too instead of failing for lack of one. + && git config --system user.name "Hackbot" \ + && git config --system user.email "hackbot@mozilla.tld" + +ENV SCREEN_WIDTH=1920 +ENV SCREEN_HEIGHT=1080 +ENV SCREEN_DEPTH=24 +# For Firefox started from the agent's shell (`./mach run`, mochitests); grizzly +# starts its own Xvfb for evaluate_testcase. +ENV DISPLAY=:99 +# Home of the session bus and the PipeWire/PulseAudio sockets. +ENV XDG_RUNTIME_DIR=/tmp/runtime-agent +ENV DBUS_SESSION_BUS_ADDRESS=unix:path=/tmp/runtime-agent/bus # `mach bootstrap` installs the toolchain here at runtime; put it on PATH so the -# agent's own `./mach build` (and the build_firefox tool) find rustc/clang. -ENV PATH="/home/agent/.cargo/bin:/home/agent/.mozbuild/clang/bin:${PATH}" +# agent's own `./mach build` (and the build_firefox tool) find rustc/clang, +# evaluate_testcase finds minidump-stackwalk to symbolize crashes, and the +# agent's shell finds node, samply and the developer tools (in ~/.cargo/bin), +# and the profiler-cli that runs with that node. +ENV PATH="/home/agent/.cargo/bin:/home/agent/.mozbuild/clang/bin:/home/agent/.mozbuild/minidump-stackwalk:/home/agent/.mozbuild/node/bin:/home/agent/.mozbuild/samply:/opt/profiler-cli/bin:${PATH}" +# Likewise for Android, once bootstrapped for it: java, adb, aapt2, apksigner, +# apkanalyzer, lint, sdkmanager. bootstrap_firefox links the versioned +# directories they are in as `current`/`latest`. +ENV PATH="${PATH}:/home/agent/.mozbuild/jdk/current/bin:/home/agent/.mozbuild/android-sdk-linux/platform-tools:/home/agent/.mozbuild/android-sdk-linux/build-tools/latest:/home/agent/.mozbuild/android-sdk-linux/cmdline-tools/latest/bin" + +# Last, as it changes with any Python change: the layers above (system +# packages, mostly) stay cached across those. +COPY --from=builder /opt/venv /opt/venv USER agent +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] CMD ["python", "-m", "hackbot_agents.bug_fix"] FROM base AS broker +COPY --from=builder /opt/venv /opt/venv + RUN useradd --create-home --shell /bin/bash broker USER broker diff --git a/agents/bug-fix/docker-entrypoint.sh b/agents/bug-fix/docker-entrypoint.sh new file mode 100755 index 0000000000..4747bfb24a --- /dev/null +++ b/agents/bug-fix/docker-entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# Give Firefox what it expects from a desktop session: an X display for anything +# launched from the agent's shell (`./mach run`, mochitests), and a session bus +# plus PipeWire so Web Audio and media tests have a working audio device. +set -e + +mkdir -p -m 700 "$XDG_RUNTIME_DIR" + +Xvfb "$DISPLAY" -screen 0 "${SCREEN_WIDTH}x${SCREEN_HEIGHT}x${SCREEN_DEPTH}" -nolisten tcp & + +dbus-daemon --session --address="$DBUS_SESSION_BUS_ADDRESS" --fork --nopidfile + +# Their logs are noisy (no system bus, bluetooth or cameras here) and not useful +# to the agent. +pipewire >/dev/null 2>&1 & +wireplumber >/dev/null 2>&1 & +pipewire-pulse >/dev/null 2>&1 & + +# The agent may start Firefox seconds after boot, so wait for the display and the +# audio server to accept connections rather than racing them. +i=0 +while [ "$i" -lt 50 ]; do + xset -display "$DISPLAY" q >/dev/null 2>&1 && [ -S "$XDG_RUNTIME_DIR/pulse/native" ] && break + i=$((i + 1)) + sleep 0.2 +done + +exec "$@" diff --git a/agents/bug-fix/hackbot_agents/bug_fix/agent.py b/agents/bug-fix/hackbot_agents/bug_fix/agent.py index c5478494b4..854c81a8be 100644 --- a/agents/bug-fix/hackbot_agents/bug_fix/agent.py +++ b/agents/bug-fix/hackbot_agents/bug_fix/agent.py @@ -133,6 +133,8 @@ def _write_mozconfig(fx_ctx: FirefoxContext) -> None: "ac_add_options --enable-debug\n" "ac_add_options --enable-optimize\n" "ac_add_options --enable-clang-plugin\n" + # For the evaluate_js_shell tool: desktop builds skip the JS shell. + "ac_add_options --enable-js-shell\n" f"mk_add_options MOZ_OBJDIR={quote(str(fx_ctx.objdir))}\n" ) diff --git a/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md b/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md index 6eadb30e0e..42d17b388d 100644 --- a/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md +++ b/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md @@ -30,7 +30,9 @@ When a bug needs reproducing or fixing, you have `firefox` MCP tools: - `evaluate_testcase(content, filename, prefs?, timeout?)` — runs a testcase in Firefox under xvfb via grizzly (the build's sanitizer configuration depends on the configured mozconfig). Returns `crashed` (bool), `crashed_parent` (bool — parent-process crash), `logs` (stderr/stdout and, if crashed, `crashdata`), and the testcase bundle. When `crashed=false`, inspect `logs.stderr` for why (JS exception, gated pref, etc). - `build_firefox()` — runs `./mach build` with the configured mozconfig. Slow. Only call this if you've patched source or the binary is missing — check with Bash first. -- `bootstrap_firefox()` — runs `./mach bootstrap` to install the build toolchain (rustc, cargo, clang, cbindgen). The image does not ship one, so if `build_firefox` reports a missing toolchain (e.g. no rustc or clang), call this once and then build again. Do not give up because a toolchain is missing. +- `bootstrap_firefox(application?)` — runs `./mach bootstrap` to install the build toolchain (rustc, cargo, clang, cbindgen). The image does not ship one, so if `build_firefox` reports a missing toolchain (e.g. no rustc or clang), call this once and then build again. Do not give up because a toolchain is missing. + +For Android bugs (GeckoView, Fenix, Focus, android-components, anything under `mobile/android`), call `bootstrap_firefox(application="mobile_android_artifact_mode")` (or `"mobile_android"` if you need to change C++ or Rust code), which installs a JDK and the Android SDK and NDK; `java`, `adb`, `aapt2`, `apksigner`, `apkanalyzer` and `lint` are then on PATH. It also writes a mozconfig for Android, which `build_firefox` then builds with; its result says where, and to run mach from Bash with it, e.g. `MOZCONFIG= ./mach gradle -p mobile/android/android-components :components:support-base:testDebugUnitTest` (use `./mach gradle` instead of `gradlew`, with `-p` for a subproject). JVM unit tests (Robolectric) work; there is no Android emulator or device, so instrumented tests can't run. Follow these rules: diff --git a/agents/bug-fix/tests/test_image.py b/agents/bug-fix/tests/test_image.py new file mode 100644 index 0000000000..69b6f1076a --- /dev/null +++ b/agents/bug-fix/tests/test_image.py @@ -0,0 +1,1174 @@ +"""Smoke tests for the bug-fix agent image. + +They check what the agent can do in the image (run Firefox, play audio, build, +run tests), not how the image is put together, so they run inside it. That is +how CI runs them, with BUG_FIX_IMAGE_INSIDE=1. From the host, name a built +image of the ``agent`` target and they run themselves in containers of it: + + docker build --target agent -t bug-fix-agent -f agents/bug-fix/Dockerfile . + cd agents/bug-fix + BUG_FIX_IMAGE=bug-fix-agent uv run --extra dev pytest -s tests/test_image.py + +They need network access: the evaluate_testcase tests download a Firefox nightly. +Three slower tests bootstrap a Firefox checkout and build it. One makes an +artifact build and runs tests in it (about 13 minutes the first time, then 8), +one makes an Android artifact build, then inspects its APKs and runs unit tests +and the formatter (about 30 minutes), and one only runs when asked: + +- BUG_FIX_IMAGE_FULL_BUILD=1: a full build with the agent's own mozconfig, then + running a testcase and tests in it (hours, and a lot of memory). + +They check what bootstrap does to a clean home directory, so each needs a fresh +container: from the host, each gets one, with the checkout kept in the +bug-fix-image-test-workspace volume so only the first run pays for the clone. +""" + +import array +import asyncio +import base64 +import contextlib +import json +import math +import os +import shlex +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +from pathlib import Path + +import pytest +from agent_tools import firefox +from agent_tools.firefox import FirefoxContext +from agent_tools.firefox.tools.evaluate_testcase import evaluate_testcase +from agent_tools.firefox.tools.js_shell_evaluator import js_shell_evaluator + +# Imported before any test bootstraps, as in the agent, so ffpuppet looks +# minidump-stackwalk up before bootstrap installs it. +from ffpuppet.minidump_parser import MinidumpParser +from hackbot_agents.bug_fix.agent import _write_mozconfig + +IMAGE = os.environ.get("BUG_FIX_IMAGE") +INSIDE = os.environ.get("BUG_FIX_IMAGE_INSIDE") == "1" +FULL_BUILD = os.environ.get("BUG_FIX_IMAGE_FULL_BUILD") == "1" + +pytestmark = pytest.mark.skipif( + not IMAGE and not INSIDE, + reason="neither BUG_FIX_IMAGE nor BUG_FIX_IMAGE_INSIDE is set", +) + +HOME = Path("/home/agent") +WORKSPACE_VOLUME = "bug-fix-image-test-workspace" +SRC = Path("/workspace/firefox") +PYTEST_VERSION = "9.1.0" + +# Where bootstrap puts minidump-stackwalk, and a release to stand in for it in the +# tests that don't bootstrap. +MDSW_DIR = HOME / ".mozbuild" / "minidump-stackwalk" +MDSW_URL = ( + "https://github.com/rust-minidump/rust-minidump/releases/download/v0.27.0/" + "minidump-stackwalk-x86_64-unknown-linux-gnu.tar.xz" +) +MDSW_SHA256 = "0020324c54cc359596e927ee907204f4c8d4da6718765536edcabaa1622122ad" + + +def step(message): + """Say what's happening, for slow tests to show progress (with pytest -s). + + With the disk space left, since builds and their checkout take a lot of it. + """ + free = shutil.disk_usage("/workspace" if Path("/workspace").exists() else "/").free + print( + f"\n=== {time.strftime('%H:%M:%S')} {message} ({free / 2**30:.1f} GiB free)", + flush=True, + ) + + +def _forward(stream, sink, lines): + for line in stream: + sink.write(line) + sink.flush() + lines.append(line) + + +def run(argv, *, timeout=600, check=True, **kwargs): + """Run ``argv``, echoing its output as it comes and returning it too. + + The echo shows live with pytest -s and is part of the report otherwise. + Raises subprocess.TimeoutExpired if it takes longer than ``timeout``, having + killed what it started too: its own process group, its own session. + """ + deadline = time.monotonic() + timeout + proc = subprocess.Popen( + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + **kwargs, + ) + stdout, stderr = [], [] + forwarders = [ + threading.Thread( + target=_forward, args=(proc.stdout, sys.stdout, stdout), daemon=True + ), + threading.Thread( + target=_forward, args=(proc.stderr, sys.stderr, stderr), daemon=True + ), + ] + for forwarder in forwarders: + forwarder.start() + + def kill_group(): + # Descendants (make, gradle, Firefox's child processes...) inherit its + # stdout and stderr: killing only it would leave them holding the pipes, + # and the output threads waiting. + with contextlib.suppress(ProcessLookupError): + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + + try: + proc.wait(timeout=timeout) + # Something it left running in the background may still hold the pipes. + for forwarder in forwarders: + forwarder.join(max(0, deadline - time.monotonic())) + if any(forwarder.is_alive() for forwarder in forwarders): + raise subprocess.TimeoutExpired(argv, timeout) + except subprocess.TimeoutExpired: + kill_group() + # Bounded, in case a descendant left the group and still holds them. + for forwarder in forwarders: + forwarder.join(10) + raise + result = subprocess.CompletedProcess( + argv, proc.returncode, "".join(stdout), "".join(stderr) + ) + if check: + assert result.returncode == 0, ( + f"{argv} exited with {result.returncode}\n" + f"stdout:\n{result.stdout[-5000:]}\nstderr:\n{result.stderr[-5000:]}" + ) + return result + + +def sh(command, **kwargs): + """Run a shell command as the agent would, return its stdout.""" + return run(["bash", "-c", command], **kwargs).stdout + + +def pids_matching(pattern): + return [ + int(pid) + for pid in subprocess.run( + ["pgrep", "-f", "--", pattern], capture_output=True, text=True + ).stdout.split() + ] + + +def proc_status(pid): + """/proc//status as a dict.""" + lines = Path(f"/proc/{pid}/status").read_text().splitlines() + fields = (line.split(":", 1) for line in lines) + return {name: value.strip() for name, value in fields} + + +def test_runs_unprivileged_with_writable_workspace(): + assert sh("whoami").strip() == "agent" + Path("/workspace/probe").touch() + Path("/workspace/probe").unlink() + + +def test_x_display_accepts_clients(): + sh("xset q") + + +def test_session_bus_answers(): + sh( + "dbus-send --session --print-reply --dest=org.freedesktop.DBus " + "/org/freedesktop/DBus org.freedesktop.DBus.ListNames" + ) + + +PCM = ["--raw", "--format=s16le", "--rate=48000", "--channels=1"] + + +def record_peak(source, *, while_recording=None): + """Peak amplitude of a recording of ``source``, optionally while running a command. + + parecord takes a second or two to start delivering audio, and on SIGTERM it + drops what it hasn't written yet, so record for a while and stop it with SIGINT. + """ + with tempfile.NamedTemporaryFile() as recording: + recorder = subprocess.Popen( + ["timeout", "-s", "INT", "5", "parecord", *PCM, "-d", source], + stdout=recording, + ) + if while_recording: + time.sleep(0.3) + run(while_recording) + recorder.wait() + data = Path(recording.name).read_bytes() + return max(map(abs, array.array("h", data[: len(data) // 2 * 2])), default=0) + + +def test_audio_plays(tmp_path): + # Play a tone the way Firefox does, through the PulseAudio interface, and + # check it comes out of the (dummy) output device. + tone = tmp_path / "tone.raw" + samples = ( + int(16000 * math.sin(2 * math.pi * 440 * i / 48000)) for i in range(4 * 48000) + ) + tone.write_bytes(array.array("h", samples).tobytes()) + peak = record_peak("@DEFAULT_MONITOR@", while_recording=["pacat", *PCM, str(tone)]) + assert peak > 8000 + + +def test_pipewire_test_audio_source(): + # Gecko's media tests create audiotestsrc nodes as dummy audio sources. + run( + [ + "pw-cli", + "create-node", + "adapter", + "{ factory.name=audiotestsrc node.name=probe-src " + "media.class=Audio/Source object.linger=true }", + ] + ) + # pw-cli exits 0 even when the factory is missing; the node must be there. + for _ in range(25): + if "probe-src" in sh("pactl list short sources"): + break + time.sleep(0.2) + assert record_peak("probe-src") > 8000 + + +@pytest.mark.parametrize( + ("pattern", "family"), + [ + # What web content asks for, and what fontconfig gives it. + ("sans-serif", "DejaVu Sans"), + ("Arial", "Liberation Sans"), + ("Times New Roman", "Liberation Serif"), + (":lang=ko", "UnDotum"), + (":lang=ja", "VL Gothic"), + ("KacstOne", "KacstOne"), + ("STIX Math", "STIX Math"), + ], +) +def test_fonts(pattern, family): + assert run(["fc-match", pattern, "family"]).stdout.startswith(family) + + +@pytest.mark.parametrize( + ("command", "expected"), + [ + ("hg version -q", "Mercurial"), + ("git --version", "git version"), + ("curl --version", "curl"), + ("zip -v", "Zip"), + ("unzip -v", "UnZip"), + ("bzip2 --help 2>&1", "bzip2"), + ("ffmpeg -version", "ffmpeg version"), + ("printf 'hay\\nneedle\\n' | rg -n needle", "2:needle"), + ("echo '{\"a\": [1, 42]}' | jq '.a[1]'", "42"), + ("sqlite3 :memory: 'select 6 * 7'", "42"), + ("uv run --no-project python -c 'print(6 * 7)'", "42"), + ], +) +def test_developer_tools(command, expected): + assert expected in sh(command) + + +def test_gdb_backtrace(tmp_path): + source = tmp_path / "segfault.c" + source.write_text( + "void crash_here(int *p) { *p = 42; }\nint main(void) { crash_here(0); }\n" + ) + run(["gcc", "-g", "-O0", "-o", str(tmp_path / "segfault"), str(source)]) + out = run( + ["gdb", "-q", "-batch", "-ex", "run", "-ex", "bt", str(tmp_path / "segfault")] + ) + assert "SIGSEGV" in out.stdout + assert f"crash_here (p=0x0) at {source}:1" in out.stdout + + +def test_strace(): + out = run(["strace", "-f", "-e", "trace=openat", "cat", "/etc/hostname"]) + assert 'openat(AT_FDCWD, "/etc/hostname"' in out.stderr + + +def test_git_commit_and_stash(tmp_path): + sh( + f"set -e; git init -q {tmp_path}; cd {tmp_path}; " + "echo a > f; git add f; git commit -qm init; " + "echo b > f; git stash -q; git stash pop -q" + ) + assert sh(f"git -C {tmp_path} log -1 --format='%an <%ae>'").strip() == ( + "Hackbot " + ) + + +def test_python_script_venv_and_pip(tmp_path): + sh( + f"set -e; python3 -m venv {tmp_path}/venv; {tmp_path}/venv/bin/pip install -q six; " + f"{tmp_path}/venv/bin/python -c 'import six, sqlite3, ssl'", + timeout=300, + ) + + +@pytest.fixture(scope="module") +def nightly(): + """A Firefox nightly debug build, plus minidump-stackwalk where bootstrap puts it.""" + if os.uname().machine != "x86_64": + pytest.skip("the minidump-stackwalk stand-in is an x86_64 binary") + sh("fuzzfetch --debug -n firefox -o /tmp", timeout=900) + sh( + f"set -e; mkdir -p {MDSW_DIR}; cd /tmp; curl -fsSL -o mdsw.tar.xz {MDSW_URL}; " + f"echo '{MDSW_SHA256} mdsw.tar.xz' | sha256sum -c -; " + f"tar xf mdsw.tar.xz --strip-components=1 -C {MDSW_DIR} " + "--wildcards '*/minidump-stackwalk'" + ) + # ffpuppet looked it up when this module imported it; tell it, as + # bootstrap_firefox does after installing it. + MinidumpParser.MDSW_BIN = str(MDSW_DIR / "minidump-stackwalk") + return Path("/tmp/firefox/firefox") + + +@pytest.fixture(scope="module") +def nightly_js_shell(): + """The JS shell of a Firefox nightly debug build.""" + sh("fuzzfetch --debug --target js -n js -o /tmp", timeout=900) + return Path("/tmp/js/dist/bin/js") + + +def signal_once_running(pattern, sig, *, settle=0): + """In the background, send ``sig`` to what matches ``pattern`` once it runs.""" + + def send(): + for _ in range(60): + time.sleep(1) + if pids := pids_matching(pattern): + time.sleep(settle) + for pid in pids: + with contextlib.suppress(ProcessLookupError): + os.kill(pid, sig) + return + + threading.Thread(target=send, daemon=True).start() + + +def evaluate(binary, content, *, crash_content=False, **kwargs): + """Run a testcase with the agent's evaluate_testcase.""" + if crash_content: + # Content JS cannot crash the browser on purpose, so do it from outside + # once the page is loaded. + signal_once_running("contentproc.*tab", signal.SIGSEGV, settle=3) + result = asyncio.run( + evaluate_testcase(content, "test.html", Path(binary), **kwargs) + ) + logs = result.get("logs", {}) + result["output"] = logs.get("stdout", "") + logs.get("stderr", "") + return result + + +PROBE = """ + +""" + +# Without a user gesture autoplay keeps the AudioContext suspended no matter what. +ALLOW_AUDIO = {"media.autoplay.default": 0, "media.autoplay.block-webaudio": False} + + +def check_testcase_without_crash(binary): + result = evaluate( + binary, + PROBE, + timeout=30, + prefs={**ALLOW_AUDIO, "general.useragent.override": "PROBE_UA"}, + ) + assert result["crashed"] is False, result["message"] + # Console output and uncaught errors reach the logs the agent reads. + assert "PROBE ua=" in result["output"] + assert "PROBE_JS_ERROR" in result["output"] + # Custom prefs are applied. + assert "ua=PROBE_UA" in result["output"] + # Web Audio gets a working device. + assert "audio=running" in result["output"] + + +def check_testcase_crash(binary): + result = evaluate( + binary, + "", + crash_content=True, + timeout=60, + ) + assert result["crashed"] is True, result["message"] + crashdata = result["logs"]["crashdata"] + assert "SIGSEGV" in crashdata + # Only there if minidump-stackwalk processed the minidump. + assert "libxul.so" in crashdata + + +# Media in the formats the web uses, which Firefox decodes itself (VP9, Opus) or +# with the system's ffmpeg (H.264, AAC): element, MIME type, ffmpeg arguments. +VIDEO = "-f lavfi -i testsrc=size=64x48:rate=10:duration=2" +AUDIO = "-f lavfi -i sine=frequency=440:duration=2" +MEDIA = { + "h264": ( + "video", + "video/mp4", + f"{VIDEO} -c:v libx264 -pix_fmt yuv420p -movflags +faststart -f mp4", + ), + "vp9": ("video", "video/webm", f"{VIDEO} -c:v libvpx-vp9 -f webm"), + "aac": ("audio", "audio/mp4", f"{AUDIO} -c:a aac -movflags +faststart -f mp4"), + "opus": ("audio", "audio/ogg", f"{AUDIO} -c:a libopus -f ogg"), +} + +# Exercises what web content commonly relies on and logs what each check got. +FEATURES_JS = """ +const results = {}; +async function check(name, fn) { + try { + results[name] = await Promise.race([ + fn(), new Promise((_, reject) => setTimeout(() => reject("timeout"), 10000))]); + } catch (e) { + results[name] = "error: " + e; + } +} +function play(kind, src) { + return new Promise((resolve, reject) => { + const el = document.createElement(kind); + el.muted = true; + el.onerror = () => reject(el.error && el.error.message); + el.ontimeupdate = () => { + if (el.currentTime > 0.2) { + resolve(kind == "video" ? el.videoWidth + "x" + el.videoHeight : "played"); + } + }; + el.src = src; + document.body.append(el); + el.play().catch(reject); + }); +} +(async () => { + await check("canvas", async () => { + const canvas = Object.assign(document.createElement("canvas"), {width: 4, height: 4}); + const ctx = canvas.getContext("2d"); + ctx.fillStyle = "rgb(255, 0, 0)"; + ctx.fillRect(0, 0, 4, 4); + return [...ctx.getImageData(1, 1, 1, 1).data].join(","); + }); + await check("webgl", async () => { + const gl = document.createElement("canvas").getContext("webgl2"); + gl.clearColor(0, 1, 0, 1); + gl.clear(gl.COLOR_BUFFER_BIT); + const pixel = new Uint8Array(4); + gl.readPixels(0, 0, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, pixel); + return [...pixel].join(","); + }); + await check("worker", () => new Promise(resolve => { + const worker = new Worker(URL.createObjectURL(new Blob(["postMessage(6 * 7)"]))); + worker.onmessage = e => resolve(e.data); + })); + await check("wasm", async () => { + // (func (export "add") (param i32 i32) (result i32) local.get 0 local.get 1 i32.add) + const add = new Uint8Array([0, 97, 115, 109, 1, 0, 0, 0, 1, 7, 1, 96, 2, 127, 127, 1, + 127, 3, 2, 1, 0, 7, 7, 1, 3, 97, 100, 100, 0, 0, 10, 9, 1, 7, 0, 32, 0, 32, 1, 106, 11]); + return (await WebAssembly.instantiate(add)).instance.exports.add(2, 3); + }); + await check("indexeddb", () => new Promise((resolve, reject) => { + const open = indexedDB.open("probe", 1); + open.onupgradeneeded = () => open.result.createObjectStore("store"); + open.onerror = () => reject(open.error); + open.onsuccess = () => { + const tx = open.result.transaction("store", "readwrite"); + tx.objectStore("store").put("stored", "key"); + tx.oncomplete = () => { + const get = open.result.transaction("store").objectStore("store").get("key"); + get.onsuccess = () => resolve(get.result); + }; + }; + })); + for (const [name, {kind, src}] of Object.entries(MEDIA)) { + await check(name, () => play(kind, src)); + } + console.log("FEATURES " + JSON.stringify(results)); + window.close(); +})(); +""" + + +def features_page(): + """The FEATURES_JS page, with the media generated by ffmpeg and inlined.""" + media = {} + with tempfile.TemporaryDirectory() as tmp: + for name, (kind, mime, args) in MEDIA.items(): + out = Path(tmp) / name + run(["ffmpeg", "-loglevel", "error", *shlex.split(args), str(out)]) + src = f"data:{mime};base64," + base64.b64encode(out.read_bytes()).decode() + media[name] = {"kind": kind, "src": src} + return ( + "" + ) + + +def check_web_features(binary): + result = evaluate(binary, features_page(), timeout=60, prefs=ALLOW_AUDIO) + assert result["crashed"] is False, result["message"] + line = next( + (log for log in result["output"].splitlines() if "FEATURES " in log), None + ) + assert line, result["output"][-5000:] + # console.log lines look like: console.log: "FEATURES {...}" + logged = json.loads(line.split("console.log: ", 1)[1]) + assert json.loads(logged.split("FEATURES ", 1)[1]) == { + "canvas": "255,0,0,255", + "webgl": "0,255,0,255", + "worker": 42, + "wasm": 5, + "indexeddb": "stored", + "h264": "64x48", + "vp9": "64x48", + "aac": "played", + "opus": "played", + } + # Rendering goes through EGL, as on a desktop, rather than a GLX fallback. + assert "libEGL missing" not in result["output"] + + +BLUE_PAGE = '' + + +def pixel_at(rgb, width, x, y): + offset = (y * width + x) * 3 + return tuple(rgb[offset : offset + 3]) + + +def check_draws_on_display(binary, tmp_path): + """Firefox started from the agent's shell, as `./mach run` would, shows up.""" + page = tmp_path / "blue.html" + page.write_text(BLUE_PAGE) + width, height = int(os.environ["SCREEN_WIDTH"]), int(os.environ["SCREEN_HEIGHT"]) + firefox = subprocess.Popen( + [binary, "--kiosk", "--no-remote", "--profile", tmp_path, page.as_uri()], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + for _ in range(60): + time.sleep(1) + screen = subprocess.run( + ["ffmpeg", "-loglevel", "error", "-f", "x11grab"] + + ["-video_size", f"{width}x{height}", "-i", os.environ["DISPLAY"]] + + ["-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], + capture_output=True, + check=True, + ).stdout + r, g, b = pixel_at(screen, width, width // 2, height // 2) + # Colour management shifts it a little. + if b > 200 and r < 100 and g < 100: + break + finally: + firefox.terminate() + firefox.wait() + assert b > 200 and r < 100 and g < 100, f"middle of the screen is {(r, g, b)}" + + +def check_js_shell(binary): + """The agent's evaluate_js_shell runs scripts and notices crashes.""" + ok = asyncio.run( + js_shell_evaluator(content="print(6 * 7)", js_binary=binary, timeout=60) + ) + assert ok["crashed"] is False, ok["message"] + assert ok["logs"]["stdout"].strip() == "42" + + # The tool runs the shell with --fuzzing-safe, which hides crash(), and + # SpiderMonkey handles SIGSEGV itself; abort it from outside instead. + signal_once_running("bugbug_js_", signal.SIGABRT) + crash = asyncio.run( + js_shell_evaluator(content="for (;;) {}", js_binary=binary, timeout=60) + ) + assert crash["crashed"] is True, crash["message"] + + +def check_screenshot(binary, tmp_path): + """Headless screenshots, for the agent to look at what a page renders.""" + page = tmp_path / "blue.html" + page.write_text(BLUE_PAGE) + screenshot = tmp_path / "screenshot.png" + run( + [binary, "--headless", "--no-remote", "--profile", tmp_path] + + ["--screenshot", screenshot, "--window-size", "200,100", page.as_uri()], + timeout=120, + ) + rgb = subprocess.run( + ["ffmpeg", "-loglevel", "error", "-i", screenshot] + + ["-f", "rawvideo", "-pix_fmt", "rgb24", "-"], + capture_output=True, + check=True, + ).stdout + assert len(rgb) == 200 * 100 * 3 + assert pixel_at(rgb, 200, 100, 50) == (0, 0, 255) + + +def content_processes(firefox_pid): + """Firefox's web content processes, which its fork server starts.""" + processes = [] + for pid in pids_matching("-contentproc -isForBrowser"): + ancestor = pid + while ancestor > 1: + try: + ancestor = int(proc_status(ancestor)["PPid"]) + except FileNotFoundError: + break + if ancestor == firefox_pid: + processes.append(pid) + break + return processes + + +def check_content_sandbox(binary, tmp_path): + """Content processes run under Firefox's seccomp-bpf sandbox.""" + firefox = subprocess.Popen( + [binary, "--headless", "--no-remote", "--profile", tmp_path, "about:blank"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + for _ in range(30): + time.sleep(1) + if content_processes(firefox.pid): + break + time.sleep(3) + processes = content_processes(firefox.pid) + assert processes, "no content processes" + # The container runtime may give every process a seccomp filter of its + # own (Docker does), Firefox's parent included; the sandbox adds one, once + # it has set no_new_privs. + unsandboxed = int(proc_status(firefox.pid)["Seccomp_filters"]) + sandboxed = [] + for pid in processes: + # A content process sandboxes itself just after the fork server starts + # it, so give one that has only just started a moment to. + deadline = time.monotonic() + 15 + while True: + try: + status = proc_status(pid) + except FileNotFoundError: + break # It exited. + if ( + status["NoNewPrivs"] == "1" + and int(status["Seccomp_filters"]) > unsandboxed + ): + sandboxed.append(pid) + break + assert time.monotonic() < deadline, f"content process {pid}: {status}" + time.sleep(0.5) + assert sandboxed, "content processes all exited" + # Namespaces depend on the container runtime: Docker's default seccomp + # profile, for one, doesn't let processes create user namespaces. + own = [ + ns + for ns in ("user", "pid", "net") + if os.readlink(f"/proc/{sandboxed[0]}/ns/{ns}") + != os.readlink(f"/proc/{firefox.pid}/ns/{ns}") + ] + step(f"Content process namespaces of their own: {own or 'none'}") + finally: + firefox.terminate() + firefox.wait() + + +def record_profile(binary, page, profile, tmp_path, *, features=None): + """Load ``page`` in Firefox with the Gecko profiler on from startup.""" + env = { + **os.environ, + "MOZ_PROFILER_STARTUP": "1", + "MOZ_PROFILER_SHUTDOWN": str(profile), + } + if features: + env["MOZ_PROFILER_STARTUP_FEATURES"] = features + (tmp_path / "profile").mkdir(exist_ok=True) + run( + [binary, "--headless", "--no-remote", "--profile", tmp_path / "profile"] + + ["--screenshot", tmp_path / "profiled.png", page.as_uri()], + env=env, + timeout=600, + ) + return json.loads(Path(profile).read_text()) + + +def check_profiler(binary, tmp_path): + """The Gecko profiler records the parent and child processes.""" + page = tmp_path / "blue.html" + page.write_text(BLUE_PAGE) + profile = record_profile(binary, page, tmp_path / "profile.json", tmp_path) + samples = {t["name"]: len(t["samples"]["data"]) for t in profile["threads"]} + assert samples.get("GeckoMain", 0) > 0, samples + assert profile["processes"], "child processes weren't profiled" + + +BUSY_PAGE = """""" + + +def profiler_cli(*args, session, **kwargs): + return run(["profiler-cli", *args, "--session", session], **kwargs).stdout + + +def check_profile_analysis(binary, tmp_path, *, native_symbols): + """Profile a page busy in a JS function, and find the function in it. + + As the Firefox tree's profiler-analysis skill would: with profiler-cli, + symbolicating with samply. Needs what bootstrap installs (node, samply). + """ + page = tmp_path / "busy.html" + page.write_text(BUSY_PAGE) + profile = tmp_path / "busy-profile.json" + record_profile(binary, page, profile, tmp_path, features="js,stackwalk,cpu") + + session = "image-test" + profiler_cli("load", "--with-samply", profile, session=session, timeout=600) + try: + threads = json.loads(profiler_cli("thread", "list", "--json", session=session)) + if isinstance(threads, dict): + threads = threads["threads"] + page_thread = next( + t + for t in threads + if t["name"] == "GeckoMain" and t["processName"].startswith("file://") + ) + profiler_cli("thread", "select", page_thread["threadHandle"], session=session) + functions = json.loads( + profiler_cli( + "thread", "functions", "--limit", "20", "--json", session=session + ) + )["functions"] + assert functions[0]["name"] == "hackbotBusyLoop", functions[:3] + if native_symbols: + # A build's own libraries are symbolicated, not left as addresses. + native = [f for f in functions if f["library"] == "libxul.so"] + assert native, functions + assert not [f for f in native if f["name"].startswith("0x")], native + finally: + profiler_cli("stop", session=session, check=False) + + +def test_evaluate_testcase_without_crash(nightly): + check_testcase_without_crash(nightly) + + +def test_evaluate_testcase_reports_crash_with_stack(nightly): + check_testcase_crash(nightly) + + +def test_firefox_web_features(nightly): + check_web_features(nightly) + + +def test_firefox_draws_on_display(nightly, tmp_path): + check_draws_on_display(nightly, tmp_path) + + +def test_gdb_runs_firefox(nightly, tmp_path): + out = run( + ["gdb", "-q", "-batch", "-ex", "handle SIGSYS SIGPIPE nostop noprint"] + + ["-ex", "run", "--args", nightly, "--headless", "--no-remote"] + + ["--profile", tmp_path, "--screenshot", tmp_path / "gdb.png", "about:blank"], + timeout=300, + ) + assert "exited normally" in out.stdout + + +def test_evaluate_js_shell(nightly_js_shell): + check_js_shell(nightly_js_shell) + + +def test_firefox_screenshot(nightly, tmp_path): + check_screenshot(nightly, tmp_path) + + +def test_firefox_content_sandbox(nightly, tmp_path): + check_content_sandbox(nightly, tmp_path) + + +def test_firefox_profiler(nightly, tmp_path): + check_profiler(nightly, tmp_path) + + +@pytest.fixture +def bootstrapped_checkout(request): + """A Firefox checkout bootstrapped the way the agent does it. + + For desktop Firefox unless parametrized (indirectly) with another + application. Returns the FirefoxContext the agent's tools then work with. + """ + return checkout_and_bootstrap(getattr(request, "param", "browser")) + + +# What every bootstrap installs, and what the agent's shell should then find. +BOOTSTRAPPED_TOOLS = ( + "minidump-stackwalk --version", + "node --version", + "treeherder-cli --version", + "searchfox-cli --version", + "socorro-cli --version", + "stmo-cli --version", + "webspec-index --version", + "samply --version", + "profiler-cli --version", + "pq --version", +) +TOOLCHAIN_TOOLS = ("rustc --version", "cargo --version", "clang --version") +ANDROID_TOOLS = ( + "java -version", + "javac -version", + "adb version", + "aapt2 version", + "apksigner --version", + "d8 --version", + "sdkmanager --version", + "lint --version", +) + + +def checkout_and_bootstrap(application="browser"): + """Get a Firefox checkout and bootstrap it the way the agent does.""" + step(f"Getting a Firefox checkout in {SRC}") + # Deep enough for an artifact build to find a recent push CI has built: the + # tip often hasn't been yet. + if (SRC / ".git").exists(): + sh( + f"set -e; cd {SRC}; git fetch -q --depth=50 origin HEAD; " + "git reset -q --hard FETCH_HEAD; git clean -fdxq", + timeout=1800, + ) + else: + run( + ["git", "clone", "-q", "--depth=50"] + + ["https://github.com/mozilla-firefox/firefox.git", SRC], + timeout=1800, + ) + ctx = FirefoxContext.from_source_repo(SRC) + _write_mozconfig(ctx) + + # Bootstrap runs unprivileged: everything it would need sudo for is in the image. + step(f"Bootstrapping for {application}, with the agent's bootstrap_firefox") + result = asyncio.run(firefox.bootstrap_firefox(ctx, application=application)) + print(result["stdout"], result["stderr"], sep="\n") + assert result["success"], result["message"] + # evaluate_testcase can symbolize crashes from then on, in the same process. + assert str(MDSW_DIR / "minidump-stackwalk") == MinidumpParser.MDSW_BIN + + # The agent's shell finds what bootstrap installed. + step("Running the bootstrapped toolchain") + tools = list(BOOTSTRAPPED_TOOLS) + if application == "browser": + tools += TOOLCHAIN_TOOLS + if application.startswith("mobile_android"): + tools += ANDROID_TOOLS + for command in tools: + sh(command) + return ctx + + +def mach(*args, mozconfig=None, timeout=900): + """Run mach in the checkout, with the agent's mozconfig unless told otherwise.""" + env = {**os.environ} + if mozconfig: + env["MOZCONFIG"] = str(mozconfig) + return run(["./mach", *args], cwd=SRC, env=env, timeout=timeout).stdout + + +def test_artifact_build_and_tests(bootstrapped_checkout, tmp_path): + + # The agent's debug + clang-plugin config finds the bootstrapped toolchain. + step("Configuring with the agent's mozconfig") + configure = mach("configure", timeout=1800) + assert f"checking for rustc... {HOME}/.cargo/bin/rustc" in configure + assert ( + f"checking for the target C compiler... {HOME}/.mozbuild/clang/bin/clang" + in configure + ) + + # Running tests needs a build; an artifact one is quick. + artifact = Path("/workspace/mozconfig.artifact") + artifact.write_text( + "ac_add_options --enable-artifact-builds\n" + "mk_add_options MOZ_OBJDIR=/workspace/obj-artifact\n" + ) + step("Making an artifact build") + mach("build", mozconfig=artifact, timeout=1800) + step("Running an xpcshell test") + mach( + "xpcshell-test", + "--sequential", + "toolkit/modules/tests/xpcshell/test_Log.js", + mozconfig=artifact, + ) + step("Running a mochitest") + mach( + "mochitest", "--headless", "dom/base/test/test_bug5141.html", mozconfig=artifact + ) + step("Running a browser-chrome mochitest") + mach( + "mochitest", + "testing/mochitest/tests/browser/browser_pass.js", + mozconfig=artifact, + ) + step("Running a web-platform test") + mach( + "wpt", + "--headless", + "testing/web-platform/tests/dom/nodes/Node-parentNode.html", + mozconfig=artifact, + ) + step("Running reftests") + mach( + "reftest", + "--headless", + "layout/reftests/reftest-sanity/reftest.list", + mozconfig=artifact, + ) + step("Running a Marionette test") + mach( + "marionette-test", + "--headless", + "testing/marionette/harness/marionette_harness/tests/unit/test_title.py", + mozconfig=artifact, + ) + + step("Profiling a page and analyzing the profile") + check_profile_analysis( + "/workspace/obj-artifact/dist/bin/firefox", tmp_path, native_symbols=True + ) + + step("Formatting code") + probe = SRC / "dom/base/HackbotFormatProbe.cpp" + probe.write_text("int probe( int a ){return a+1;}\n") + try: + mach("format", probe.relative_to(SRC), mozconfig=artifact) + formatted = probe.read_text() + finally: + probe.unlink() + assert formatted == "int probe(int a) { return a + 1; }\n" + + +def build_firefox_showing_progress(ctx): + """The agent's build_firefox tool, printing mach's log as it goes. + + build_firefox returns mach's output only at the end. Under a coding agent (it + sets CLAUDECODE) mach also writes it to a log as it goes; follow that. + """ + done = threading.Event() + + def follow_build_log(): + started = time.time() + logs = ctx.build_config()[1] / ".mozbuild" / "logs" / "build" + while not done.is_set(): + new = [ + p for p in logs.glob("build_log_*.log") if p.stat().st_mtime >= started + ] + if new: + break + time.sleep(5) + else: + return + with max(new, key=lambda p: p.stat().st_mtime).open() as log: + while True: + if line := log.readline(): + print(line, end="", flush=True) + elif done.is_set(): + return + else: + time.sleep(1) + + follower = threading.Thread(target=follow_build_log) + follower.start() + try: + return asyncio.run(firefox.build_firefox(ctx)) + finally: + done.set() + follower.join() + + +@pytest.mark.skipif(not FULL_BUILD, reason="BUG_FIX_IMAGE_FULL_BUILD is not set") +def test_full_build_and_tests(bootstrapped_checkout, tmp_path_factory): + fx = bootstrapped_checkout + + # What the agent builds, with its own tool and mozconfig. + step("Building, with the agent's build_firefox and mozconfig") + result = build_firefox_showing_progress(fx) + step("Built" if result["success"] else "Build failed") + assert result["success"], result["message"] + + step("Running testcases in the build") + check_testcase_without_crash(fx.binary) + check_testcase_crash(fx.binary) + check_web_features(fx.binary) + check_draws_on_display(fx.binary, tmp_path_factory.mktemp("display")) + check_screenshot(fx.binary, tmp_path_factory.mktemp("screenshot")) + check_content_sandbox(fx.binary, tmp_path_factory.mktemp("sandbox")) + check_profiler(fx.binary, tmp_path_factory.mktemp("profiler")) + + step("Profiling a page and analyzing the profile") + check_profile_analysis( + fx.binary, tmp_path_factory.mktemp("analysis"), native_symbols=True + ) + + step("Running a script in the JS shell") + check_js_shell(fx.js_binary) + + step("Running an xpcshell test") + mach("xpcshell-test", "--sequential", "toolkit/modules/tests/xpcshell/test_Log.js") + step("Running a mochitest") + mach("mochitest", "--headless", "dom/base/test/test_bug5141.html") + step("Running a gtest") + mach("gtest", "Strings.IsChar", timeout=3600) + step("Running clang-tidy") + mach("static-analysis", "check", "xpcom/string/nsReadableUtils.cpp", timeout=3600) + + +GECKOVIEW_EXAMPLE_APK = ( + "gradle/build/mobile/android/geckoview_example/outputs/apk/debug" + "/geckoview_example-debug.apk" +) + + +@pytest.mark.parametrize( + "bootstrapped_checkout", ["mobile_android_artifact_mode"], indirect=True +) +def test_android_build_and_tests(bootstrapped_checkout): + ctx = bootstrapped_checkout + # Bootstrapping for Android set the agent's tools up to build it. + android, objdir = ctx.build_config() + assert android == ctx.android_mozconfig and android.exists() + + step("Making an Android artifact build, with the agent's build_firefox") + result = build_firefox_showing_progress(ctx) + step("Built" if result["success"] else "Build failed") + assert result["success"], result["message"] + + step("Inspecting the APKs it built") + apk = objdir / GECKOVIEW_EXAMPLE_APK + assert ( + run(["apkanalyzer", "manifest", "application-id", apk]) + .stdout.strip() + .endswith("org.mozilla.geckoview_example") + ) + run(["apksigner", "verify", apk]) + + # The tree's android skill: `./mach gradle -p `, not gradlew. + step("Running android-components unit tests") + mach( + "gradle", + "-p", + "mobile/android/android-components", + ":components:support-base:testDebugUnitTest", + "--tests", + "mozilla.components.support.base.log.LogTest", + mozconfig=android, + timeout=3600, + ) + report = ( + objdir / "gradle/build/mobile/android/android-components" + "/components/support/base/test-results/testDebugUnitTest" + "/TEST-mozilla.components.support.base.log.LogTest.xml" + ).read_text() + assert 'failures="0"' in report and 'errors="0"' in report + assert 'tests="0"' not in report + + step("Formatting Java") + probe = SRC / ( + "mobile/android/geckoview_example/src/main/java/org/mozilla" + "/geckoview_example/HackbotFormatProbe.java" + ) + probe.write_text( + "/* This Source Code Form is subject to the terms of the Mozilla Public\n" + " * License, v. 2.0. If a copy of the MPL was not distributed with this\n" + " * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n\n" + "package org.mozilla.geckoview_example;\n\n" + "class HackbotFormatProbe { int probe( int a ){return a+1;} }\n" + ) + try: + lint = ["lint", "-l", "android-format", probe.relative_to(SRC)] + check = run( + ["./mach", *lint], + cwd=SRC, + env={**os.environ, "MOZCONFIG": str(android)}, + timeout=1800, + check=False, + ) + assert check.returncode != 0, "misformatted Java went unnoticed" + mach(*lint, "--fix", mozconfig=android, timeout=1800) + formatted = probe.read_text() + finally: + probe.unlink() + assert " int probe(int a) {\n return a + 1;\n }" in formatted + + +# Outside the image, run the tests above in containers of it instead: one for the +# quick tests and one for each slow test, which needs a fresh home directory. +if not INSIDE: + for _name in [name for name in globals() if name.startswith("test_")]: + del globals()[_name] + + @pytest.mark.parametrize( + ("selection", "env"), + [ + pytest.param( + "not artifact_build and not full_build and not android", + {}, + id="quick", + ), + pytest.param("artifact_build", {}, id="artifact_build"), + pytest.param( + "full_build", + {"BUG_FIX_IMAGE_FULL_BUILD": "1"}, + id="full_build", + marks=pytest.mark.skipif( + not FULL_BUILD, reason="BUG_FIX_IMAGE_FULL_BUILD is not set" + ), + ), + pytest.param("android", {}, id="android"), + ], + ) + def test_in_image(selection, env): + tests = Path(__file__).parent + env_args = [ + arg for name, value in env.items() for arg in ("-e", f"{name}={value}") + ] + # The image's Python has no pip; install pytest with the system one. + command = ( + "/usr/local/bin/python3 -m pip install --quiet --disable-pip-version-check " + f"--target /tmp/pydeps pytest=={PYTEST_VERSION} && cd /tmp && " + "PYTHONPATH=/tmp/pydeps python -m pytest -s -v -rs -p no:cacheprovider " + f"/tests/{Path(__file__).name} -k {shlex.quote(selection)}" + ) + run( + ["docker", "run", "--rm", "-e", "BUG_FIX_IMAGE_INSIDE=1", *env_args] + + ["-v", f"{WORKSPACE_VOLUME}:/workspace", "-v", f"{tests}:/tests:ro"] + + [IMAGE, "bash", "-c", command], + timeout=8 * 3600, + ) diff --git a/libs/agent-tools/agent_tools/firefox/__init__.py b/libs/agent-tools/agent_tools/firefox/__init__.py index 268efa2cab..277451ff38 100644 --- a/libs/agent-tools/agent_tools/firefox/__init__.py +++ b/libs/agent-tools/agent_tools/firefox/__init__.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from pathlib import Path -from typing import Annotated +from typing import Annotated, Literal from pydantic import Field @@ -35,6 +35,27 @@ class FirefoxContext: objdir: Path binary: Path js_binary: Path + # What bootstrap_firefox last set up, which build_firefox then builds for. + application: str = "browser" + + @property + def android_mozconfig(self) -> Path: + # /.mozconfig* is in the tree's .gitignore, so it stays out of patches. + return self.source_dir / ".mozconfig-android" + + def build_config(self) -> tuple[Path, Path]: + """The mozconfig and objdir to build with, for ``application``. + + The agent's own for desktop Firefox; for Android, the mozconfig + bootstrap_firefox writes, with an objdir for artifact builds and one for + full ones so switching between them doesn't need a clobber. + """ + if not self.application.startswith("mobile_android"): + return self.mozconfig, self.objdir + objdir = "objdir-android" + if self.application.endswith("_artifact_mode"): + objdir += "-artifact" + return self.android_mozconfig, self.source_dir / objdir @classmethod def from_source_repo( @@ -135,6 +156,9 @@ async def build_firefox( ) -> dict: """Build Firefox using the configured mozconfig. + For Android once ``bootstrap_firefox`` has been called for it, with the + mozconfig it wrote for that; for desktop Firefox otherwise. + Slow on a full tree build (tens of minutes cold, faster incremental); pass a `target` directory to build just the part you changed. Returns JSON: success (bool), build_dir (str), message (str), stdout/stderr. Only call this if @@ -142,8 +166,9 @@ async def build_firefox( first. """ firefox_dir_p = Path(firefox_dir) if firefox_dir else ctx.source_dir - mozconfig_p = Path(mozconfig_path) if mozconfig_path else ctx.mozconfig - return await _build_firefox(firefox_dir_p, mozconfig_p, ctx.objdir, target=target) + mozconfig, objdir = ctx.build_config() + mozconfig_p = Path(mozconfig_path) if mozconfig_path else mozconfig + return await _build_firefox(firefox_dir_p, mozconfig_p, objdir, target=target) @tool @@ -197,17 +222,57 @@ async def bootstrap_firefox( description="Firefox source directory. Optional — defaults to the configured source dir." ), ] = None, + application: Annotated[ + Literal[ + "browser", + "browser_artifact_mode", + "mobile_android", + "mobile_android_artifact_mode", + ], + Field( + description="What to bootstrap for: desktop Firefox (the default), or " + "Android (GeckoView, Fenix, Focus, android-components), which installs " + "a JDK and the Android SDK and NDK." + ), + ] = "browser", ) -> dict: """Run ``./mach bootstrap`` to install the Firefox build toolchain. - Installs rust, clang, cbindgen under the running user's ~/.mozbuild/. - Required before a full (non-artifact) build. Slow — ~10-15 min on a fresh - image, fast on re-runs. Returns JSON: success, message, stdout, stderr. Only - call this if you intend to do a full build; artifact builds don't need - bootstrap. + Installs rust, clang, cbindgen under the running user's ~/.mozbuild/, or for + Android a JDK, the Android SDK and NDK. Required before a full (non-artifact) + build, and before any Android build. Slow — ~10-15 min on a fresh image, + fast on re-runs. Returns JSON: success, message, stdout, stderr. Only call + this if you intend to do a full build or an Android one; desktop artifact + builds don't need bootstrap. + + Bootstrapping for Android also writes a mozconfig for it, which + ``build_firefox`` then uses; bootstrapping for desktop switches back. """ firefox_dir_p = Path(firefox_dir) if firefox_dir else ctx.source_dir - return await _bootstrap_firefox(firefox_dir_p) + result = await _bootstrap_firefox(firefox_dir_p, application) + if result["success"]: + ctx.application = application + if application.startswith("mobile_android"): + mozconfig, objdir = ctx.build_config() + mozconfig.write_text(_android_mozconfig(application, objdir)) + result["message"] += ( + f"; build_firefox now builds Firefox for Android, with {mozconfig} " + f"(for mach from Bash: MOZCONFIG={mozconfig} ./mach ...)" + ) + return result + + +def _android_mozconfig(application: str, objdir: Path) -> str: + """What `mach bootstrap` suggests for ``application``, with our objdir.""" + lines = [ + f"# Written by bootstrap_firefox(application={application!r}), again on", + "# each Android bootstrap.", + "ac_add_options --enable-project=mobile/android", + ] + if application.endswith("_artifact_mode"): + lines.append("ac_add_options --enable-artifact-builds") + lines.append(f"mk_add_options MOZ_OBJDIR={objdir}") + return "\n".join(lines) + "\n" TOOLS = tools_in(__name__) diff --git a/libs/agent-tools/agent_tools/firefox/tools/bootstrap_firefox.py b/libs/agent-tools/agent_tools/firefox/tools/bootstrap_firefox.py index 6d6137d9ef..821479bf78 100644 --- a/libs/agent-tools/agent_tools/firefox/tools/bootstrap_firefox.py +++ b/libs/agent-tools/agent_tools/firefox/tools/bootstrap_firefox.py @@ -1,13 +1,99 @@ """Install the Firefox build toolchain via `./mach bootstrap`.""" import asyncio +import os +import re from pathlib import Path +from shutil import which from typing import Any +APPLICATIONS = ( + "browser", + "browser_artifact_mode", + "mobile_android", + "mobile_android_artifact_mode", +) -async def bootstrap_firefox(firefox_dir: Path) -> dict[str, Any]: + +def _refresh_minidump_stackwalk() -> None: + """Point ffpuppet at the minidump-stackwalk bootstrap just installed. + + ffpuppet resolves minidump-stackwalk on PATH once, when it is imported, which + is before bootstrap has had a chance to install it; without a refresh crashes + caught by evaluate_testcase come back without a symbolized stack. + """ + try: + from ffpuppet.minidump_parser import MinidumpParser + except ImportError: + return + MinidumpParser.MDSW_BIN = which("minidump-stackwalk") + + +# Run with `./mach python`, for the tree's mozboot and mozbuild. +_MACH_PYTHON = Path(__file__).parent / "mach_python" + + +async def _install_developer_tools(firefox_dir: Path) -> tuple[int | None, str, str]: + process = await asyncio.create_subprocess_exec( + "./mach", + "python", + str(_MACH_PYTHON / "install_developer_tools.py"), + cwd=firefox_dir, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + start_new_session=True, + ) + stdout, stderr = await process.communicate() + return ( + process.returncode, + stdout.decode("utf-8", errors="ignore"), + stderr.decode("utf-8", errors="ignore"), + ) + + +def _version_key(path: Path) -> tuple[int, ...]: + return tuple(int(n) for n in re.findall(r"\d+", path.name)) + + +def _link_android_tools() -> None: + """Give the JDK and Android SDK tools bootstrap installed stable paths. + + They go in versioned directories (jdk/jdk-17.0.20.1+1, build-tools/37.0.0, + cmdline-tools/23.0), so link the newest of each as `current` or `latest` + (the SDK's own convention), for PATH to point at. + """ + mozbuild = Path(os.environ.get("MOZBUILD_STATE_PATH", Path.home() / ".mozbuild")) + sdk = mozbuild / "android-sdk-linux" + for parent, link in ( + (mozbuild / "jdk", "current"), + (sdk / "build-tools", "latest"), + (sdk / "cmdline-tools", "latest"), + ): + versions = [ + p + for p in parent.glob("*") + if p.is_dir() and not p.is_symlink() and re.match(r"(jdk-)?\d", p.name) + ] + if not versions: + continue + target = parent / link + target.unlink(missing_ok=True) + target.symlink_to(max(versions, key=_version_key).name) + + +async def bootstrap_firefox( + firefox_dir: Path, application: str = "browser" +) -> dict[str, Any]: """Run `./mach bootstrap` to install rust/clang/cbindgen for full builds. + With ``application="mobile_android"`` (or ``"mobile_android_artifact_mode"``) + it installs what Android builds need instead: a JDK, the Android SDK and NDK. + + Also installs the developer tools bootstrap offers coding agents + (treeherder-cli, searchfox-cli, socorro-cli, stmo-cli, webspec-index); + failing to is reported in the message but not an error. + Required before a full (non-artifact) build can succeed. On a fresh image bootstrap takes ~10-15 min and downloads the toolchain into the running user's ~/.mozbuild/. Idempotent: re-runs are fast once @@ -15,11 +101,20 @@ async def bootstrap_firefox(firefox_dir: Path) -> dict[str, Any]: Args: firefox_dir: Firefox source directory (contains ./mach). + application: What to bootstrap for, one of APPLICATIONS. Returns: Dict with success, message, stdout, stderr. Never raises. """ try: + if application not in APPLICATIONS: + return { + "success": False, + "message": f"Unknown application {application!r}, expected one of " + + ", ".join(APPLICATIONS), + "stdout": "", + "stderr": "", + } if not firefox_dir.exists(): return { "success": False, @@ -32,10 +127,16 @@ async def bootstrap_firefox(firefox_dir: Path) -> dict[str, Any]: "./mach", "--no-interactive", "bootstrap", - "--application-choice=browser", + f"--application-choice={application}", cwd=firefox_dir, + stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + # Bootstrap tries to install system packages with `su` (without + # sudo, as in the agent images), which asks for a password on the + # controlling terminal. Without one it fails at once and bootstrap + # carries on; with one (e.g. `docker run -t`) it would wait forever. + start_new_session=True, ) stdout, stderr = await process.communicate() @@ -43,11 +144,22 @@ async def bootstrap_firefox(firefox_dir: Path) -> dict[str, Any]: stderr_output = stderr.decode("utf-8", errors="ignore") if stderr else "" if process.returncode == 0: + _refresh_minidump_stackwalk() + if application.startswith("mobile_android"): + _link_android_tools() + message = "mach bootstrap completed successfully" + returncode, tools_stdout, tools_stderr = await _install_developer_tools( + firefox_dir + ) + if returncode != 0: + message += ( + f"; installing developer tools failed with exit code {returncode}" + ) return { "success": True, - "message": "mach bootstrap completed successfully", - "stdout": stdout_output, - "stderr": stderr_output, + "message": message, + "stdout": stdout_output + tools_stdout, + "stderr": stderr_output + tools_stderr, } return { "success": False, diff --git a/libs/agent-tools/agent_tools/firefox/tools/mach_python/install_developer_tools.py b/libs/agent-tools/agent_tools/firefox/tools/mach_python/install_developer_tools.py new file mode 100644 index 0000000000..100e1e556e --- /dev/null +++ b/libs/agent-tools/agent_tools/firefox/tools/mach_python/install_developer_tools.py @@ -0,0 +1,25 @@ +"""Install the developer tools `mach bootstrap` offers coding agents. + +Non-interactive bootstrap skips them (treeherder-cli, searchfox-cli, ...), so +install them the way it would: from the tree's toolchain artifacts into +~/.cargo/bin. Not needed once bug 2080915 makes non-interactive bootstrap +install them. + +Runs with `./mach python`, which has the tree's mozboot and mozbuild; this +directory isn't a package, so nothing imports it by mistake elsewhere. +""" + +import shutil +from pathlib import Path + +from mozboot.base import BaseBootstrapper +from mozbuild.bootstrap import bootstrap_toolchain + +cargo_bin = Path.home() / ".cargo" / "bin" +cargo_bin.mkdir(parents=True, exist_ok=True) +for tool in BaseBootstrapper.CARGO_TOOLS: + tool_dir = bootstrap_toolchain(tool) + if tool_dir: + shutil.copy2(Path(tool_dir) / tool, cargo_bin / tool) + else: + print(f"Could not install {tool}.") diff --git a/libs/agent-tools/tests/test_firefox.py b/libs/agent-tools/tests/test_firefox.py index 9de7a3b208..e0afb79f19 100644 --- a/libs/agent-tools/tests/test_firefox.py +++ b/libs/agent-tools/tests/test_firefox.py @@ -1,7 +1,12 @@ """Tests for the Firefox tools.""" +import os +from pathlib import Path + from agent_tools import firefox from agent_tools.claude_sdk import build_sdk_server +from agent_tools.firefox.tools import bootstrap_firefox +from ffpuppet.minidump_parser import MinidumpParser from mcp.types import ListToolsRequest @@ -24,3 +29,219 @@ async def test_exposes_firefox_tools(tmp_path): "evaluate_js_shell", "bootstrap_firefox", } + + +def _fake_tree(tmp_path, exit_code): + """A source dir whose `./mach` exits with ``exit_code``.""" + src = tmp_path / "firefox" + src.mkdir() + mach = src / "mach" + mach.write_text(f"#!/bin/sh\nexit {exit_code}\n") + mach.chmod(0o755) + return src + + +def _fake_minidump_stackwalk(tmp_path, monkeypatch): + """Put a minidump-stackwalk on PATH, as bootstrap does.""" + bin_dir = tmp_path / "minidump-stackwalk" + bin_dir.mkdir() + mdsw = bin_dir / "minidump-stackwalk" + mdsw.write_text("#!/bin/sh\n") + mdsw.chmod(0o755) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}") + return mdsw + + +async def test_bootstrap_points_ffpuppet_at_minidump_stackwalk(tmp_path, monkeypatch): + # ffpuppet looked minidump-stackwalk up at import time, before bootstrap + # installed it. + monkeypatch.setattr(MinidumpParser, "MDSW_BIN", None) + mdsw = _fake_minidump_stackwalk(tmp_path, monkeypatch) + + result = await bootstrap_firefox(_fake_tree(tmp_path, 0)) + + assert result["success"] + assert str(mdsw) == MinidumpParser.MDSW_BIN + + +async def test_failed_bootstrap_leaves_ffpuppet_alone(tmp_path, monkeypatch): + monkeypatch.setattr(MinidumpParser, "MDSW_BIN", None) + _fake_minidump_stackwalk(tmp_path, monkeypatch) + + result = await bootstrap_firefox(_fake_tree(tmp_path, 1)) + + assert not result["success"] + assert MinidumpParser.MDSW_BIN is None + + +def _recording_tree(tmp_path, tools_exit_code): + """A source dir whose `./mach` logs its commands; `mach python` exits as told.""" + src = tmp_path / "firefox" + src.mkdir() + mach = src / "mach" + mach.write_text( + "#!/bin/sh\n" + f'echo "$@" >> "{tmp_path}/mach.log"\n' + f'[ "$1" = python ] && exit {tools_exit_code}\n' + "exit 0\n" + ) + mach.chmod(0o755) + return src + + +async def test_bootstrap_installs_developer_tools(tmp_path): + result = await bootstrap_firefox(_recording_tree(tmp_path, 0)) + + assert result["success"] + assert result["message"] == "mach bootstrap completed successfully" + log = (tmp_path / "mach.log").read_text().splitlines() + assert log[0] == "--no-interactive bootstrap --application-choice=browser" + script = log[1].split(" ", 1)[1] + assert log[1] == f"python {script}" + assert script.endswith("/mach_python/install_developer_tools.py") + assert Path(script).is_file() + + +async def test_developer_tools_failure_does_not_fail_bootstrap(tmp_path): + result = await bootstrap_firefox(_recording_tree(tmp_path, 3)) + + assert result["success"] + assert "installing developer tools failed with exit code 3" in result["message"] + + +async def test_bootstrap_runs_without_controlling_terminal(tmp_path): + # So a password prompt (bootstrap runs `su` for system packages) fails + # instead of waiting on the terminal. + src = tmp_path / "firefox" + src.mkdir() + mach = src / "mach" + mach.write_text( + f"#!/bin/sh\nawk '{{print $6}}' /proc/$$/stat >> {tmp_path}/sessions\n" + ) + mach.chmod(0o755) + + result = await bootstrap_firefox(src) + + assert result["success"] + sessions = (tmp_path / "sessions").read_text().split() + assert len(sessions) == 2 + assert str(os.getsid(0)) not in sessions + + +async def test_bootstrap_for_android(tmp_path, monkeypatch): + src = _recording_tree(tmp_path, 0) + mozbuild = tmp_path / "mozbuild" + for path in ( + "jdk/jdk-17.0.9+1", + "jdk/jdk-17.0.20.1+1", + "jdk/mozboot", + "android-sdk-linux/build-tools/36.1.0", + "android-sdk-linux/build-tools/37.0.0", + "android-sdk-linux/cmdline-tools/23.0", + "android-sdk-linux/cmdline-tools/mozboot", + ): + (mozbuild / path).mkdir(parents=True) + monkeypatch.setenv("MOZBUILD_STATE_PATH", str(mozbuild)) + + result = await bootstrap_firefox(src, "mobile_android") + + assert result["success"] + assert (tmp_path / "mach.log").read_text().splitlines()[0] == ( + "--no-interactive bootstrap --application-choice=mobile_android" + ) + # The newest of each, for PATH to point at. + assert os.readlink(mozbuild / "jdk/current") == "jdk-17.0.20.1+1" + sdk = mozbuild / "android-sdk-linux" + assert os.readlink(sdk / "build-tools/latest") == "37.0.0" + assert os.readlink(sdk / "cmdline-tools/latest") == "23.0" + + +async def test_bootstrap_rejects_unknown_application(tmp_path): + result = await bootstrap_firefox(_recording_tree(tmp_path, 0), "ios") + + assert not result["success"] + assert "Unknown application 'ios'" in result["message"] + assert not (tmp_path / "mach.log").exists() + + +async def test_bootstrap_tool_offers_android(tmp_path): + ctx = firefox.FirefoxContext.from_source_repo(tmp_path) + tools = await _list(build_sdk_server("firefox", ctx, firefox.TOOLS)["instance"]) + bootstrap = next(t for t in tools if t.name == "bootstrap_firefox") + assert ( + "mobile_android" in bootstrap.inputSchema["properties"]["application"]["enum"] + ) + + +def _mach_logging_mozconfig(tmp_path, bootstrap_exit_code=0): + """A source dir whose `./mach` logs the MOZCONFIG it runs with.""" + src = tmp_path / "firefox" + src.mkdir() + mach = src / "mach" + mach.write_text( + "#!/bin/sh\n" + f'echo "$MOZCONFIG $1 $2" >> "{tmp_path}/mach.log"\n' + f'[ "$2" = bootstrap ] && exit {bootstrap_exit_code}\n' + "exit 0\n" + ) + mach.chmod(0o755) + (src / ".mozconfig").write_text("ac_add_options --enable-debug\n") + return src + + +async def test_android_bootstrap_switches_builds_to_android(tmp_path, monkeypatch): + monkeypatch.setenv("MOZBUILD_STATE_PATH", str(tmp_path / "mozbuild")) + src = _mach_logging_mozconfig(tmp_path) + ctx = firefox.FirefoxContext.from_source_repo(src) + + result = await firefox.bootstrap_firefox( + ctx, application="mobile_android_artifact_mode" + ) + + assert result["success"] + android = src / ".mozconfig-android" + assert f"MOZCONFIG={android}" in result["message"] + assert android.read_text().splitlines()[2:] == [ + "ac_add_options --enable-project=mobile/android", + "ac_add_options --enable-artifact-builds", + f"mk_add_options MOZ_OBJDIR={src / 'objdir-android-artifact'}", + ] + assert ctx.build_config() == (android, src / "objdir-android-artifact") + + result = await firefox.build_firefox(ctx) + + assert result["success"] + assert result["build_dir"] == str(src / "objdir-android-artifact") + assert (tmp_path / "mach.log").read_text().splitlines()[-1] == f"{android} build " + + # Bootstrapping for desktop goes back to the agent's own mozconfig. + await firefox.bootstrap_firefox(ctx, application="browser") + await firefox.build_firefox(ctx) + + assert ctx.build_config() == (src / ".mozconfig", src / "objdir-ff-asan") + assert (tmp_path / "mach.log").read_text().splitlines()[-1] == ( + f"{src / '.mozconfig'} build " + ) + + +async def test_full_android_build_gets_its_own_objdir(tmp_path, monkeypatch): + monkeypatch.setenv("MOZBUILD_STATE_PATH", str(tmp_path / "mozbuild")) + src = _mach_logging_mozconfig(tmp_path) + ctx = firefox.FirefoxContext.from_source_repo(src) + + await firefox.bootstrap_firefox(ctx, application="mobile_android") + + android = (src / ".mozconfig-android").read_text() + assert "--enable-artifact-builds" not in android + assert f"MOZ_OBJDIR={src / 'objdir-android'}\n" in android + + +async def test_failed_android_bootstrap_keeps_building_desktop(tmp_path): + src = _mach_logging_mozconfig(tmp_path, bootstrap_exit_code=1) + ctx = firefox.FirefoxContext.from_source_repo(src) + + result = await firefox.bootstrap_firefox(ctx, application="mobile_android") + + assert not result["success"] + assert not (src / ".mozconfig-android").exists() + assert ctx.build_config() == (src / ".mozconfig", src / "objdir-ff-asan")