From f0d88b4153bedc1e2894a9012de94a0c6ad0afdb Mon Sep 17 00:00:00 2001 From: Ksenia Berezina Date: Thu, 9 Jul 2026 16:06:52 -0400 Subject: [PATCH] Add Chrome support to autowebcompat repro agent --- agents/autowebcompat-repro/Dockerfile | 18 +-- .../autowebcompat_repro/agent.py | 42 +++++-- .../chrome_devtools_mcp.py | 42 +++++++ .../autowebcompat_repro/chrome_install.py | 111 ++++++++++++++++++ .../autowebcompat_repro/config.py | 28 +++++ .../autowebcompat_repro/result.py | 10 ++ 6 files changed, 233 insertions(+), 18 deletions(-) create mode 100644 agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_devtools_mcp.py create mode 100644 agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_install.py diff --git a/agents/autowebcompat-repro/Dockerfile b/agents/autowebcompat-repro/Dockerfile index 4740cc4ce6..3694298971 100644 --- a/agents/autowebcompat-repro/Dockerfile +++ b/agents/autowebcompat-repro/Dockerfile @@ -28,17 +28,21 @@ ENV PATH="/opt/venv/bin:$PATH" FROM base AS agent -# The Firefox DevTools MCP server is an npm package launched via `npx`, so the -# agent image needs Node.js + npm (the python base ships neither). It also -# needs the shared libraries Firefox requires to run headless; the Firefox -# binary itself is downloaded at agent startup (a fresh Nightly per run) via -# mozdownload/mozinstall, not baked in here. +# The Firefox and Chrome DevTools MCP servers are npm packages launched via +# `npx`, so the agent image needs Node.js + npm (the python base ships neither). +# It also needs the shared libraries the browsers require to run headless. The +# browser binaries themselves are downloaded at agent startup (a fresh build per +# run): Firefox Nightly via mozdownload/mozinstall, Chrome for Testing +# from https://googlechromelabs.github.io/chrome-for-testing/ RUN apt-get update \ && apt-get install -y --no-install-recommends \ nodejs npm \ ca-certificates \ - libgtk-3-0 libdbus-glib-1-2 libx11-xcb1 libxtst6 libxt6 \ - libasound2 libpci3 \ + libgtk-3-0 libdbus-glib-1-2 libx11-xcb1 libxtst6 libxt6 libpci3 \ + libnss3 libnspr4 libatk1.0-0t64 libatk-bridge2.0-0t64 libatspi2.0-0t64 \ + libcups2t64 libdbus-1-3 libgbm1 libxcomposite1 libxdamage1 libxfixes3 \ + libxrandr2 libxkbcommon0 libasound2t64 libpango-1.0-0 libcairo2 \ + fonts-liberation \ && rm -rf /var/lib/apt/lists/* # hackbot.toml lives at the agent root (not inside the package), so copy it into diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py index cc4a8a222d..60a6865373 100644 --- a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py @@ -27,7 +27,9 @@ from hackbot_runtime.claude import Reporter from pydantic import BaseModel -from .config import BUGZILLA_READ_TOOLS, DEVTOOLS_TOOLS +from .chrome_devtools_mcp import build_chrome_devtools_server +from .chrome_install import install_chrome +from .config import BUGZILLA_READ_TOOLS, CHROME_DEVTOOLS_TOOLS, DEVTOOLS_TOOLS from .devtools_mcp import build_devtools_server from .firefox_install import install_firefox_nightly from .result import ( @@ -76,6 +78,7 @@ class AutowebcompatReproResult(BaseModel): failure_reason: str | None steps: str screenshot_path: str | None + chrome_reproduced: bool | None = None chrome_mask_fixed: bool | None = None @@ -240,6 +243,7 @@ def __init__( run_tracker: RunTracker, firefox_path: Path, profile_path: Path, + chrome_path: Path, input_data: AutoWebcompatInput, bugzilla_mcp_server: McpServerConfig, screenshot_dir: Path, @@ -260,6 +264,11 @@ def __init__( ), DEVTOOLS_TOOLS, ) + self.add_mcp_server( + "chrome-devtools", + build_chrome_devtools_server(chrome_path=chrome_path, headless=True), + CHROME_DEVTOOLS_TOOLS, + ) if self.input_data.type == "bug_id" != None: self.add_mcp_server("bugzilla", bugzilla_mcp_server, BUGZILLA_READ_TOOLS) @@ -273,16 +282,24 @@ def system_prompt(self) -> str: .format( task_details=f""" 1. Identify the affected URL and the described broken behavior. -2. Baseline: Navigate to the URL with the Firefox DevTools MCP and - try to reproduce the described broken behaviour. -3. If the issue reproduces AND the breakage is visual in nature (incorrect - layout or rendering, not broken interaction), capture a screenshot showing - it: call `screenshot_page` with `saveTo` set to `{self.screenshot_path}`. - This writes the image to that file instead of returning it — do not capture - or paste the image data yourself. Then set `screenshot_path` in your result - to exactly `{self.screenshot_path}`. For non-visual issues, take no - screenshot and leave `screenshot_path` null. -4. Submit your findings via `submit_result` (see "Reporting your result"). +2. Baseline: Navigate to the URL with the Firefox DevTools MCP + (`mcp__firefox-devtools__*` tools) and try to reproduce the described broken + behaviour. This determines `reproduced`. +3. If the issue reproduces in Firefox AND the breakage is visual in nature + (incorrect layout or rendering, not broken interaction), capture a screenshot + showing it: call `screenshot_page` with `saveTo` set to + `{self.screenshot_path}`. This writes the image to that file instead of + returning it — do not capture or paste the image data yourself. Then set + `screenshot_path` in your result to exactly `{self.screenshot_path}`. For + non-visual issues, take no screenshot and leave `screenshot_path` null. +4. Cross-browser check in Chrome (always run this, regardless of whether the + issue reproduced in Firefox): re-run the exact same reproduction steps using + the Chrome DevTools MCP (`mcp__chrome-devtools__*` tools), starting from a + fresh `new_page`. Judge whether the same broken behaviour appears in Chrome + and set `chrome_reproduced` accordingly (true = also broken in Chrome, + false = Chrome behaves correctly). An issue that reproduces in Firefox but + not in Chrome is a strong signal of a Firefox-specific webcompat problem. +5. Submit your findings via `submit_result` (see "Reporting your result"). """ ) ) @@ -372,6 +389,7 @@ async def run_autowebcompat_repro( :class:`AgentError` if the agent ends in an error. """ nightly_path = install_firefox_nightly() + chrome_path = install_chrome() screenshots_dir = Path(tempfile.mkdtemp(prefix="autowebcompat-screenshots-")) @@ -381,6 +399,7 @@ async def run_autowebcompat_repro( tracker, nightly_path, setup_profile(nightly_path, extensions=[]), + chrome_path, input_data, bugzilla_mcp_server, screenshots_dir, @@ -400,6 +419,7 @@ async def run_autowebcompat_repro( failure_reason=repro_result.failure_reason, steps=repro_result.steps, screenshot_path=screenshot_path, + chrome_reproduced=repro_result.chrome_reproduced, ) if repro_result.reproduced: diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_devtools_mcp.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_devtools_mcp.py new file mode 100644 index 0000000000..91c3e721bd --- /dev/null +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_devtools_mcp.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from pathlib import Path + +from claude_agent_sdk.types import McpStdioServerConfig + +PACKAGE = "chrome-devtools-mcp@latest" + + +def build_chrome_devtools_server( + chrome_path: Path | None = None, + *, + headless: bool = True, + no_sandbox: bool = True, +) -> McpStdioServerConfig: + """Build the stdio config for the Chrome DevTools MCP server. + + Args: + chrome_path: Chrome binary to drive (the Chrome for Testing build from + ``chrome_install.install_chrome``). When ``None`` the server lets its + bundled Puppeteer discover a Chrome installation itself. + headless: Run Chrome without a visible window (required in + container/CI environments). + no_sandbox: Pass ``--no-sandbox`` to Chrome. Required when running as an + unprivileged user inside a container, where Chrome's setuid sandbox + cannot initialize and the browser otherwise fails to launch. + """ + args = ["-y", PACKAGE] + if headless: + args.append("--headless") + if chrome_path is not None: + args += ["--executablePath", str(chrome_path)] + + # Opt out of the MCP server's own data collection: its usage statistics and + # the CrUX API calls that send performance-trace URLs to Google. This does + # not touch Chrome's own behavior, only what the MCP server itself reports. + args += ["--usageStatistics=false", "--performanceCrux=false"] + + if no_sandbox: + args += ["--chromeArg=--no-sandbox", "--chromeArg=--disable-setuid-sandbox"] + + return McpStdioServerConfig(command="npx", args=args) diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_install.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_install.py new file mode 100644 index 0000000000..97b7913786 --- /dev/null +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_install.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import logging +import platform +import shutil +import stat +import sys +import zipfile +from pathlib import Path + +import requests + +VERSIONS_URL = ( + "https://googlechromelabs.github.io/chrome-for-testing/" + "last-known-good-versions-with-downloads.json" +) +CHANNEL = "Stable" +PLATFORM = "linux64" +INSTALL_DIR = Path.home() / "chrome" +DOWNLOAD_TIMEOUT = 120 +CHUNK_SIZE = 1 << 20 + +logger = logging.getLogger("autowebcompat-repro") + + +def reset_install_dir(install_dir: Path) -> None: + if install_dir.exists(): + shutil.rmtree(install_dir) + install_dir.mkdir(parents=True) + + +def resolve_download_url() -> str: + """Look up the Stable channel's Chrome download URL from the API.""" + response = requests.get(VERSIONS_URL, timeout=DOWNLOAD_TIMEOUT) + response.raise_for_status() + data = response.json() + + channel = data["channels"][CHANNEL] + logger.info("Chrome for Testing %s: version %s", CHANNEL, channel["version"]) + + downloads = channel["downloads"]["chrome"] + for download in downloads: + if download["platform"] == PLATFORM: + return download["url"] + + raise RuntimeError( + f"no Chrome for Testing '{PLATFORM}' download in {CHANNEL} channel" + ) + + +def download_archive(url: str, install_dir: Path) -> Path: + archive = install_dir / f"chrome-{PLATFORM}.zip" + + logger.info("downloading Chrome for Testing from %s", url) + with requests.get(url, stream=True, timeout=DOWNLOAD_TIMEOUT) as response: + response.raise_for_status() + with archive.open("wb") as out: + for chunk in response.iter_content(chunk_size=CHUNK_SIZE): + if chunk: + out.write(chunk) + + return archive + + +def extract_archive(archive: Path, install_dir: Path) -> None: + with zipfile.ZipFile(archive) as zf: + zf.extractall(install_dir) + archive.unlink() + + +def chrome_binary_path(install_dir: Path) -> Path: + binary = install_dir / f"chrome-{PLATFORM}" / "chrome" + + if not binary.exists(): + raise RuntimeError(f"Chrome binary not found at {binary} after unpacking") + + return binary + + +def ensure_executable(binary: Path) -> None: + # zipfile does not preserve the executable bit; restore it. + binary.chmod(binary.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +def require_amd64_linux() -> None: + machine = platform.machine().lower() + + if sys.platform == "linux" and machine in {"x86_64", "amd64"}: + return + + raise RuntimeError( + "Chrome for Testing is only supported on linux/amd64 here; got " + f"{sys.platform}/{platform.machine()}. Run the agent image as " + "linux/amd64, e.g. DOCKER_DEFAULT_PLATFORM=linux/amd64." + ) + + +def install_chrome() -> Path: + """Download Chrome for Testing and return the browser binary path.""" + require_amd64_linux() + reset_install_dir(INSTALL_DIR) + + url = resolve_download_url() + archive = download_archive(url, INSTALL_DIR) + extract_archive(archive, INSTALL_DIR) + + binary = chrome_binary_path(INSTALL_DIR) + ensure_executable(binary) + + logger.info("installed Chrome at %s", binary) + return binary diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/config.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/config.py index c331f40d28..cb6635148e 100644 --- a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/config.py +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/config.py @@ -42,3 +42,31 @@ "mcp__firefox-devtools__get_firefox_info", "mcp__firefox-devtools__get_firefox_output", ] + +# Chrome DevTools MCP tools (chrome-devtools-mcp), exposed under the +# "chrome-devtools" server name. Web-compat reproduction subset mirroring the +# Firefox list: page navigation, accessibility snapshots + UID-based +# interaction, console/network inspection, screenshots, and scripted DOM +# probing (evaluate_script). +CHROME_DEVTOOLS_TOOLS = [ + "mcp__chrome-devtools__list_pages", + "mcp__chrome-devtools__new_page", + "mcp__chrome-devtools__navigate_page", + "mcp__chrome-devtools__select_page", + "mcp__chrome-devtools__close_page", + "mcp__chrome-devtools__take_snapshot", + "mcp__chrome-devtools__click", + "mcp__chrome-devtools__hover", + "mcp__chrome-devtools__fill", + "mcp__chrome-devtools__fill_form", + "mcp__chrome-devtools__drag", + "mcp__chrome-devtools__upload_file", + "mcp__chrome-devtools__list_console_messages", + "mcp__chrome-devtools__list_network_requests", + "mcp__chrome-devtools__get_network_request", + "mcp__chrome-devtools__take_screenshot", + "mcp__chrome-devtools__evaluate_script", + "mcp__chrome-devtools__handle_dialog", + "mcp__chrome-devtools__wait_for", + "mcp__chrome-devtools__resize_page", +] diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py index fdb3733c67..bb66f0b827 100644 --- a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py @@ -77,6 +77,16 @@ class ReproductionResult(BaseModel): layout rather than broken interaction. Otherwise it must be null.""" ), ) + chrome_reproduced: bool = Field( + description=( + "Result of re-running the same reproduction steps in Chrome (via the " + "Chrome DevTools MCP): true if the issue also reproduced in Chrome, " + "false if Chrome behaved correctly. Chrome is always tested, " + "regardless of the Firefox outcome. When it reproduces in Firefox " + "but not in Chrome, that indicates a Firefox-specific webcompat " + "issue." + ), + ) @field_validator("screenshot_path", mode="after") @classmethod