diff --git a/agents/autowebcompat-repro/Dockerfile b/agents/autowebcompat-repro/Dockerfile index 4740cc4ce6..ff5264b0fc 100644 --- a/agents/autowebcompat-repro/Dockerfile +++ b/agents/autowebcompat-repro/Dockerfile @@ -28,17 +28,22 @@ ENV PATH="/opt/venv/bin:$PATH" FROM base AS agent -# The Firefox DevTools MCP server is an npm package launched via `npx`, so the -# agent image needs Node.js + npm (the python base ships neither). It also -# needs the shared libraries Firefox requires to run headless; the Firefox -# binary itself is downloaded at agent startup (a fresh Nightly per run) via -# mozdownload/mozinstall, not baked in here. +# The Firefox and Chrome DevTools MCP servers are npm packages launched via +# `npx`, so the agent image needs Node.js + npm (the python base ships neither). +# It also needs the shared libraries the browsers require to run headless. The +# browser binaries themselves are downloaded at agent startup (a fresh build per +# run): Firefox via mozdownload/mozinstall and Chrome for Testing via the +# Chrome-for-Testing JSON API (see browser.py). The first `lib*` group below is +# Firefox's runtime deps; the rest are the extra deps headless Chrome needs. RUN apt-get update \ && apt-get install -y --no-install-recommends \ nodejs npm \ ca-certificates \ - libgtk-3-0 libdbus-glib-1-2 libx11-xcb1 libxtst6 libxt6 \ - libasound2 libpci3 \ + libgtk-3-0 libdbus-glib-1-2 libx11-xcb1 libxtst6 libxt6 libpci3 \ + libnss3 libnspr4 libatk1.0-0t64 libatk-bridge2.0-0t64 libatspi2.0-0t64 \ + libcups2t64 libdbus-1-3 libgbm1 libxcomposite1 libxdamage1 libxfixes3 \ + libxrandr2 libxkbcommon0 libasound2t64 libpango-1.0-0 libcairo2 \ + fonts-liberation \ && rm -rf /var/lib/apt/lists/* # hackbot.toml lives at the agent root (not inside the package), so copy it into diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py index 529bbfe722..ad4da76372 100644 --- a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py @@ -28,8 +28,9 @@ from hackbot_runtime.claude import Reporter from pydantic import BaseModel -from .browser import FirefoxBrowsers -from .config import BUGZILLA_READ_TOOLS, DEVTOOLS_TOOLS +from .browser import ChromeBrowsers, FirefoxBrowsers +from .chrome_devtools_mcp import build_chrome_devtools_server +from .config import BUGZILLA_READ_TOOLS, CHROME_DEVTOOLS_TOOLS, DEVTOOLS_TOOLS from .devtools_mcp import build_devtools_server from .result import ( RESULT_SERVER_NAME, @@ -82,6 +83,7 @@ class AutowebcompatReproResult(BaseModel): plan_result: TestPlanResult reproductions: list[tuple[str, BugReproductionResult | ReproductionResult]] chrome_mask_fixed: bool | None + chrome_reproduced: bool | None @dataclass @@ -317,9 +319,13 @@ def __init__( input_data: AutoWebcompatInput, bugzilla_mcp_server: McpServerConfig, screenshot_dir: Path, + chrome_path: Path | None = None, ): super().__init__(task_config, run_tracker) self.input_data = input_data + # Chrome cross-check is only added when a Chrome binary is supplied + # (the initial nightly attempt). Later channel attempts run Firefox-only. + self.cross_check_chrome = chrome_path is not None self.screenshot_path = make_empty_temp_file( screenshot_dir, "reproduction=", ".png" ) @@ -334,6 +340,16 @@ def __init__( ), DEVTOOLS_TOOLS, ) + # The initial reproduction also drives Chrome, so the agent can + # cross-check both browsers in one context: if the same steps behave + # identically in Firefox and Chrome it can iterate on whether the steps + # are wrong versus it genuinely not being a compat issue. + if chrome_path is not None: + self.add_mcp_server( + "chrome-devtools", + build_chrome_devtools_server(chrome_path=chrome_path, headless=True), + CHROME_DEVTOOLS_TOOLS, + ) if self.input_data.type == "bug_id" != None: self.add_mcp_server("bugzilla", bugzilla_mcp_server, BUGZILLA_READ_TOOLS) @@ -341,22 +357,47 @@ def subject(self) -> Any: return self.input_data.subject() def system_prompt(self) -> str: + if self.cross_check_chrome: + chrome_step = """ +3. Cross-check in Chrome: run the same steps in Chrome using the Chrome DevTools + MCP. A genuine web-compat issue reproduces in Firefox but not in in Chrome. + - If the behavior is identical in both browsers, your steps may be wrong or + this may not be a compat issue. Iterate: refine the steps and re-check both + browsers before concluding. Use the difference (or lack of it) between the + two browsers to decide whether you have really reproduced the reported + issue. + - Set `reproduced` for the Firefox outcome and `chrome_reproduced` for the + Chrome outcome (`true` if the issue also reproduces in Chrome, `false` if + the issue is not reproducible in Chrome).""" + else: + chrome_step = """ +3. No Chrome cross-check is available for this task: leave `chrome_reproduced` + null.""" + return ( super() .system_prompt() .format( task_details=f""" 1. Identify the affected URL and the described broken behavior. -2. Baseline: Navigate to the URL with the Firefox DevTools MCP and - try to reproduce the described broken behaviour. -3. If the issue reproduces AND the breakage is visual in nature (incorrect - layout or rendering, not broken interaction), capture a screenshot showing - it: call `screenshot_page` with `saveTo` set to `{self.screenshot_path}`. - This writes the image to that file instead of returning it — do not capture - or paste the image data yourself. Then set `screenshot_path` in your result - to exactly `{self.screenshot_path}`. For non-visual issues, take no - screenshot and leave `screenshot_path` null. -4. Submit your findings via `submit_result` (see "Reporting your result"). +2. Baseline: Navigate to the URL with the Firefox DevTools MCP + and try to reproduce the described broken behaviour. + - Reproduce against the actual reported site. If you cannot reach or + reproduce on that site — e.g. it is behind a login wall, blocked, + gated by a captcha, or down — do not substitute a reduced testcase, + minimal example, or other bug attachment as a stand-in reproduction. + Report `reproduced` as false with the appropriate `failure_reason`. A + testcase may inform your analysis, but reproducing on it does not count + as reproducing the reported site issue. +{chrome_step} +4. If the issue reproduces AND the breakage is visual in nature (incorrect + layout or rendering, not broken interaction), capture a screenshot in Firefox + showing it: call `screenshot_page` with `saveTo` set to + `{self.screenshot_path}`. This writes the image to that file instead of + returning it — do not capture or paste the image data yourself. Then set + `screenshot_path` in your result to exactly `{self.screenshot_path}`. For + non-visual issues, take no screenshot and leave `screenshot_path` null. +5. Submit your findings via `submit_result` (see "Reporting your result"). """ ) ) @@ -502,6 +543,7 @@ def __init__(self, publish_file: PublishFile, plan_result: TestPlanResult): ] = {} self.initial_repro: InitialReproduction | None = None self.chrome_mask_fixed: bool | None = None + self.chrome_reproduced: bool | None = None @property def reproduced(self) -> bool: @@ -551,12 +593,19 @@ def set_result( key = (channel, extra) if key in self.results: raise ValueError(f"Got duplicate results for {channel}, {extra}") - if isinstance(result, BugReproductionResult) and result.reproduced: - if self.initial_repro is not None: - raise ValueError("Got duplicate steps / summary") - self.initial_repro = InitialReproduction( - channel, result.steps, result.summary, result.screenshot_path - ) + if isinstance(result, BugReproductionResult): + if result.reproduced: + if self.initial_repro is not None: + raise ValueError("Got duplicate steps / summary") + self.initial_repro = InitialReproduction( + channel, result.steps, result.summary, result.screenshot_path + ) + # Only the Chrome-enabled (initial nightly) reproduction reports a + # Chrome verdict; other attempts leave it null. + if result.chrome_reproduced is not None: + if self.chrome_reproduced is not None: + raise ValueError("Got duplicate chrome cross-check results") + self.chrome_reproduced = result.chrome_reproduced elif isinstance(result, ChromeMaskResult): if self.chrome_mask_fixed is not None: raise ValueError("Got duplicate results for chrome mask") @@ -578,6 +627,7 @@ def into_result(self) -> AutowebcompatReproResult: if isinstance(value, ReproductionResult) ], chrome_mask_fixed=self.chrome_mask_fixed, + chrome_reproduced=self.chrome_reproduced, ) @@ -594,6 +644,7 @@ async def run_autowebcompat_repro( :class:`AgentError` if the agent ends in an error. """ firefox_browser = FirefoxBrowsers() + chrome_browser = ChromeBrowsers() test_plan_task = TestPlan(default_config, tracker, input_data, bugzilla_mcp_server) test_plan_result = await test_plan_task.run() @@ -614,6 +665,7 @@ async def next_repro_task( channel: FirefoxChannel, extra: str | None = None, config: TaskConfig = default_config, + cross_check_chrome: bool = False, ) -> None: browser = getattr(firefox_browser, channel.value) profile = setup_profile(browser) @@ -626,6 +678,9 @@ async def next_repro_task( input_data, bugzilla_mcp_server, screenshots_dir, + # Only cross-check Chrome when asked (the initial nightly + # attempt); later channel attempts stay Firefox-only. + chrome_path=chrome_browser.stable if cross_check_chrome else None, ) else: task = StepsReproduction( @@ -644,8 +699,8 @@ async def next_repro_task( screenshots_dir = Path(tempfile.mkdtemp(prefix="autowebcompat-screenshots-")) - # Always try in nightly first - await next_repro_task(FirefoxChannel.nightly) + # Always try in nightly first, cross-checking in Chrome. + await next_repro_task(FirefoxChannel.nightly, cross_check_chrome=True) if not repro_results.reproduced and test_plan_result.affects_platforms == [ "android" diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/browser.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/browser.py index 189e3dd551..a06ff057af 100644 --- a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/browser.py +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/browser.py @@ -1,14 +1,24 @@ import logging import platform +import stat import tempfile +import zipfile from pathlib import Path from typing import Literal import mozdownload import mozinstall +import requests logger = logging.getLogger("autowebcompat-repro") +CHROME_VERSIONS_URL = ( + "https://googlechromelabs.github.io/chrome-for-testing/" + "last-known-good-versions-with-downloads.json" +) +CHROME_DOWNLOAD_TIMEOUT = 120 +CHROME_CHUNK_SIZE = 1 << 20 + def install_firefox( channel: Literal["nightly"] | Literal["stable"] | Literal["esr"], @@ -72,3 +82,82 @@ def esr(self) -> Path: if self._esr is None: self._esr = install_firefox(channel="esr") return self._esr + + +def chrome_platform() -> str: + """Chrome for Testing platform string for the current host.""" + system = platform.system() + machine = platform.machine().lower() + if system == "Linux": + if machine not in {"x86_64", "amd64"}: + raise RuntimeError( + "Chrome for Testing has no linux build for " + f"{platform.machine()}; only x86_64/amd64 is supported. Run the " + "agent image as linux/amd64, e.g. DOCKER_DEFAULT_PLATFORM=linux/amd64." + ) + return "linux64" + if system == "Darwin": + return "mac-arm64" if machine in {"arm64", "aarch64"} else "mac-x64" + if system == "Windows": + return "win64" if machine in {"x86_64", "amd64"} else "win32" + raise RuntimeError(f"Unsupported platform for Chrome for Testing: {system}") + + +def resolve_chrome_download_url(channel: str, cft_platform: str) -> str: + """Look up the Chrome for Testing download URL for a channel + platform.""" + response = requests.get(CHROME_VERSIONS_URL, timeout=CHROME_DOWNLOAD_TIMEOUT) + response.raise_for_status() + data = response.json() + + entry = data["channels"][channel.capitalize()] + logger.info("Chrome for Testing %s: version %s", channel, entry["version"]) + + for download in entry["downloads"]["chrome"]: + if download["platform"] == cft_platform: + return download["url"] + + raise RuntimeError( + f"no Chrome for Testing '{cft_platform}' download in {channel} channel" + ) + + +def install_chrome(channel: Literal["stable"] = "stable") -> Path: + """Download Chrome for Testing and return the browser binary path.""" + cft_platform = chrome_platform() + install_dir = Path(tempfile.mkdtemp(prefix=f"chrome-{channel}-", dir=Path.home())) + + url = resolve_chrome_download_url(channel, cft_platform) + archive = install_dir / f"chrome-{cft_platform}.zip" + + logger.info("downloading Chrome for Testing from %s", url) + with requests.get(url, stream=True, timeout=CHROME_DOWNLOAD_TIMEOUT) as response: + response.raise_for_status() + with archive.open("wb") as out: + for chunk in response.iter_content(chunk_size=CHROME_CHUNK_SIZE): + if chunk: + out.write(chunk) + + with zipfile.ZipFile(archive) as zf: + zf.extractall(install_dir) + archive.unlink() + + binary = install_dir / f"chrome-{cft_platform}" / "chrome" + if not binary.exists(): + raise RuntimeError(f"Chrome binary not found at {binary} after unpacking") + + # zipfile does not preserve the executable bit; restore it. + binary.chmod(binary.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + logger.info("installed Chrome at %s", binary) + return binary + + +class ChromeBrowsers: + def __init__(self) -> None: + self._stable: Path | None = None + + @property + def stable(self) -> Path: + if self._stable is None: + self._stable = install_chrome(channel="stable") + return self._stable diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_devtools_mcp.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_devtools_mcp.py new file mode 100644 index 0000000000..3cbd24dfbf --- /dev/null +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/chrome_devtools_mcp.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from pathlib import Path + +from claude_agent_sdk.types import McpStdioServerConfig + +PACKAGE = "chrome-devtools-mcp@latest" + + +def build_chrome_devtools_server( + chrome_path: Path | None = None, + *, + headless: bool = True, + no_sandbox: bool = True, +) -> McpStdioServerConfig: + """Build the stdio config for the Chrome DevTools MCP server. + + Args: + chrome_path: Chrome binary to drive (the Chrome for Testing build from + ``browser.install_chrome``). When ``None`` the server lets its + bundled Puppeteer discover a Chrome installation itself. + headless: Run Chrome without a visible window (required in + container/CI environments). + no_sandbox: Pass ``--no-sandbox`` to Chrome. Required when running as an + unprivileged user inside a container, where Chrome's setuid sandbox + cannot initialize and the browser otherwise fails to launch. + """ + args = ["-y", PACKAGE] + if headless: + args.append("--headless") + if chrome_path is not None: + args += ["--executablePath", str(chrome_path)] + + # Opt out of the MCP server's own data collection: its usage statistics and + # the CrUX API calls that send performance-trace URLs to Google. This does + # not touch Chrome's own behavior, only what the MCP server itself reports. + args += ["--usageStatistics=false", "--performanceCrux=false"] + + if no_sandbox: + args += ["--chromeArg=--no-sandbox", "--chromeArg=--disable-setuid-sandbox"] + + return McpStdioServerConfig(command="npx", args=args) diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/config.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/config.py index c331f40d28..84eb9b5395 100644 --- a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/config.py +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/config.py @@ -11,8 +11,7 @@ # the "firefox-devtools" server name. Web-compat reproduction subset: page # navigation, accessibility snapshots + UID-based interaction, console/network # inspection, screenshots, and scripted DOM probing (evaluate_script needs -# --enable-script). Privileged-context and extension tools are intentionally -# omitted for now. +# --enable-script). DEVTOOLS_TOOLS = [ "mcp__firefox-devtools__list_pages", "mcp__firefox-devtools__new_page", @@ -42,3 +41,31 @@ "mcp__firefox-devtools__get_firefox_info", "mcp__firefox-devtools__get_firefox_output", ] + +# Chrome DevTools MCP tools (chrome-devtools-mcp), exposed under the +# "chrome-devtools" server name. Web-compat reproduction subset mirroring the +# Firefox list: page navigation, accessibility snapshots + UID-based +# interaction, console/network inspection, screenshots, and scripted DOM +# probing (evaluate_script). +CHROME_DEVTOOLS_TOOLS = [ + "mcp__chrome-devtools__list_pages", + "mcp__chrome-devtools__new_page", + "mcp__chrome-devtools__navigate_page", + "mcp__chrome-devtools__select_page", + "mcp__chrome-devtools__close_page", + "mcp__chrome-devtools__take_snapshot", + "mcp__chrome-devtools__click", + "mcp__chrome-devtools__hover", + "mcp__chrome-devtools__fill", + "mcp__chrome-devtools__fill_form", + "mcp__chrome-devtools__drag", + "mcp__chrome-devtools__upload_file", + "mcp__chrome-devtools__list_console_messages", + "mcp__chrome-devtools__list_network_requests", + "mcp__chrome-devtools__get_network_request", + "mcp__chrome-devtools__take_screenshot", + "mcp__chrome-devtools__evaluate_script", + "mcp__chrome-devtools__handle_dialog", + "mcp__chrome-devtools__wait_for", + "mcp__chrome-devtools__resize_page", +] diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py index 4450428077..b9c1ada70f 100644 --- a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py @@ -112,13 +112,27 @@ def validate_screenshot_path(cls, path: Path | None) -> Path | None: class BugReproductionResult(ReproductionResult): - """Canonical result the agent produces for a web-compat investigation.""" + """Canonical result the agent produces for a web-compat investigation. + + Produced by the initial reproduction task, which drives both Firefox and + Chrome so it can cross-check the two browsers in a single context. + """ summary: str = Field( description="""A concise account of whether the issue represents a real webcompat issue i.e. it can be reproduced in Firefox.""" ) + chrome_reproduced: bool | None = Field( + description=( + "Result of running the cross-check step in Chrome: " + "true if the issue also reproduces in Chrome, false if the " + "issue does not reproduce, or null if this task had " + "no Chrome DevTools MCP. When the issue reproduces " + "in Firefox but not in Chrome, that indicates a Firefox-specific webcompat issue." + ), + ) + steps: str = Field( description=( "The ordered steps you took, as a single numbered list (1., 2., 3., " @@ -128,8 +142,8 @@ class BugReproductionResult(ReproductionResult): "provide (a file, image, account, or any other test data), state its " "exact origin — the URL you fetched it from, the command you ran, or " 'how you generated it — not just that you "used" or "saved" it. A ' - "reader must be able to obtain the same inputs. Omit the reproduction " - "screenshot step." + "reader must be able to obtain the same inputs. Omit the Chrome cross-check " + "reproduction and screenshot steps." ), )