diff --git a/agents/autowebcompat-repro/Dockerfile b/agents/autowebcompat-repro/Dockerfile index 97be311881..561bbdabbf 100644 --- a/agents/autowebcompat-repro/Dockerfile +++ b/agents/autowebcompat-repro/Dockerfile @@ -47,11 +47,15 @@ RUN apt-get update \ fonts-liberation \ && rm -rf /var/lib/apt/lists/* -# Install the DevTools MCP servers from the pinned package.json/lockfile +# Install the DevTools MCP servers and Puppeteer from the pinned +# package.json/lockfile. PUPPETEER_SKIP_DOWNLOAD stops Puppeteer's install +# script from fetching its own bundled Chrome: the browsers this agent drives +# are downloaded at startup instead (see browser.py). COPY agents/autowebcompat-repro/package.json \ agents/autowebcompat-repro/package-lock.json \ - /app/node/ -RUN cd /app/node && npm ci --omit=dev + agents/autowebcompat-repro/repro_reference.mjs \ + /app/repro/ +RUN cd /app/repro && PUPPETEER_SKIP_DOWNLOAD=1 npm ci --omit=dev # hackbot.toml lives at the agent root (not inside the package), so copy it into # the working dir; the runtime discovers it there (cwd) at startup. @@ -59,7 +63,7 @@ COPY agents/autowebcompat-repro/hackbot.toml /app/hackbot.toml RUN useradd --create-home --shell /bin/bash agent \ && mkdir -p /workspace \ - && chown agent:agent /workspace + && chown agent:agent /workspace /app/repro USER agent diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py index 8047eb020d..72599154fa 100644 --- a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py @@ -9,6 +9,7 @@ import logging import os +import subprocess import tempfile from abc import ABC, abstractmethod from collections.abc import Callable @@ -30,7 +31,11 @@ from .browser import ChromeBrowsers, FirefoxBrowsers from .config import BUGZILLA_READ_TOOLS, CHROME_DEVTOOLS_TOOLS, DEVTOOLS_TOOLS -from .mcp_servers import build_chrome_devtools_server, build_firefox_devtools_server +from .mcp_servers import ( + REPRO_DIR, + build_chrome_devtools_server, + build_firefox_devtools_server, +) from .result import ( RESULT_SERVER_NAME, SUBMIT_RESULT_TOOL, @@ -48,7 +53,6 @@ logger = logging.getLogger("autowebcompat-repro") - PublishFile = Callable[[str, Path, str | None], str] @@ -79,6 +83,7 @@ class AutowebcompatReproResult(BaseModel): failure_reason: str | None steps: str screenshot_url: str | None + script_url: str | None plan_result: TestPlanResult reproductions: list[tuple[str, BugReproductionResult | ReproductionResult]] chrome_mask_fixed: bool | None @@ -238,6 +243,42 @@ async def run(self) -> ResultT: return self.result_collector.result +def run_confirmation_script( + script_path: Path, firefox_path: Path, chrome_path: Path +) -> ReproductionResult | None: + """Re-run a confirmation script against another Firefox build.""" + script_timeout = 5 * 60 + try: + proc = subprocess.run( + ["node", str(script_path)], + env={ + **os.environ, + "FIREFOX_BIN": str(firefox_path), + "CHROME_BIN": str(chrome_path), + }, + capture_output=True, + text=True, + timeout=script_timeout, + ) + except subprocess.TimeoutExpired: + logger.warning("Confirmation script timed out after %ss", script_timeout) + return None + + logger.info( + "Confirmation script exited %s\nstdout:\n%s\nstderr:\n%s", + proc.returncode, + proc.stdout, + proc.stderr, + ) + if proc.returncode == 0: + return ReproductionResult( + reproduced=True, + failure_reason=None, + screenshot_path=None, + ) + return None + + def make_empty_temp_file(dir: Path, prefix: str | None, suffix: str) -> Path: fd, path = tempfile.mkstemp(prefix=prefix, suffix=suffix, dir=dir) f = os.fdopen(fd) @@ -322,9 +363,12 @@ def __init__( ): super().__init__(task_config, run_tracker) self.input_data = input_data + self.firefox_path = firefox_path + self.chrome_path = chrome_path self.screenshot_path = make_empty_temp_file( screenshot_dir, "reproduction=", ".png" ) + self.script_path = make_empty_temp_file(REPRO_DIR, "reproduction=", ".mjs") self.add_mcp_server( "firefox-devtools", build_firefox_devtools_server( @@ -349,6 +393,7 @@ def subject(self) -> Any: return self.input_data.subject() def system_prompt(self) -> str: + repro_reference = REPRO_DIR / "repro_reference.mjs" return ( super() .system_prompt() @@ -371,14 +416,26 @@ def system_prompt(self) -> str: web-compat issue; refine the steps and re-check before concluding. - If the reported broken behaviour reproduces in both browsers, it is not a Firefox web-compat issue: set `failure_reason` to `non_compat`. -4. If the issue reproduces AND the breakage is visual in nature (incorrect +4. Write and run a Puppeteer script that drives the real reported site in both + browsers and demonstrates the divergence: follow the spec in + `{repro_reference}`, write your script to exactly `{self.script_path}`, then run + it with + + `FIREFOX_BIN={self.firefox_path} CHROME_BIN={self.chrome_path} node {self.script_path}` + + On exit 0, set `script_path` in your result to that path. Otherwise revise and + re-run until the script both executes cleanly and demonstrates the difference. + If you're unable to do so, you have not confirmed the issue, so report + `reproduced` as false with `script_path` null and the `failure_reason` you + observed (likely `not_reproducable`). +5. If the issue reproduces AND the breakage is visual in nature (incorrect layout or rendering, not broken interaction), capture a screenshot in Firefox showing it: call `screenshot_page` with `saveTo` set to `{self.screenshot_path}`. This writes the image to that file instead of returning it — do not capture or paste the image data yourself. Then set `screenshot_path` in your result to exactly `{self.screenshot_path}`. For non-visual issues, take no screenshot and leave `screenshot_path` null. -5. Submit your findings via `submit_result` (see "Reporting your result"). +6. Submit your findings via `submit_result` (see "Reporting your result"). """ ) ) @@ -513,6 +570,7 @@ class InitialReproduction: steps: str summary: str screenshot_path: Path | None + script_path: Path | None chrome_reproduced: bool | None @@ -578,6 +636,20 @@ def screenshot_url(self) -> str | None: return None + @property + def script_url(self) -> str | None: + if ( + self.initial_repro is not None + and self.initial_repro.script_path is not None + ): + return self.publish_file( + f"reproduction-{self.initial_repro.channel}.mjs", + self.initial_repro.script_path, + "text/javascript", + ) + + return None + def set_result( self, channel: FirefoxChannel, @@ -595,6 +667,7 @@ def set_result( result.steps, result.summary, result.screenshot_path, + result.script_path, result.chrome_reproduced, ) elif isinstance(result, ChromeMaskResult): @@ -611,9 +684,15 @@ def into_result(self) -> AutowebcompatReproResult: failure_reason=self.failure_reason, steps=self.steps, screenshot_url=self.screenshot_url, + script_url=self.script_url, plan_result=self.plan_result, reproductions=[ - (key[0].value, value.model_copy(update={"screenshot_path": None})) + ( + key[0].value, + value.model_copy( + update={"screenshot_path": None, "script_path": None} + ), + ) for key, value in self.results.items() if isinstance(value, ReproductionResult) ], @@ -659,7 +738,14 @@ async def next_repro_task( ) -> None: browser = getattr(firefox_browser, channel.value) profile = setup_profile(browser) - if repro_results.initial_repro is None: + logger.info( + "Trying reproduction in %s%s", + channel, + f" {extra}" if extra is not None else "", + ) + + initial_repro = repro_results.initial_repro + if initial_repro is None: task: Task = BugReproduction( config, tracker, @@ -671,18 +757,25 @@ async def next_repro_task( chrome_browser.stable, ) else: + if initial_repro.script_path is not None: + script_result = run_confirmation_script( + initial_repro.script_path, browser, chrome_browser.stable + ) + if script_result is not None: + repro_results.set_result(channel, extra, script_result) + return + logger.info( + "Confirmation script did not reach a verdict in %s; " + "falling back to the reproduction steps", + channel, + ) task = StepsReproduction( config, tracker, browser, profile, - repro_results.initial_repro.steps, + initial_repro.steps, ) - logger.info( - "Trying reproduction in %s%s", - channel, - f" {extra}" if extra is not None else "", - ) repro_results.set_result(channel, extra, await task.run()) screenshots_dir = Path(tempfile.mkdtemp(prefix="autowebcompat-screenshots-")) diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/mcp_servers.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/mcp_servers.py index 9cbdd380dc..8837ed5920 100644 --- a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/mcp_servers.py +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/mcp_servers.py @@ -10,10 +10,12 @@ from claude_agent_sdk.types import McpStdioServerConfig +REPRO_DIR = Path("/app/repro") + def resolve_bin(bin_name: str) -> str: """Resolve an installed MCP server binary to an absolute path.""" - binary = Path("/app/node") / "node_modules" / ".bin" / bin_name + binary = REPRO_DIR / "node_modules" / ".bin" / bin_name if not binary.exists(): raise RuntimeError( f"MCP server binary not found at {binary}; the image should install " diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/prompts/system.md b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/prompts/system.md index 2b31c5dc51..e1d0a91a51 100644 --- a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/prompts/system.md +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/prompts/system.md @@ -10,6 +10,9 @@ issues that reproduce in Firefox. section. - Your job is to analyze and, when instructed, reproduce the reported issue. Do not attempt to debug or perform root cause analysis. +- No `Monitor` or `ScheduleWakeup` tools are available. If you attempt + to use these tools, nothing will notify you, and you will stall and + lose your findings. **Stay focused on reproduction. Avoid:** diff --git a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py index 239564a385..aff9c02d0b 100644 --- a/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py +++ b/agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py @@ -7,7 +7,13 @@ from typing import Generic, Literal, TypeVar from claude_agent_sdk import McpServerConfig, create_sdk_mcp_server, tool -from pydantic import BaseModel, Field, ValidationError, field_validator +from pydantic import ( + BaseModel, + Field, + ValidationError, + field_validator, + model_validator, +) RESULT_SERVER_NAME = "autowebcompat-repro" SUBMIT_RESULT_TOOL = f"mcp__{RESULT_SERVER_NAME}__submit_result" @@ -148,6 +154,44 @@ class BugReproductionResult(ReproductionResult): ), ) + script_path: Path | None = Field( + description=( + """The file path of the Puppeteer confirmation script you wrote and + ran successfully (exit code 0). Use the exact path you were given + to write to (do NOT paste the script source). Must be null when + `reproduced` is false.""" + ), + ) + + @field_validator("script_path", mode="after") + @classmethod + def validate_script_path(cls, path: Path | None) -> Path | None: + if path is None: + return None + + if not path.exists(): + raise ValueError(f"Script path {path} doesn't exist") + if not path.read_text().strip(): + raise ValueError(f"Script path {path} is empty") + return path + + @model_validator(mode="after") + def validate_script_matches_reproduced(self) -> BugReproductionResult: + """A reproduction is only real once a script has confirmed it.""" + if self.reproduced and self.script_path is None: + raise ValueError( + "reproduced is true but script_path is null: a reproduction " + "requires a Puppeteer script that is run in both Firefox and Chrome" + "Write and run one, then resubmit — or set reproduced to false " + "with the appropriate failure_reason if you cannot." + ) + if not self.reproduced and self.script_path is not None: + raise ValueError( + "script_path must be null when reproduced is false; a script " + "that does not demonstrate the issue is not a confirmation." + ) + return self + class ChromeMaskResult(BaseModel): chrome_mask_fixed: bool | None = Field( diff --git a/agents/autowebcompat-repro/package-lock.json b/agents/autowebcompat-repro/package-lock.json index 863531ace5..8ffe81547d 100644 --- a/agents/autowebcompat-repro/package-lock.json +++ b/agents/autowebcompat-repro/package-lock.json @@ -1,15 +1,16 @@ { "name": "hackbot-agent-autowebcompat-repro-mcp", - "version": "0.0.0", + "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "hackbot-agent-autowebcompat-repro-mcp", - "version": "0.0.0", + "version": "1.0.0", "dependencies": { "@mozilla/firefox-devtools-mcp-moz": "0.9.12", - "chrome-devtools-mcp": "1.5.0" + "chrome-devtools-mcp": "1.5.0", + "puppeteer": "25.4.0" } }, "node_modules/@bazel/runfiles": { @@ -89,6 +90,168 @@ "node": ">=20.19.0" } }, + "node_modules/@puppeteer/browsers": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.0.6.tgz", + "integrity": "sha512-B/gKoqlFkzhvzsI6jo9K1cZz9o5ypviVv/xu8CwA4grZzyVwN+XfkT+tu8T1zrauuEXv6VhS2oGX+6NL95WcKA==", + "license": "Apache-2.0", + "dependencies": { + "modern-tar": "^0.7.6", + "yargs": "^18.0.0" + }, + "bin": { + "browsers": "lib/main-cli.js" + }, + "engines": { + "node": ">=22.12.0" + }, + "peerDependencies": { + "proxy-agent": ">=8.0.1", + "yauzl": "^2.10.0 || ^3.4.0" + }, + "peerDependenciesMeta": { + "proxy-agent": { + "optional": true + }, + "yauzl": { + "optional": true + } + } + }, + "node_modules/@puppeteer/browsers/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "license": "ISC", + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@puppeteer/browsers/node_modules/cliui/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@puppeteer/browsers/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/@puppeteer/browsers/node_modules/modern-tar": { + "version": "0.7.7", + "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.7.7.tgz", + "integrity": "sha512-t9VmxaqrmANnEOBhpSDI6HD192Ge48k8vmWqQQL7hSFEqHEYwZbbsu49+aKLWZeRvFs3j1pMhXOqqF4kPlvjkQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@puppeteer/browsers/node_modules/string-width": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.2.tgz", + "integrity": "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@puppeteer/browsers/node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@puppeteer/browsers/node_modules/wrap-ansi/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@puppeteer/browsers/node_modules/yargs": { + "version": "18.1.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.1.0.tgz", + "integrity": "sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==", + "license": "MIT", + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^8.2.1", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/@puppeteer/browsers/node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, "node_modules/@wdio/logger": { "version": "9.29.1", "resolved": "https://registry.npmjs.org/@wdio/logger/-/logger-9.29.1.tgz", @@ -306,6 +469,31 @@ } } }, + "node_modules/chromium-bidi": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-17.0.2.tgz", + "integrity": "sha512-5v9GQFhTktFvotn/OFNJBmKLKRAb6n9r0bVCwf7sHgWc3/JryK0bj1nn93L3pHFrfgcsu6Be6EWsDi+1XHTGDg==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "engines": { + "node": ">=20.19.0 <22.0.0 || >=22.12.0" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/chromium-bidi/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -474,6 +662,12 @@ "node": ">= 0.8" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1653615", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1653615.tgz", + "integrity": "sha512-pGVkY3T/qXxAp2nFPodwYqOevk6ncNMSmvL8QfRCx5ZWGd6Vor7AFNmyaA8Zs6uJyP1QAfjuLandCgvSix1BNA==", + "license": "BSD-3-Clause" + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -745,6 +939,18 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-intrinsic": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", @@ -988,6 +1194,18 @@ "immediate": "~3.0.5" } }, + "node_modules/lilconfig": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", + "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, "node_modules/loglevel": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz", @@ -1062,6 +1280,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, "node_modules/modern-tar": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.3.5.tgz", @@ -1190,6 +1414,65 @@ "node": ">= 0.10" } }, + "node_modules/puppeteer": { + "version": "25.4.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-25.4.0.tgz", + "integrity": "sha512-xfQp8dFBcGaLc1hEMaVr7s+oW4ZkAurr8Y9H81ilKhu6QoLfSTkZjU7IavnyJ/VWpB9ni3KNJUQHUatslLWyGw==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "3.0.6", + "chromium-bidi": "17.0.2", + "devtools-protocol": "0.0.1653615", + "lilconfig": "^3.1.3", + "puppeteer-core": "25.4.0", + "typed-query-selector": "^2.12.2" + }, + "bin": { + "puppeteer": "lib/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/puppeteer-core": { + "version": "25.4.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.4.0.tgz", + "integrity": "sha512-K1plkLOdeoUnGeT1OvdqF3qxl33v+Ra/uH5VyPEhXdMcpvGiEskHzxxEU3fgpccJpJLIipB/rPUsvkZRWeKqOA==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "3.0.6", + "chromium-bidi": "17.0.2", + "devtools-protocol": "0.0.1653615", + "typed-query-selector": "^2.12.2", + "webdriver-bidi-protocol": "0.4.2", + "ws": "^8.21.1" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/puppeteer-core/node_modules/ws": { + "version": "8.21.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", + "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/qs": { "version": "6.15.3", "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", @@ -1618,6 +1901,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/typed-query-selector": { + "version": "2.12.2", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz", + "integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==", + "license": "MIT" + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -1642,6 +1931,12 @@ "node": ">= 0.8" } }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.2.tgz", + "integrity": "sha512-VSV+fzfChirL3e7jay2yUC7B4HQCGtEWEg/MSSQbK+qWbqeGlRLlXTzPpYr3XGUvbpDHumWZBJxgesg4N7dbtA==", + "license": "Apache-2.0" + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/agents/autowebcompat-repro/package.json b/agents/autowebcompat-repro/package.json index e019291d0e..0a8f06e964 100644 --- a/agents/autowebcompat-repro/package.json +++ b/agents/autowebcompat-repro/package.json @@ -5,6 +5,7 @@ "description": "MCP servers the autowebcompat-repro agent drives.", "dependencies": { "@mozilla/firefox-devtools-mcp-moz": "0.9.12", - "chrome-devtools-mcp": "1.5.0" + "chrome-devtools-mcp": "1.5.0", + "puppeteer": "25.4.0" } } diff --git a/agents/autowebcompat-repro/repro_reference.mjs b/agents/autowebcompat-repro/repro_reference.mjs new file mode 100644 index 0000000000..919b415777 --- /dev/null +++ b/agents/autowebcompat-repro/repro_reference.mjs @@ -0,0 +1,68 @@ +// REFERENCE — write your own script in this shape; don't copy this comment. +// +// Fill in `probe` (run the steps, return measurements) and `reproduced` (assert +// broken in Firefox AND working in Chrome). Keep the rest as-is. Add a comment in +// your script covering the bug data, the expected behaviour, and what breaks in Firefox. +// +// Run: +// FIREFOX_BIN=/path/to/firefox CHROME_BIN=/path/to/chrome node this-script.mjs +// +// Exit code: +// 0 = Firefox-specific breakage reproduced: broken in Firefox, working in Chrome. +// 1 = no Firefox-specific breakage: it worked in both, broke in both, or only Chrome broke. +// 2 = no verdict — a browser wouldn't launch, or the script itself broke. + +import puppeteer from "puppeteer"; + +const { FIREFOX_BIN, CHROME_BIN } = process.env; +if (!FIREFOX_BIN || !CHROME_BIN) { + console.error("set FIREFOX_BIN and CHROME_BIN to the browser binaries"); + process.exit(2); +} + +const TARGET = "https://example.com/"; + +async function probe(name, executablePath) { + const browser = await puppeteer.launch({ + browser: name, + executablePath, + headless: true, + ...(name === "chrome" ? { args: ["--no-sandbox"] } : {}), + }); + try { + const page = await browser.newPage(); + await page.goto(TARGET, { waitUntil: "networkidle0" }); + return await page.evaluate(() => ({})); + } catch (error) { + return { error: String(error?.message ?? error) }; + } finally { + await browser.close(); + } +} + +function reproduced(ff, cr) { + return false; +} + +const RUNS = 3 + +let reproductions = 0; +try { + for (let i = 1; i <= RUNS; i++) { + const ff = await probe("firefox", FIREFOX_BIN); + const cr = await probe("chrome", CHROME_BIN); + const ok = reproduced(ff, cr); + if (ok) reproductions++; + console.log( + `Run ${i}: firefox=${JSON.stringify(ff)} chrome=${JSON.stringify( + cr + )} reproduced=${ok}` + ); + } +} catch (error) { + console.error("FATAL:", error); + process.exit(2); +} + +console.log(`\nReproduced ${reproductions}/${RUNS} runs.`); +process.exit(reproductions === RUNS ? 0 : 1);