Make creation of Puppeteer script part of reproduction step for autowebcompat

agents/autowebcompat-repro/Dockerfile

@@ -47,19 +47,23 @@ RUN apt-get update \
         fonts-liberation \
     && rm -rf /var/lib/apt/lists/*
 
-# Install the DevTools MCP servers from the pinned package.json/lockfile
+# Install the DevTools MCP servers and Puppeteer from the pinned
+# package.json/lockfile. PUPPETEER_SKIP_DOWNLOAD stops Puppeteer's install
+# script from fetching its own bundled Chrome: the browsers this agent drives
+# are downloaded at startup instead (see browser.py).
 COPY agents/autowebcompat-repro/package.json \
      agents/autowebcompat-repro/package-lock.json \
-     /app/node/
-RUN cd /app/node && npm ci --omit=dev
+     agents/autowebcompat-repro/repro_reference.mjs \
+     /app/repro/
+RUN cd /app/repro && PUPPETEER_SKIP_DOWNLOAD=1 npm ci --omit=dev
 
 # hackbot.toml lives at the agent root (not inside the package), so copy it into
 # the working dir; the runtime discovers it there (cwd) at startup.
 COPY agents/autowebcompat-repro/hackbot.toml /app/hackbot.toml
 
 RUN useradd --create-home --shell /bin/bash agent \
     && mkdir -p /workspace \
-    && chown agent:agent /workspace
+    && chown agent:agent /workspace /app/repro
 
 USER agent
 

agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/agent.py

@@ -9,6 +9,7 @@
 
 import logging
 import os
+import subprocess
 import tempfile
 from abc import ABC, abstractmethod
 from collections.abc import Callable
@@ -30,7 +31,11 @@
 
 from .browser import ChromeBrowsers, FirefoxBrowsers
 from .config import BUGZILLA_READ_TOOLS, CHROME_DEVTOOLS_TOOLS, DEVTOOLS_TOOLS
-from .mcp_servers import build_chrome_devtools_server, build_firefox_devtools_server
+from .mcp_servers import (
+    REPRO_DIR,
+    build_chrome_devtools_server,
+    build_firefox_devtools_server,
+)
 from .result import (
     RESULT_SERVER_NAME,
     SUBMIT_RESULT_TOOL,
@@ -48,7 +53,6 @@
 
 logger = logging.getLogger("autowebcompat-repro")
 
-
 PublishFile = Callable[[str, Path, str | None], str]
 
 
@@ -79,6 +83,7 @@ class AutowebcompatReproResult(BaseModel):
     failure_reason: str | None
     steps: str
     screenshot_url: str | None
+    script_url: str | None
     plan_result: TestPlanResult
     reproductions: list[tuple[str, BugReproductionResult | ReproductionResult]]
     chrome_mask_fixed: bool | None
@@ -238,6 +243,42 @@ async def run(self) -> ResultT:
         return self.result_collector.result
 
 
+def run_confirmation_script(
+    script_path: Path, firefox_path: Path, chrome_path: Path
+) -> ReproductionResult | None:
+    """Re-run a confirmation script against another Firefox build."""
+    script_timeout = 5 * 60
+    try:
+        proc = subprocess.run(
+            ["node", str(script_path)],
+            env={
+                **os.environ,
+                "FIREFOX_BIN": str(firefox_path),
+                "CHROME_BIN": str(chrome_path),
+            },
+            capture_output=True,
+            text=True,
+            timeout=script_timeout,
+        )
+    except subprocess.TimeoutExpired:
+        logger.warning("Confirmation script timed out after %ss", script_timeout)
+        return None
+
+    logger.info(
+        "Confirmation script exited %s\nstdout:\n%s\nstderr:\n%s",
+        proc.returncode,
+        proc.stdout,
+        proc.stderr,
+    )
+    if proc.returncode == 0:
+        return ReproductionResult(
+            reproduced=True,
+            failure_reason=None,
+            screenshot_path=None,
+        )
+    return None
+
+
 def make_empty_temp_file(dir: Path, prefix: str | None, suffix: str) -> Path:
     fd, path = tempfile.mkstemp(prefix=prefix, suffix=suffix, dir=dir)
     f = os.fdopen(fd)
@@ -322,9 +363,12 @@ def __init__(
     ):
         super().__init__(task_config, run_tracker)
         self.input_data = input_data
+        self.firefox_path = firefox_path
+        self.chrome_path = chrome_path
         self.screenshot_path = make_empty_temp_file(
             screenshot_dir, "reproduction=", ".png"
         )
+        self.script_path = make_empty_temp_file(REPRO_DIR, "reproduction=", ".mjs")
         self.add_mcp_server(
             "firefox-devtools",
             build_firefox_devtools_server(
@@ -349,6 +393,7 @@ def subject(self) -> Any:
         return self.input_data.subject()
 
     def system_prompt(self) -> str:
+        repro_reference = REPRO_DIR / "repro_reference.mjs"
         return (
             super()
             .system_prompt()
@@ -371,14 +416,26 @@ def system_prompt(self) -> str:
      web-compat issue; refine the steps and re-check before concluding.
    - If the reported broken behaviour reproduces in both browsers, it is not a
      Firefox web-compat issue: set `failure_reason` to `non_compat`.
-4. If the issue reproduces AND the breakage is visual in nature (incorrect
+4. Write and run a Puppeteer script that drives the real reported site in both
+   browsers and demonstrates the divergence: follow the spec in
+   `{repro_reference}`, write your script to exactly `{self.script_path}`, then run
+   it with
+
+   `FIREFOX_BIN={self.firefox_path} CHROME_BIN={self.chrome_path} node {self.script_path}`
+
+   On exit 0, set `script_path` in your result to that path. Otherwise revise and
+   re-run until the script both executes cleanly and demonstrates the difference.
+   If you're unable to do so, you have not confirmed the issue, so report
+   `reproduced` as false with `script_path` null and the `failure_reason` you
+   observed (likely `not_reproducable`).
+5. If the issue reproduces AND the breakage is visual in nature (incorrect
    layout or rendering, not broken interaction), capture a screenshot in Firefox
    showing it: call `screenshot_page` with `saveTo` set to
    `{self.screenshot_path}`. This writes the image to that file instead of
    returning it — do not capture or paste the image data yourself. Then set
    `screenshot_path` in your result to exactly `{self.screenshot_path}`. For
    non-visual issues, take no screenshot and leave `screenshot_path` null.
-5. Submit your findings via `submit_result` (see "Reporting your result").
+6. Submit your findings via `submit_result` (see "Reporting your result").
 """
             )
         )
@@ -513,6 +570,7 @@ class InitialReproduction:
     steps: str
     summary: str
     screenshot_path: Path | None
+    script_path: Path | None
     chrome_reproduced: bool | None
 
 
@@ -578,6 +636,20 @@ def screenshot_url(self) -> str | None:
 
         return None
 
+    @property
+    def script_url(self) -> str | None:
+        if (
+            self.initial_repro is not None
+            and self.initial_repro.script_path is not None
+        ):
+            return self.publish_file(
+                f"reproduction-{self.initial_repro.channel}.mjs",
+                self.initial_repro.script_path,
+                "text/javascript",
+            )
+
+        return None
+
     def set_result(
         self,
         channel: FirefoxChannel,
@@ -595,6 +667,7 @@ def set_result(
                 result.steps,
                 result.summary,
                 result.screenshot_path,
+                result.script_path,
                 result.chrome_reproduced,
             )
         elif isinstance(result, ChromeMaskResult):
@@ -611,9 +684,15 @@ def into_result(self) -> AutowebcompatReproResult:
             failure_reason=self.failure_reason,
             steps=self.steps,
             screenshot_url=self.screenshot_url,
+            script_url=self.script_url,
             plan_result=self.plan_result,
             reproductions=[
-                (key[0].value, value.model_copy(update={"screenshot_path": None}))
+                (
+                    key[0].value,
+                    value.model_copy(
+                        update={"screenshot_path": None, "script_path": None}
+                    ),
+                )
                 for key, value in self.results.items()
                 if isinstance(value, ReproductionResult)
             ],
@@ -659,7 +738,14 @@ async def next_repro_task(
     ) -> None:
         browser = getattr(firefox_browser, channel.value)
         profile = setup_profile(browser)
-        if repro_results.initial_repro is None:
+        logger.info(
+            "Trying reproduction in %s%s",
+            channel,
+            f" {extra}" if extra is not None else "",
+        )
+
+        initial_repro = repro_results.initial_repro
+        if initial_repro is None:
             task: Task = BugReproduction(
                 config,
                 tracker,
@@ -671,18 +757,25 @@ async def next_repro_task(
                 chrome_browser.stable,
             )
         else:
+            if initial_repro.script_path is not None:
+                script_result = run_confirmation_script(
+                    initial_repro.script_path, browser, chrome_browser.stable
+                )
+                if script_result is not None:
+                    repro_results.set_result(channel, extra, script_result)
+                    return
+                logger.info(
+                    "Confirmation script did not reach a verdict in %s; "
+                    "falling back to the reproduction steps",
+                    channel,
+                )
             task = StepsReproduction(
                 config,
                 tracker,
                 browser,
                 profile,
-                repro_results.initial_repro.steps,
+                initial_repro.steps,
             )
-        logger.info(
-            "Trying reproduction in %s%s",
-            channel,
-            f" {extra}" if extra is not None else "",
-        )
         repro_results.set_result(channel, extra, await task.run())
 
     screenshots_dir = Path(tempfile.mkdtemp(prefix="autowebcompat-screenshots-"))

agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/mcp_servers.py

@@ -10,10 +10,12 @@
 
 from claude_agent_sdk.types import McpStdioServerConfig
 
+REPRO_DIR = Path("/app/repro")
+
 
 def resolve_bin(bin_name: str) -> str:
     """Resolve an installed MCP server binary to an absolute path."""
-    binary = Path("/app/node") / "node_modules" / ".bin" / bin_name
+    binary = REPRO_DIR / "node_modules" / ".bin" / bin_name
     if not binary.exists():
         raise RuntimeError(
             f"MCP server binary not found at {binary}; the image should install "

agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/prompts/system.md

@@ -10,6 +10,9 @@ issues that reproduce in Firefox.
   section.
 - Your job is to analyze and, when instructed, reproduce the reported
   issue. Do not attempt to debug or perform root cause analysis.
+- No `Monitor` or `ScheduleWakeup` tools are available. If you attempt
+  to use these tools, nothing will notify you, and you will stall and 
+  lose your findings.
 
 **Stay focused on reproduction. Avoid:**
 

agents/autowebcompat-repro/hackbot_agents/autowebcompat_repro/result.py

@@ -7,7 +7,13 @@
 from typing import Generic, Literal, TypeVar
 
 from claude_agent_sdk import McpServerConfig, create_sdk_mcp_server, tool
-from pydantic import BaseModel, Field, ValidationError, field_validator
+from pydantic import (
+    BaseModel,
+    Field,
+    ValidationError,
+    field_validator,
+    model_validator,
+)
 
 RESULT_SERVER_NAME = "autowebcompat-repro"
 SUBMIT_RESULT_TOOL = f"mcp__{RESULT_SERVER_NAME}__submit_result"
@@ -148,6 +154,44 @@ class BugReproductionResult(ReproductionResult):
         ),
     )
 
+    script_path: Path | None = Field(
+        description=(
+            """The file path of the Puppeteer confirmation script you wrote and
+            ran successfully (exit code 0). Use the exact path you were given
+            to write to (do NOT paste the script source). Must be null when
+            `reproduced` is false."""
+        ),
+    )
+
+    @field_validator("script_path", mode="after")
+    @classmethod
+    def validate_script_path(cls, path: Path | None) -> Path | None:
+        if path is None:
+            return None
+
+        if not path.exists():
+            raise ValueError(f"Script path {path} doesn't exist")
+        if not path.read_text().strip():
+            raise ValueError(f"Script path {path} is empty")
+        return path
+
+    @model_validator(mode="after")
+    def validate_script_matches_reproduced(self) -> BugReproductionResult:
+        """A reproduction is only real once a script has confirmed it."""
+        if self.reproduced and self.script_path is None:
+            raise ValueError(
+                "reproduced is true but script_path is null: a reproduction "
+                "requires a Puppeteer script that is run in both Firefox and Chrome"
+                "Write and run one, then resubmit — or set reproduced to false "
+                "with the appropriate failure_reason if you cannot."
+            )
+        if not self.reproduced and self.script_path is not None:
+            raise ValueError(
+                "script_path must be null when reproduced is false; a script "
+                "that does not demonstrate the issue is not a confirmation."
+            )
+        return self
+
 
 class ChromeMaskResult(BaseModel):
     chrome_mask_fixed: bool | None = Field(