ci: verify toolkit docs before publishing

Run the generator's isolated type-check and test suite before publishing, and support manual verification branches without production review noise.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: jottakka

.github/workflows/generate-toolkit-docs.md

@@ -4,8 +4,8 @@ This workflow regenerates toolkit JSON and opens a PR with the changes. It can b
 
 ## What it does
 
-1. Builds the toolkit docs generator.
-2. Generates toolkit JSON in `toolkit-docs-generator/data/toolkits` using the Engine tool metadata and summary endpoints.
+1. Type-checks and tests the toolkit docs generator.
+2. Generates toolkit JSON in `toolkit-docs-generator/data/toolkits` using the Engine tool metadata endpoint.
 3. Syncs integrations sidebar navigation from the generated JSON.
 4. Creates or updates a PR on the stable `automation/toolkit-docs` branch if any files changed. Later runs overwrite that open PR with the latest generated docs.
 

.github/workflows/generate-toolkit-docs.yml

@@ -1,7 +1,7 @@
 name: Generate toolkit docs
 # Description: Generate toolkit JSON from Engine API, then sync sidebar navigation.
 # Flow:
-# 1) Build the toolkit docs generator
+# 1) Type-check and test the toolkit docs generator
 # 2) Generate toolkit JSON into toolkit-docs-generator/data/toolkits
 # 3) Sync integrations sidebar _meta.tsx from toolkit-docs-generator/data/toolkits
 # 4) Create or update a PR if changes were produced
@@ -10,6 +10,11 @@ on:
   repository_dispatch:
     types: [porter_deploy_succeeded]
   workflow_dispatch:
+    inputs:
+      pr_branch:
+        description: "Optional isolated branch for a verification PR"
+        required: false
+        type: string
   # 11:00 UTC = 3 AM PST / 4 AM PDT — late enough that DST drift doesn't matter.
   schedule:
     - cron: "0 11 * * *"
@@ -48,9 +53,8 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: Build toolkit docs generator
-        run: pnpm build
-        working-directory: toolkit-docs-generator
+      - name: Validate toolkit docs generator
+        run: pnpm run toolkit-docs:check
 
       - name: Generate toolkit docs
         run: |
@@ -96,22 +100,24 @@ jobs:
         with:
           token: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }}
           commit-message: "[AUTO] Adding MCP Servers docs update"
-          title: "[AUTO] Adding MCP Servers docs update"
+          title: ${{ inputs.pr_branch && '[TEST] Generated toolkit docs verification' || '[AUTO] Adding MCP Servers docs update' }}
           body: |
-            This PR was generated after a Porter deploy succeeded.
+            This PR was generated by the toolkit docs workflow.
 
             - Trigger: ${{ github.event_name }}
+            - Source ref: ${{ github.ref_name }}
             - Deploy env: ${{ github.event.client_payload.env || 'unknown' }}
             - Deploy SHA: ${{ github.event.client_payload.deploy_sha || 'unknown' }}
             - Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
           # Stable branch so later runs update the open auto-PR instead of
           # opening a new one each time. create-pull-request force-pushes the
           # latest generated docs onto this branch.
-          branch: automation/toolkit-docs
+          base: ${{ inputs.pr_branch && github.ref_name || github.event.repository.default_branch || 'main' }}
+          branch: ${{ inputs.pr_branch || 'automation/toolkit-docs' }}
           delete-branch: true
 
       - name: Request team review
-        if: steps.cpr.outputs.pull-request-number != ''
+        if: steps.cpr.outputs.pull-request-number != '' && (github.event_name != 'workflow_dispatch' || inputs.pr_branch == '')
         continue-on-error: true
         run: gh pr edit ${{ steps.cpr.outputs.pull-request-number }} --add-reviewer ArcadeAI/engineering-tools-and-dx
         env:

package.json

@@ -10,6 +10,9 @@
     "lint": "pnpm exec ultracite check",
     "format": "pnpm exec ultracite fix",
     "prepare": "husky install",
+    "toolkit-docs:typecheck": "tsc --noEmit --project toolkit-docs-generator/tsconfig.json",
+    "toolkit-docs:test": "vitest run --root toolkit-docs-generator --config vitest.config.ts",
+    "toolkit-docs:check": "pnpm run toolkit-docs:typecheck && pnpm run toolkit-docs:test",
     "translate": "pnpm dlx tsx scripts/i18n-sync/index.ts && pnpm format",
     "llmstxt": "pnpm dlx tsx scripts/generate-llmstxt.ts",
     "test": "vitest --run",

toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts

@@ -20,6 +20,9 @@ test("porter workflow includes required triggers", () => {
 });
 
 test("porter workflow generates docs and opens a PR", () => {
+  expect(workflowContents).toContain("pnpm run toolkit-docs:check");
+  expect(workflowContents).not.toContain("run: pnpm build");
+  expect(workflowContents).not.toContain("name: Build toolkit docs generator");
   expect(workflowContents).toContain("pnpm dlx tsx src/cli/index.ts generate");
   expect(workflowContents).toContain("--skip-unchanged");
   expect(workflowContents).toContain("--require-complete");
@@ -60,3 +63,13 @@ test("workflow dispatch keeps default full-run behavior", () => {
   expect(workflowContents).not.toContain("inputs.providers");
   expect(workflowContents).not.toContain("PROVIDERS_INPUT=");
 });
+
+test("workflow dispatch can create an isolated verification PR", () => {
+  expect(workflowContents).toContain("pr_branch:");
+  expect(workflowContents).toContain(
+    "inputs.pr_branch || 'automation/toolkit-docs'"
+  );
+  expect(workflowContents).toContain(
+    "inputs.pr_branch && github.ref_name || github.event.repository.default_branch || 'main'"
+  );
+});