Add expected results for verifiable steps in the test-plan-generator agent

agents/test-plan-generator/hackbot_agents/test_plan_generator/prompts/system.md

@@ -12,6 +12,7 @@ report only pass/fail/unsuitable results. Do not try to fix, patch or make chang
    - A title.
    - A primary execution context label: `chrome` or `content`.
    - Ordered test steps.
+   - Step-indexed expectations for only the verification step(s).
 3. Run the generated cases and steps in order.
 4. Submit one final structured result with `submit_result`.
    - Use the provided feature name as the structured result feature.
@@ -47,6 +48,11 @@ bypass a failing content interaction.
   expected observable behavior. Keep exact inputs, actions, and detailed
   conditions in the test steps.
 - Write concise, ordered test steps.
+- Put only the action a QA engineer should perform in each step.
+- Put expected results in `step_expectations` only for the step or steps that
+  directly verify the observable behavior described by the test case title.
+- Do not add expectations for setup, navigation, or routine happy-path
+  confirmation steps, leave the corresponding `step_expectations` entries null.
 
 ## Unsuitable cases
 

agents/test-plan-generator/hackbot_agents/test_plan_generator/result.py

@@ -5,7 +5,12 @@
 from typing import Literal
 
 from claude_agent_sdk import McpServerConfig, create_sdk_mcp_server, tool
-from pydantic import BaseModel, Field, ValidationError, model_validator
+from pydantic import (
+    BaseModel,
+    Field,
+    ValidationError,
+    model_validator,
+)
 
 RESULT_SERVER_NAME = "test-plan-generator"
 SUBMIT_RESULT_TOOL = f"mcp__{RESULT_SERVER_NAME}__submit_result"
@@ -22,14 +27,24 @@ class GeneratedTestCase(BaseModel):
         )
     )
     preconditions: str | None = None
-    steps: list[str] = Field(
-        description="Concise test steps for this case; between 1 and 6 steps."
+    steps: list[str] = Field(description="Concise ordered test steps for this case.")
+    step_expectations: list[str | None] = Field(
+        description=(
+            "Expected results aligned by index with steps when present. Use null "
+            "for setup, navigation, or happy-path steps that do not directly "
+            "verify the case title."
+        )
     )
 
     @model_validator(mode="after")
     def _validate_steps(self) -> "GeneratedTestCase":
-        if not 1 <= len(self.steps) <= 6:
-            raise ValueError("each generated test case must have 1 to 6 steps")
+        if not self.steps:
+            raise ValueError("each generated test case must have at least one step")
+        if not any(self.step_expectations):
+            raise ValueError(
+                "each generated test case must include an expected result on at "
+                "least one verification step"
+            )
         return self
 
 

libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py

@@ -70,19 +70,35 @@ def _case_payload(
     case_type_id: int,
     template_id: int,
 ) -> dict[str, Any]:
-    steps = [str(step) for step in test_case.get("steps", [])]
     payload: dict[str, Any] = {
         "title": test_case["title"],
         "type_id": case_type_id,
         "template_id": template_id,
         "labels": [_CASE_LABEL],
-        "custom_steps_separated": [{"content": step, "expected": ""} for step in steps],
+        "custom_steps_separated": _separated_steps(test_case),
     }
     if test_case.get("preconditions"):
         payload["custom_preconds"] = test_case["preconditions"]
     return payload
 
 
+def _separated_steps(test_case: dict[str, Any]) -> list[dict[str, str]]:
+    steps = [str(step) for step in test_case.get("steps", [])]
+    expectations = list(test_case.get("step_expectations") or [])
+
+    separated_steps = []
+    for index, step in enumerate(steps):
+        expected = expectations[index] if index < len(expectations) else ""
+        separated_steps.append(
+            {
+                "content": step,
+                "expected": str(expected or ""),
+            }
+        )
+
+    return separated_steps
+
+
 class SubmitTestPlanHandler:
     async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult:
         feature = str(params.get("feature") or "").strip()

libs/hackbot-runtime/hackbot_runtime/actions/testrail.py

@@ -10,7 +10,13 @@
 from typing import Annotated, Any
 
 from agent_tools.registry import ToolError, tool, tools_in
-from pydantic import BaseModel, Field, ValidationError, field_validator
+from pydantic import (
+    BaseModel,
+    Field,
+    ValidationError,
+    field_validator,
+    model_validator,
+)
 
 from hackbot_runtime.actions.recorder import ActionsRecorder
 
@@ -29,6 +35,12 @@ class TestRailCaseInput(BaseModel):
     steps: list[str] = Field(
         min_length=1, description="Ordered steps a QA engineer should follow."
     )
+    step_expectations: list[str | None] = Field(
+        description=(
+            "Expected results aligned by index with steps when present. Use null "
+            "for steps that do not directly verify the case title."
+        ),
+    )
 
     @field_validator("title")
     @classmethod
@@ -46,6 +58,12 @@ def steps_must_not_be_blank(cls, value: list[str]) -> list[str]:
             raise ValueError("steps must not contain blank items")
         return steps
 
+    @model_validator(mode="after")
+    def expectations_must_include_verification(self) -> "TestRailCaseInput":
+        if not any(self.step_expectations):
+            raise ValueError("at least one step must include an expected result")
+        return self
+
 
 class SubmitTestPlanInput(BaseModel):
     feature: str = Field(description="Feature covered by the generated test cases.")

libs/hackbot-runtime/tests/test_testrail_action.py

@@ -14,6 +14,7 @@ def _cases():
             "context": "content",
             "preconditions": "A PDF is available.",
             "steps": ["Open the PDF"],
+            "step_expectations": ["The PDF is displayed."],
         }
     ]
 
@@ -47,5 +48,47 @@ async def test_submit_test_plan_tool_rejects_invalid_input():
     assert recorder.actions == []
 
 
+async def test_submit_test_plan_tool_rejects_cases_without_expectation():
+    recorder = ActionsRecorder()
+
+    with pytest.raises(ToolError) as exc:
+        await testrail.submit_test_plan(
+            recorder,
+            feature="Feature",
+            generated_test_cases=[
+                {
+                    "id": 1,
+                    "title": "Case",
+                    "steps": ["Open the PDF"],
+                    "step_expectations": [""],
+                }
+            ],
+        )
+
+    assert "invalid TestRail submission" in str(exc.value)
+    assert recorder.actions == []
+
+
+async def test_submit_test_plan_tool_preserves_blank_expectations():
+    recorder = ActionsRecorder()
+
+    await testrail.submit_test_plan(
+        recorder,
+        feature="Feature",
+        generated_test_cases=[
+            {
+                "id": 1,
+                "title": "Case",
+                "steps": ["Open the PDF", "Select text"],
+                "step_expectations": ["", "Text is selected."],
+            }
+        ],
+    )
+
+    assert recorder.actions[0]["params"]["generated_test_cases"][0][
+        "step_expectations"
+    ] == ["", "Text is selected."]
+
+
 def test_submit_test_plan_handler_is_registered():
     assert isinstance(get_handler(ACTION_TYPE), SubmitTestPlanHandler)

libs/hackbot-runtime/tests/test_testrail_handler.py

@@ -20,13 +20,20 @@ def _plan():
                 "context": "content",
                 "preconditions": "A PDF is available.",
                 "steps": ["Open the PDF", "Select some text"],
+                "step_expectations": [
+                    None,
+                    "Text selection is highlighted in the PDF.",
+                ],
             },
             {
                 "id": 2,
                 "title": "The toolbar remains available",
                 "context": "chrome",
                 "preconditions": None,
                 "steps": ["Open the toolbar"],
+                "step_expectations": [
+                    "The toolbar remains visible and usable.",
+                ],
             },
         ],
         "results": [
@@ -143,11 +150,39 @@ async def test_submit_test_plan_creates_suite_section_and_cases(monkeypatch):
         "custom_preconds": "A PDF is available.",
         "custom_steps_separated": [
             {"content": "Open the PDF", "expected": ""},
-            {"content": "Select some text", "expected": ""},
+            {
+                "content": "Select some text",
+                "expected": "Text selection is highlighted in the PDF.",
+            },
         ],
     }
 
 
+def test_separated_steps_maps_expectations_by_step_index():
+    assert testrail_handler._separated_steps(
+        {
+            "steps": ["Open the PDF", "Select text", "Copy text"],
+            "step_expectations": [None, "", "Text is copied."],
+        }
+    ) == [
+        {"content": "Open the PDF", "expected": ""},
+        {"content": "Select text", "expected": ""},
+        {"content": "Copy text", "expected": "Text is copied."},
+    ]
+
+
+def test_separated_steps_allows_fewer_expectations_than_steps():
+    assert testrail_handler._separated_steps(
+        {
+            "steps": ["Open the PDF", "Select text"],
+            "step_expectations": ["The PDF is displayed."],
+        }
+    ) == [
+        {"content": "Open the PDF", "expected": "The PDF is displayed."},
+        {"content": "Select text", "expected": ""},
+    ]
+
+
 async def test_submit_test_plan_reports_api_failure(monkeypatch):
     client = _FakeClient(responses=[RuntimeError("TestRail is unavailable")])
     monkeypatch.setattr(testrail_handler, "_client", lambda: client)