diff --git a/agents/test-plan-generator/hackbot_agents/test_plan_generator/prompts/system.md b/agents/test-plan-generator/hackbot_agents/test_plan_generator/prompts/system.md index 427bbb276b..fe70a956cb 100644 --- a/agents/test-plan-generator/hackbot_agents/test_plan_generator/prompts/system.md +++ b/agents/test-plan-generator/hackbot_agents/test_plan_generator/prompts/system.md @@ -12,6 +12,7 @@ report only pass/fail/unsuitable results. Do not try to fix, patch or make chang - A title. - A primary execution context label: `chrome` or `content`. - Ordered test steps. + - Step-indexed expectations for only the verification step(s). 3. Run the generated cases and steps in order. 4. Submit one final structured result with `submit_result`. - Use the provided feature name as the structured result feature. @@ -47,6 +48,11 @@ bypass a failing content interaction. expected observable behavior. Keep exact inputs, actions, and detailed conditions in the test steps. - Write concise, ordered test steps. +- Put only the action a QA engineer should perform in each step. +- Put expected results in `step_expectations` only for the step or steps that + directly verify the observable behavior described by the test case title. +- Do not add expectations for setup, navigation, or routine happy-path + confirmation steps, leave the corresponding `step_expectations` entries null. ## Unsuitable cases diff --git a/agents/test-plan-generator/hackbot_agents/test_plan_generator/result.py b/agents/test-plan-generator/hackbot_agents/test_plan_generator/result.py index c48d0384f3..78b3cf6695 100644 --- a/agents/test-plan-generator/hackbot_agents/test_plan_generator/result.py +++ b/agents/test-plan-generator/hackbot_agents/test_plan_generator/result.py @@ -5,7 +5,12 @@ from typing import Literal from claude_agent_sdk import McpServerConfig, create_sdk_mcp_server, tool -from pydantic import BaseModel, Field, ValidationError, model_validator +from pydantic import ( + BaseModel, + Field, + ValidationError, + model_validator, +) RESULT_SERVER_NAME = "test-plan-generator" SUBMIT_RESULT_TOOL = f"mcp__{RESULT_SERVER_NAME}__submit_result" @@ -22,14 +27,24 @@ class GeneratedTestCase(BaseModel): ) ) preconditions: str | None = None - steps: list[str] = Field( - description="Concise test steps for this case; between 1 and 6 steps." + steps: list[str] = Field(description="Concise ordered test steps for this case.") + step_expectations: list[str | None] = Field( + description=( + "Expected results aligned by index with steps when present. Use null " + "for setup, navigation, or happy-path steps that do not directly " + "verify the case title." + ) ) @model_validator(mode="after") def _validate_steps(self) -> "GeneratedTestCase": - if not 1 <= len(self.steps) <= 6: - raise ValueError("each generated test case must have 1 to 6 steps") + if not self.steps: + raise ValueError("each generated test case must have at least one step") + if not any(self.step_expectations): + raise ValueError( + "each generated test case must include an expected result on at " + "least one verification step" + ) return self diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py index bdddf00382..45d63327a1 100644 --- a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py +++ b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py @@ -70,19 +70,35 @@ def _case_payload( case_type_id: int, template_id: int, ) -> dict[str, Any]: - steps = [str(step) for step in test_case.get("steps", [])] payload: dict[str, Any] = { "title": test_case["title"], "type_id": case_type_id, "template_id": template_id, "labels": [_CASE_LABEL], - "custom_steps_separated": [{"content": step, "expected": ""} for step in steps], + "custom_steps_separated": _separated_steps(test_case), } if test_case.get("preconditions"): payload["custom_preconds"] = test_case["preconditions"] return payload +def _separated_steps(test_case: dict[str, Any]) -> list[dict[str, str]]: + steps = [str(step) for step in test_case.get("steps", [])] + expectations = list(test_case.get("step_expectations") or []) + + separated_steps = [] + for index, step in enumerate(steps): + expected = expectations[index] if index < len(expectations) else "" + separated_steps.append( + { + "content": step, + "expected": str(expected or ""), + } + ) + + return separated_steps + + class SubmitTestPlanHandler: async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult: feature = str(params.get("feature") or "").strip() diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/testrail.py b/libs/hackbot-runtime/hackbot_runtime/actions/testrail.py index 0941b1cc04..c546809e71 100644 --- a/libs/hackbot-runtime/hackbot_runtime/actions/testrail.py +++ b/libs/hackbot-runtime/hackbot_runtime/actions/testrail.py @@ -10,7 +10,13 @@ from typing import Annotated, Any from agent_tools.registry import ToolError, tool, tools_in -from pydantic import BaseModel, Field, ValidationError, field_validator +from pydantic import ( + BaseModel, + Field, + ValidationError, + field_validator, + model_validator, +) from hackbot_runtime.actions.recorder import ActionsRecorder @@ -29,6 +35,12 @@ class TestRailCaseInput(BaseModel): steps: list[str] = Field( min_length=1, description="Ordered steps a QA engineer should follow." ) + step_expectations: list[str | None] = Field( + description=( + "Expected results aligned by index with steps when present. Use null " + "for steps that do not directly verify the case title." + ), + ) @field_validator("title") @classmethod @@ -46,6 +58,12 @@ def steps_must_not_be_blank(cls, value: list[str]) -> list[str]: raise ValueError("steps must not contain blank items") return steps + @model_validator(mode="after") + def expectations_must_include_verification(self) -> "TestRailCaseInput": + if not any(self.step_expectations): + raise ValueError("at least one step must include an expected result") + return self + class SubmitTestPlanInput(BaseModel): feature: str = Field(description="Feature covered by the generated test cases.") diff --git a/libs/hackbot-runtime/tests/test_testrail_action.py b/libs/hackbot-runtime/tests/test_testrail_action.py index d059911863..ac1b1302de 100644 --- a/libs/hackbot-runtime/tests/test_testrail_action.py +++ b/libs/hackbot-runtime/tests/test_testrail_action.py @@ -14,6 +14,7 @@ def _cases(): "context": "content", "preconditions": "A PDF is available.", "steps": ["Open the PDF"], + "step_expectations": ["The PDF is displayed."], } ] @@ -47,5 +48,47 @@ async def test_submit_test_plan_tool_rejects_invalid_input(): assert recorder.actions == [] +async def test_submit_test_plan_tool_rejects_cases_without_expectation(): + recorder = ActionsRecorder() + + with pytest.raises(ToolError) as exc: + await testrail.submit_test_plan( + recorder, + feature="Feature", + generated_test_cases=[ + { + "id": 1, + "title": "Case", + "steps": ["Open the PDF"], + "step_expectations": [""], + } + ], + ) + + assert "invalid TestRail submission" in str(exc.value) + assert recorder.actions == [] + + +async def test_submit_test_plan_tool_preserves_blank_expectations(): + recorder = ActionsRecorder() + + await testrail.submit_test_plan( + recorder, + feature="Feature", + generated_test_cases=[ + { + "id": 1, + "title": "Case", + "steps": ["Open the PDF", "Select text"], + "step_expectations": ["", "Text is selected."], + } + ], + ) + + assert recorder.actions[0]["params"]["generated_test_cases"][0][ + "step_expectations" + ] == ["", "Text is selected."] + + def test_submit_test_plan_handler_is_registered(): assert isinstance(get_handler(ACTION_TYPE), SubmitTestPlanHandler) diff --git a/libs/hackbot-runtime/tests/test_testrail_handler.py b/libs/hackbot-runtime/tests/test_testrail_handler.py index fc2599e3ce..242a13daa8 100644 --- a/libs/hackbot-runtime/tests/test_testrail_handler.py +++ b/libs/hackbot-runtime/tests/test_testrail_handler.py @@ -20,6 +20,10 @@ def _plan(): "context": "content", "preconditions": "A PDF is available.", "steps": ["Open the PDF", "Select some text"], + "step_expectations": [ + None, + "Text selection is highlighted in the PDF.", + ], }, { "id": 2, @@ -27,6 +31,9 @@ def _plan(): "context": "chrome", "preconditions": None, "steps": ["Open the toolbar"], + "step_expectations": [ + "The toolbar remains visible and usable.", + ], }, ], "results": [ @@ -143,11 +150,39 @@ async def test_submit_test_plan_creates_suite_section_and_cases(monkeypatch): "custom_preconds": "A PDF is available.", "custom_steps_separated": [ {"content": "Open the PDF", "expected": ""}, - {"content": "Select some text", "expected": ""}, + { + "content": "Select some text", + "expected": "Text selection is highlighted in the PDF.", + }, ], } +def test_separated_steps_maps_expectations_by_step_index(): + assert testrail_handler._separated_steps( + { + "steps": ["Open the PDF", "Select text", "Copy text"], + "step_expectations": [None, "", "Text is copied."], + } + ) == [ + {"content": "Open the PDF", "expected": ""}, + {"content": "Select text", "expected": ""}, + {"content": "Copy text", "expected": "Text is copied."}, + ] + + +def test_separated_steps_allows_fewer_expectations_than_steps(): + assert testrail_handler._separated_steps( + { + "steps": ["Open the PDF", "Select text"], + "step_expectations": ["The PDF is displayed."], + } + ) == [ + {"content": "Open the PDF", "expected": "The PDF is displayed."}, + {"content": "Select text", "expected": ""}, + ] + + async def test_submit_test_plan_reports_api_failure(monkeypatch): client = _FakeClient(responses=[RuntimeError("TestRail is unavailable")]) monkeypatch.setattr(testrail_handler, "_client", lambda: client)