feat(runner): add task and persona selection

Author: wu-uk

docs/evaluation.md

@@ -73,6 +73,32 @@ docker compose -f docker/docker-compose.yaml run --rm workspace-bench \
   --dataset full
 ```
 
+### Running Selected Tasks
+
+Use `--task-ids` to run an exact task subset. IDs may be separated by spaces or commas, and they run in the specified order:
+
+```bash
+docker compose -f docker/docker-compose.yaml run --rm workspace-bench \
+  bash /workspace/Workspace-Bench/evaluation/docker/run-benchmark.sh \
+  --harness codex \
+  --model kimi-k2.5 \
+  --dataset lite \
+  --task-ids 45 55 386
+```
+
+Use `--persona` to run every task whose metadata `persona` exactly matches the supplied value:
+
+```bash
+docker compose -f docker/docker-compose.yaml run --rm workspace-bench \
+  bash /workspace/Workspace-Bench/evaluation/docker/run-benchmark.sh \
+  --harness codex \
+  --model kimi-k2.5 \
+  --dataset lite \
+  --persona "Product Manager"
+```
+
+`--task-ids`, `--persona`, and `--task-limit` are mutually exclusive. Missing task IDs, duplicate task IDs, and unknown personas stop the run instead of silently changing the sample. The generated run name includes the selection, so a subset does not reuse the default Lite or Full output directory. Task selection changes which tasks execute; it does not shrink the standard profile workspace prepared from the dataset.
+
 ### Using Different Harnesses
 
 ```bash

evaluation/scripts/build_run_config.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import argparse
+import hashlib
 import json
 import re
 from pathlib import Path
@@ -37,6 +38,34 @@ def _display_slug(value: str) -> str:
     return re.sub(r"[^A-Za-z0-9._-]+", "_", value.strip()) or "Custom"
 
 
+def _normalize_task_ids(values: list[str] | None) -> list[str]:
+    task_ids: list[str] = []
+    for value in values or []:
+        task_ids.extend(part.strip() for part in str(value).split(",") if part.strip())
+    if not task_ids:
+        return []
+    invalid = [task_id for task_id in task_ids if not re.fullmatch(r"[A-Za-z0-9._-]+", task_id)]
+    if invalid:
+        raise SystemExit(f"invalid task id(s): {', '.join(invalid)}")
+    duplicates = sorted({task_id for task_id in task_ids if task_ids.count(task_id) > 1})
+    if duplicates:
+        raise SystemExit(f"duplicate task id(s): {', '.join(duplicates)}")
+    return task_ids
+
+
+def _selection_suffix(*, task_ids: list[str], persona: str | None) -> tuple[str, str]:
+    if task_ids:
+        joined = "-".join(task_ids)
+        if len(task_ids) <= 3 and len(joined) <= 48:
+            return f"tasks-{_safe_slug(joined)}", f"Tasks-{_display_slug(joined)}"
+        digest = hashlib.sha256("\0".join(task_ids).encode("utf-8")).hexdigest()[:10]
+        return f"tasks-{len(task_ids)}-{digest}", f"Tasks-{len(task_ids)}-{digest}"
+    if persona:
+        slug = _safe_slug(persona)[:60]
+        return f"persona-{slug}", f"Persona-{_display_slug(persona)[:60]}"
+    return "", ""
+
+
 def _normalize_harness(value: str) -> str:
     mapping = {
         "codex": "Codex",
@@ -102,10 +131,23 @@ def build_config(args: argparse.Namespace) -> Path:
     if dataset not in {"smoke", "lite", "full"}:
         raise SystemExit(f"unsupported dataset: {args.dataset}")
 
-    run_name = args.run_name or {"smoke": "Smoke", "lite": "Lite", "full": "Full"}[dataset]
+    task_ids = _normalize_task_ids(getattr(args, "task_ids", None))
+    persona_value = getattr(args, "persona", None)
+    persona = str(persona_value).strip() if persona_value is not None else None
+    if persona_value is not None and not persona:
+        raise SystemExit("--persona must not be empty")
+    task_limit = getattr(args, "task_limit", None)
+    selected = sum([task_limit is not None, bool(task_ids), persona is not None])
+    if selected > 1:
+        raise SystemExit("--task-limit, --task-ids, and --persona are mutually exclusive")
+
+    selection_slug, selection_name = _selection_suffix(task_ids=task_ids, persona=persona)
+    default_run_name = {"smoke": "Smoke", "lite": "Lite", "full": "Full"}[dataset]
+    run_name = args.run_name or (
+        f"{default_run_name}-{selection_name}" if selection_name else default_run_name
+    )
     task_path = eval_root / ("tasks" if dataset == "full" else "tasks_lite")
-    task_limit = args.task_limit
-    if task_limit is None and dataset == "smoke":
+    if task_limit is None and not task_ids and persona is None and dataset == "smoke":
         task_limit = 1
     task_parallel = not bool(args.no_task_parallel)
     task_parallel_workers = max(1, int(args.task_parallel_workers or 10))
@@ -117,6 +159,10 @@ def build_config(args: argparse.Namespace) -> Path:
     fs_map_dir.mkdir(parents=True, exist_ok=True)
 
     config_slug = f"{harness.lower()}-{_safe_slug(args.model)}-{dataset}"
+    if selection_slug:
+        config_slug = f"{config_slug}-{selection_slug}"
+    elif args.run_name:
+        config_slug = f"{config_slug}-{_safe_slug(args.run_name)}"
     fs_map_path = fs_map_dir / f"fs_map_{harness}_{_display_slug(model_name)}.json"
     fs_map_path.write_text(
         json.dumps(_fs_map(eval_root, harness, model_name), ensure_ascii=False, indent=2) + "\n",
@@ -146,6 +192,10 @@ def build_config(args: argparse.Namespace) -> Path:
     }
     if task_limit is not None:
         config["task_limit"] = int(task_limit)
+    elif task_ids:
+        config["task_ids"] = task_ids
+    elif persona is not None:
+        config["persona"] = persona
 
     config_path = runs_dir / f"{config_slug}.yaml"
     config_path.write_text(yaml.safe_dump(config, allow_unicode=True, sort_keys=False), encoding="utf-8")
@@ -163,7 +213,14 @@ def main() -> None:
     parser.add_argument("--model-name", help="Display name used in output directory")
     parser.add_argument("--env-prefix", help="Environment variable prefix for BASE_URL/API_KEY")
     parser.add_argument("--run-name", help="Output run name; defaults to Smoke/Lite/Full")
-    parser.add_argument("--task-limit", type=int)
+    selection = parser.add_mutually_exclusive_group()
+    selection.add_argument("--task-limit", type=int, help="Run the first N tasks in deterministic order")
+    selection.add_argument(
+        "--task-ids",
+        nargs="+",
+        help="Run exact task IDs; accepts spaces or comma-separated values",
+    )
+    selection.add_argument("--persona", help="Run every task whose metadata persona exactly matches this value")
     parser.add_argument("--timeout-sec", type=float, default=2000.0)
     parser.add_argument("--task-parallel-workers", type=int, help="Number of isolated task-level workers; defaults to 10")
     parser.add_argument("--no-task-parallel", action="store_true", help="Disable isolated task-level parallelism")

evaluation/src/agent_runner.py

@@ -331,9 +331,46 @@ def _iter_metadata_paths(root: str, *, limit: Optional[int] = None) -> List[str]
     return out
 
 
-def _load_metadatas(tasks_root: str, *, limit: Optional[int]) -> List[Dict[str, Json]]:
+def _metadata_task_id(meta: Dict[str, Json], metadata_path: str) -> str:
+    value = meta.get("id")
+    if value in (None, ""):
+        value = meta.get("absolute_id")
+    if value in (None, ""):
+        value = os.path.basename(os.path.dirname(metadata_path))
+    return str(value).strip()
+
+
+def _normalize_config_task_ids(value: Json) -> List[str]:
+    if value is None:
+        return []
+    if not isinstance(value, list):
+        raise ValueError("task_ids must be a list")
+    task_ids = [str(item).strip() for item in value]
+    if any(not task_id for task_id in task_ids):
+        raise ValueError("task_ids must not contain empty values")
+    duplicates = sorted({task_id for task_id in task_ids if task_ids.count(task_id) > 1})
+    if duplicates:
+        raise ValueError(f"duplicate task id(s): {', '.join(duplicates)}")
+    return task_ids
+
+
+def _load_metadatas(
+    tasks_root: str,
+    *,
+    limit: Optional[int] = None,
+    task_ids: Json = None,
+    persona: Json = None,
+) -> List[Dict[str, Json]]:
+    requested_ids = _normalize_config_task_ids(task_ids)
+    persona_value = None if persona is None else str(persona).strip()
+    if persona is not None and not persona_value:
+        raise ValueError("persona must not be empty")
+    selected = sum([limit is not None, bool(requested_ids), persona_value is not None])
+    if selected > 1:
+        raise ValueError("task_limit, task_ids, and persona are mutually exclusive")
+
     metas: List[Dict[str, Json]] = []
-    for mp in _iter_metadata_paths(tasks_root, limit=limit):
+    for mp in _iter_metadata_paths(tasks_root):
         try:
             meta = _read_json(mp)
         except Exception:
@@ -343,6 +380,35 @@ def _load_metadatas(tasks_root: str, *, limit: Optional[int]) -> List[Dict[str,
         m = dict(meta)
         m["__metadata_path"] = mp
         metas.append(m)
+
+    if requested_ids:
+        by_id: Dict[str, Dict[str, Json]] = {}
+        duplicate_metadata_ids: List[str] = []
+        for meta in metas:
+            metadata_path = str(meta["__metadata_path"])
+            task_id = _metadata_task_id(meta, metadata_path)
+            if task_id in by_id:
+                duplicate_metadata_ids.append(task_id)
+            else:
+                by_id[task_id] = meta
+        if duplicate_metadata_ids:
+            duplicate_text = ", ".join(sorted(set(duplicate_metadata_ids)))
+            raise ValueError(f"duplicate task id(s) in dataset: {duplicate_text}")
+        missing = [task_id for task_id in requested_ids if task_id not in by_id]
+        if missing:
+            raise ValueError(f"task id(s) not found: {', '.join(missing)}")
+        return [by_id[task_id] for task_id in requested_ids]
+
+    if persona_value is not None:
+        matched = [meta for meta in metas if str(meta.get("persona") or "").strip() == persona_value]
+        if not matched:
+            available = sorted({str(meta.get("persona") or "").strip() for meta in metas} - {""})
+            suffix = f"; available personas: {', '.join(available)}" if available else ""
+            raise ValueError(f"persona not found: {persona_value}{suffix}")
+        return matched
+
+    if limit is not None:
+        return metas[: max(0, int(limit))]
     return metas
 
 
@@ -1417,6 +1483,8 @@ def main() -> None:
     prompt_head_by_language = _language_text_map(cfg.get("prompt_head_by_language"))
     prompt_tail_by_language = _language_text_map(cfg.get("prompt_tail_by_language"))
     task_limit = cfg.get("task_limit")
+    task_ids = cfg.get("task_ids")
+    persona = cfg.get("persona")
     timeout_sec = float(cfg.get("timeout_sec") or 300.0)
     api_provider = cfg.get("api_provider") if isinstance(cfg.get("api_provider"), dict) else {}
 
@@ -1442,7 +1510,15 @@ def main() -> None:
     raw_work_dir_map = fs_map_all.get("raw_work_dir", {})
     standard_work_dir_map = fs_map_all.get("standard_work_dir", {})
 
-    metas = _load_metadatas(task_path, limit=int(task_limit) if task_limit is not None else None)
+    try:
+        metas = _load_metadatas(
+            task_path,
+            limit=int(task_limit) if task_limit is not None else None,
+            task_ids=task_ids,
+            persona=persona,
+        )
+    except (TypeError, ValueError) as e:
+        raise SystemExit(f"invalid task selection: {e}") from e
     if not metas:
         raise SystemExit("no tasks found")