diff --git a/scripts_bazel/BUILD b/scripts_bazel/BUILD index e2d0402d2..4e5090c30 100644 --- a/scripts_bazel/BUILD +++ b/scripts_bazel/BUILD @@ -45,3 +45,11 @@ py_binary( visibility = ["//visibility:public"], deps = [], ) + +py_binary( + name = "needs_to_lobster", + srcs = ["needs_to_lobster.py"], + main = "needs_to_lobster.py", + visibility = ["//visibility:public"], + deps = [], +) diff --git a/scripts_bazel/README.md b/scripts_bazel/README.md index 7728a1736..f7296c36c 100644 --- a/scripts_bazel/README.md +++ b/scripts_bazel/README.md @@ -1,3 +1,46 @@ # Scripts Bazel This folder contains executables to be used within Bazel rules. + +## `needs_to_lobster` + +Converts a sphinx-needs `needs.json` into a LOBSTER `lobster-req-trace` +(`.lobster`) file so sphinx-needs requirements can be aggregated with the +code/test `.lobster` artifacts by the LOBSTER CLI (`lobster-report` / +`lobster-ci-report`). Authoring (RST) and rendering (sphinx-needs dashboards) +are untouched; LOBSTER only sits downstream of `needs.json`. + +```console +bazel run //scripts_bazel:needs_to_lobster -- \ + --needs-json _build/needs.json \ + --output tool_reqs.lobster \ + --types tool_req \ + --up-links satisfies +``` + +Under `bazel run` the process working directory is the runfiles tree, so +relative `--needs-json` / `--output` paths are resolved against +`$BUILD_WORKING_DIRECTORY` (the directory you invoked `bazel run` from), not the +sandbox. Absolute paths are always used as-is. + +The `satisfies` links (e.g. `tool_req__* -> gd_req__*`) are emitted as LOBSTER +`refs`, so the process-requirement <-> tool-requirement relationship authored in +RST is preserved as a LOBSTER up-trace. Model the two sides as separate +requirement levels in a LOBSTER tracing policy (`Tool Requirements` `trace to:` +`Process Requirements`). A ready-to-run example policy is provided in +[`lobster.conf.example`](./lobster.conf.example): + +```console +python3 scripts_bazel/needs_to_lobster.py --needs-json _build/needs.json --types tool_req --output tool_reqs.lobster +python3 scripts_bazel/needs_to_lobster.py --needs-json _build/needs.json --types gd_req --output process_reqs.lobster +lobster-report --lobster-config=scripts_bazel/lobster.conf.example --out=report.json +lobster-ci-report report.json +``` + +The tracing policy is intentionally *not* owned by this converter; long term it +should live centrally (e.g. next to the `rules_score` `*.conf.tpl` templates), +so it is not duplicated per module. + +Licensing: LOBSTER is AGPLv3. This tool never imports LOBSTER; it writes the +documented JSON schema and is consumed by the LOBSTER CLI as an external +process, keeping the Apache-2.0 code at arm's length. diff --git a/scripts_bazel/lobster.conf.example b/scripts_bazel/lobster.conf.example new file mode 100644 index 000000000..dc5f1974c --- /dev/null +++ b/scripts_bazel/lobster.conf.example @@ -0,0 +1,58 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# Example LOBSTER tracing policy for sphinx-needs requirements. +# +# This is the "tracing policy" the LOBSTER CLI enforces: it declares the +# requirement/implementation/activity levels and which level must `trace to:` +# which. It is plain config -- it does NOT import LOBSTER (AGPLv3); the LOBSTER +# CLI reads it as an external process. +# +# Produce the .lobster inputs from needs.json with needs_to_lobster.py, e.g.: +# +# python3 scripts_bazel/needs_to_lobster.py \ +# --needs-json _build/needs.json --types tool_req --output tool_reqs.lobster +# python3 scripts_bazel/needs_to_lobster.py \ +# --needs-json _build/needs.json --types gd_req --output process_reqs.lobster +# +# then: +# +# lobster-report --lobster-config=scripts_bazel/lobster.conf.example --out=report.json +# lobster-ci-report report.json +# +# This example models the process-requirement <-> tool-requirement relationship +# that is authored in RST via `:satisfies:` links (tool_req__* -> gd_req__*). + +requirements "Process Requirements" { + source: "process_reqs.lobster"; +} + +requirements "Tool Requirements" { + source: "tool_reqs.lobster"; + trace to: "Process Requirements"; +} + +# --- Extension points ------------------------------------------------------- +# To grow this into full requirement -> code -> test coverage, add the code and +# test .lobster files (e.g. produced by rules_score's lobster-cpp / lobster-python +# / gtest_report) as extra levels and trace them to the requirement levels: +# +# implementation "Code" { +# source: "code.lobster"; +# trace to: "Tool Requirements"; +# } +# +# activity "Tests" { +# source: "tests.lobster"; +# trace to: "Tool Requirements"; +# } diff --git a/scripts_bazel/needs_to_lobster.py b/scripts_bazel/needs_to_lobster.py new file mode 100644 index 000000000..c39ac32f1 --- /dev/null +++ b/scripts_bazel/needs_to_lobster.py @@ -0,0 +1,272 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Convert a sphinx-needs ``needs.json`` into a LOBSTER ``lobster-req-trace`` file. + +Data flow:: + + docs build -> sphinx-needs writes needs.json (single source of truth) + this tool -> needs.json -> .lobster (lobster-req-trace, v3) + LOBSTER CLI -> lobster-report / lobster-ci-report over all *.lobster + +The converter reads the already metamodel-validated ``needs.json`` and emits the +LOBSTER common interchange format so ``lobster-report`` can aggregate the +sphinx-needs requirements with the code/test ``.lobster`` artifacts produced by +``rules_score``. Authoring (RST) and rendering (sphinx-needs dashboards) are +untouched -- LOBSTER only sits downstream of ``needs.json``. + +Licensing note: LOBSTER is AGPLv3. This converter deliberately produces the +plain ``.lobster`` JSON via the documented schema and never imports the LOBSTER +Python packages, keeping the Apache-2.0 code at arm's length from the AGPL tool. +The emitted files are consumed by the LOBSTER CLI as an external process. + +Schema reference: LOBSTER ``lobster-req-trace``, item version 3 +(``tag``, ``location``, ``name``, ``refs``, ``just_up``/``just_down``/ +``just_global``, plus requirement ``framework``/``kind``/``text``/``status``). +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +# LOBSTER lobster-req-trace item schema version we emit (see documentation/schemas.md). +_LOBSTER_SCHEMA = "lobster-req-trace" +_LOBSTER_VERSION = 3 +_GENERATOR = "needs_to_lobster" + +# sphinx-needs link fields treated as up-traces (child -> parent) by default. +# ``satisfies`` is how tool_req__* needs reference their process gd_req__* parents. +_DEFAULT_UP_LINKS = ("satisfies",) + +# Need fields that may already carry LOBSTER-style justifications, if authored. +_JUST_FIELDS = ("just_up", "just_down", "just_global") + + +def _select_version(needs_data: dict[str, Any], version: str | None) -> dict[str, Any]: + """Return the ``versions`` entry to convert. + + Uses ``version`` when given, else ``current_version``, else the last key. + """ + versions = needs_data.get("versions") + if not versions: + raise ValueError("needs.json contains no 'versions' object") + if version is not None: + if version not in versions: + raise ValueError(f"version {version!r} not found in needs.json") + return versions[version] + current = needs_data.get("current_version") + if current and current in versions: + return versions[current] + return versions[list(versions)[-1]] + + +def _as_id_list(value: Any) -> list[str]: + """Normalise a sphinx-needs link field into a list of target ids. + + Link fields may be a list, a comma-separated string, or a single id. + """ + if value is None: + return [] + if isinstance(value, list): + return [str(item) for item in value if item] + if isinstance(value, str): + return [part.strip() for part in value.split(",") if part.strip()] + return [str(value)] + + +def _location(need: dict[str, Any]) -> dict[str, Any]: + """Build a LOBSTER 'file' source reference from a need's docname/lineno.""" + docname = need.get("docname") + filename = f"{docname}.rst" if docname else "unknown" + lineno = need.get("lineno") + return { + "kind": "file", + "file": filename, + "line": int(lineno) if isinstance(lineno, int) else None, + "column": None, + } + + +def _refs(need: dict[str, Any], namespace: str, up_links: tuple[str, ...]) -> list[str]: + """Collect up-trace targets from the configured link fields as LOBSTER tags.""" + refs: list[str] = [] + seen: set[str] = set() + for field in up_links: + for target in _as_id_list(need.get(field)): + tag = f"{namespace} {target}" + if tag not in seen: + seen.add(tag) + refs.append(tag) + return refs + + +def _text(need: dict[str, Any]) -> str | None: + """Return the need body text, tolerating sphinx-needs field naming.""" + for field in ("content", "description"): + value = need.get(field) + if isinstance(value, str) and value.strip(): + return value + return None + + +def need_to_item( + need: dict[str, Any], + namespace: str, + up_links: tuple[str, ...], +) -> dict[str, Any]: + """Convert a single sphinx-needs need into a LOBSTER requirement item.""" + need_id = str(need["id"]) + item: dict[str, Any] = { + "tag": f"{namespace} {need_id}", + "location": _location(need), + "name": need.get("title") or need_id, + "messages": [], + "just_up": _as_id_list(need.get("just_up")), + "just_down": _as_id_list(need.get("just_down")), + "just_global": _as_id_list(need.get("just_global")), + "refs": _refs(need, namespace, up_links), + "framework": "sphinx-needs", + "kind": need.get("type") or "need", + "text": _text(need), + "status": need.get("status"), + } + return item + + +def convert( + needs_data: dict[str, Any], + namespace: str = "req", + up_links: tuple[str, ...] = _DEFAULT_UP_LINKS, + include_types: set[str] | None = None, + version: str | None = None, +) -> dict[str, Any]: + """Convert a parsed ``needs.json`` document into a lobster-req-trace document.""" + version_data = _select_version(needs_data, version) + needs = version_data.get("needs", {}) + items = [ + need_to_item(need, namespace, up_links) + for need in needs.values() + if include_types is None or need.get("type") in include_types + ] + return { + "data": items, + "generator": _GENERATOR, + "schema": _LOBSTER_SCHEMA, + "version": _LOBSTER_VERSION, + } + + +def _parse_csv(value: str | None) -> tuple[str, ...]: + if not value: + return () + return tuple(part.strip() for part in value.split(",") if part.strip()) + + +def _resolve(path: Path) -> Path: + """Resolve a relative path against the invocation directory. + + Under ``bazel run`` the process working directory is the runfiles tree, so + relative paths would resolve there instead of where the user invoked the + command. Bazel exposes the original directory via ``BUILD_WORKING_DIRECTORY``; + resolve relative paths against it when present so ``--needs-json``/``--output`` + behave as expected. Absolute paths are returned unchanged. + """ + if path.is_absolute(): + return path + base = os.environ.get("BUILD_WORKING_DIRECTORY") + return Path(base) / path if base else path + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Convert a sphinx-needs needs.json into a LOBSTER " + "lobster-req-trace (.lobster) file." + ) + ) + parser.add_argument( + "--needs-json", + required=True, + type=Path, + help="Path to the input needs.json produced by the docs build.", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Path to the output .lobster file (default: stdout).", + ) + parser.add_argument( + "--namespace", + default="req", + help="LOBSTER tag namespace for the emitted items (default: 'req').", + ) + parser.add_argument( + "--up-links", + default=",".join(_DEFAULT_UP_LINKS), + help=( + "Comma-separated sphinx-needs link fields to emit as up-trace refs " + "(default: 'satisfies')." + ), + ) + parser.add_argument( + "--types", + default=None, + help=( + "Comma-separated need types to include (default: all types). " + "Example: 'tool_req' or 'gd_req'." + ), + ) + parser.add_argument( + "--version", + default=None, + help="needs.json version key to convert (default: current_version).", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + + needs_json = _resolve(args.needs_json) + output = _resolve(args.output) if args.output is not None else None + + needs_data = json.loads(needs_json.read_text(encoding="utf-8")) + include_types = set(_parse_csv(args.types)) or None + report = convert( + needs_data, + namespace=args.namespace, + up_links=_parse_csv(args.up_links) or _DEFAULT_UP_LINKS, + include_types=include_types, + version=args.version, + ) + + payload = json.dumps(report, indent=2, sort_keys=True) + if output is None: + print(payload) + else: + output.write_text(payload + "\n", encoding="utf-8") + print( + f"Wrote {len(report['data'])} requirement item(s) to {output}", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts_bazel/tests/BUILD b/scripts_bazel/tests/BUILD index e6fab5f1c..9bcf15253 100644 --- a/scripts_bazel/tests/BUILD +++ b/scripts_bazel/tests/BUILD @@ -41,3 +41,12 @@ score_pytest( ] + all_requirements, pytest_config = "//:pyproject.toml", ) + +score_pytest( + name = "needs_to_lobster_test", + srcs = ["needs_to_lobster_test.py"], + deps = [ + "//scripts_bazel:needs_to_lobster", + ] + all_requirements, + pytest_config = "//:pyproject.toml", +) diff --git a/scripts_bazel/tests/needs_to_lobster_test.py b/scripts_bazel/tests/needs_to_lobster_test.py new file mode 100644 index 000000000..0c0b1dc0d --- /dev/null +++ b/scripts_bazel/tests/needs_to_lobster_test.py @@ -0,0 +1,188 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Tests for the needs.json -> LOBSTER lobster-req-trace converter.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +import scripts_bazel.needs_to_lobster as conv + + +def _needs_doc() -> dict[str, Any]: + return { + "current_version": "1.0", + "project": "test", + "versions": { + "1.0": { + "needs": { + "tool_req__example": { + "id": "tool_req__example", + "type": "tool_req", + "title": "Example tool requirement", + "status": "valid", + "docname": "internals/requirements/requirements", + "lineno": 42, + "content": "The tool shall do the thing.", + "satisfies": ["gd_req__process_one", "gd_req__process_two"], + }, + "gd_req__process_one": { + "id": "gd_req__process_one", + "type": "gd_req", + "title": "Process requirement one", + "status": "valid", + "docname": "process/reqs", + "lineno": 7, + }, + } + } + }, + } + + +def test_convert_emits_req_trace_envelope() -> None: + report = conv.convert(_needs_doc()) + assert report["schema"] == "lobster-req-trace" + assert report["version"] == 3 + assert report["generator"] == "needs_to_lobster" + assert len(report["data"]) == 2 + + +def test_satisfies_becomes_refs() -> None: + report = conv.convert(_needs_doc()) + items = {item["tag"]: item for item in report["data"]} + tool = items["req tool_req__example"] + assert tool["refs"] == ["req gd_req__process_one", "req gd_req__process_two"] + assert tool["kind"] == "tool_req" + assert tool["framework"] == "sphinx-needs" + assert tool["status"] == "valid" + assert tool["text"] == "The tool shall do the thing." + + +def test_location_is_file_reference() -> None: + report = conv.convert(_needs_doc()) + items = {item["tag"]: item for item in report["data"]} + tool = items["req tool_req__example"] + assert tool["location"] == { + "kind": "file", + "file": "internals/requirements/requirements.rst", + "line": 42, + "column": None, + } + + +def test_custom_namespace() -> None: + report = conv.convert(_needs_doc(), namespace="sn") + tags = {item["tag"] for item in report["data"]} + assert "sn tool_req__example" in tags + tool = next(i for i in report["data"] if i["tag"] == "sn tool_req__example") + assert tool["refs"] == ["sn gd_req__process_one", "sn gd_req__process_two"] + + +def test_type_filter() -> None: + report = conv.convert(_needs_doc(), include_types={"tool_req"}) + assert [item["tag"] for item in report["data"]] == ["req tool_req__example"] + + +def test_up_links_configurable() -> None: + doc = _needs_doc() + doc["versions"]["1.0"]["needs"]["tool_req__example"]["implements"] = ["arch__x"] + report = conv.convert(doc, up_links=("satisfies", "implements")) + tool = next(i for i in report["data"] if i["tag"] == "req tool_req__example") + assert "req arch__x" in tool["refs"] + + +def test_comma_separated_links_are_normalised() -> None: + doc = _needs_doc() + doc["versions"]["1.0"]["needs"]["tool_req__example"]["satisfies"] = ( + "gd_req__a, gd_req__b" + ) + report = conv.convert(doc) + tool = next(i for i in report["data"] if i["tag"] == "req tool_req__example") + assert tool["refs"] == ["req gd_req__a", "req gd_req__b"] + + +def test_missing_optional_fields_are_tolerated() -> None: + doc = { + "current_version": "1.0", + "versions": {"1.0": {"needs": {"n": {"id": "n", "type": "feat_req"}}}}, + } + report = conv.convert(doc) + item = report["data"][0] + assert item["name"] == "n" + assert item["refs"] == [] + assert item["text"] is None + assert item["location"]["file"] == "unknown" + assert item["location"]["line"] is None + + +def test_select_version_explicit_and_fallback() -> None: + doc = { + "versions": { + "0.9": {"needs": {"a": {"id": "a", "type": "t"}}}, + "1.0": {"needs": {"b": {"id": "b", "type": "t"}}}, + } + } + # No current_version -> last key wins. + report = conv.convert(doc) + assert report["data"][0]["tag"] == "req b" + # Explicit version selection. + report_09 = conv.convert(doc, version="0.9") + assert report_09["data"][0]["tag"] == "req a" + with pytest.raises(ValueError): + conv.convert(doc, version="2.0") + + +def test_relative_paths_resolve_against_build_working_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + workdir = tmp_path / "workdir" + workdir.mkdir() + (workdir / "needs.json").write_text(json.dumps(_needs_doc()), encoding="utf-8") + + # Simulate `bazel run`: process cwd differs from the invocation directory. + elsewhere = tmp_path / "runfiles" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + monkeypatch.setenv("BUILD_WORKING_DIRECTORY", str(workdir)) + + rc = conv.main(["--needs-json", "needs.json", "--output", "out.lobster"]) + assert rc == 0 + assert (workdir / "out.lobster").is_file() + assert not (elsewhere / "out.lobster").exists() + + +def test_main_writes_output_file(tmp_path: Path) -> None: + needs_path = tmp_path / "needs.json" + needs_path.write_text(json.dumps(_needs_doc()), encoding="utf-8") + out_path = tmp_path / "out.lobster" + rc = conv.main( + [ + "--needs-json", + str(needs_path), + "--output", + str(out_path), + "--types", + "tool_req", + ] + ) + assert rc == 0 + report = json.loads(out_path.read_text(encoding="utf-8")) + assert report["schema"] == "lobster-req-trace" + assert [item["tag"] for item in report["data"]] == ["req tool_req__example"]