From 48a18cd81aa7c92fcc5ab9c411bde0fc133d1b53 Mon Sep 17 00:00:00 2001 From: Jochen Hoenle <173445474+hoe-jo@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:04:05 +0200 Subject: [PATCH] [rules score] implement requirements coverage --- MODULE.bazel | 2 +- .../docs/requirements/tool_requirements.trlc | 101 +++++ .../docs/requirements/use_cases.trlc | 47 +++ .../rules_score/docs/tool_reference/index.rst | 1 + .../docs/tool_reference/req_coverage.rst | 137 +++++++ .../rules_score/docs/user_guide/index.rst | 1 + .../rules_score/docs/user_guide/validation.md | 73 ++++ bazel/rules/rules_score/examples/seooc/BUILD | 3 + .../examples/seooc/coverage.lock.yaml | 72 ++++ .../examples/seooc/unit_1/foo_test.cpp | 18 + .../examples/seooc/unit_2/bar_test.cpp | 4 + .../lobster/config/lobster_de.conf.tpl | 5 + bazel/rules/rules_score/private/component.bzl | 141 ++++++- .../private/dependable_element.bzl | 65 ++- bazel/rules/rules_score/providers.bzl | 17 +- .../rules/rules_score/src/req_coverage/BUILD | 172 ++++++++ .../rules_score/src/req_coverage/__init__.py | 20 + .../assets/component_diagram.puml | 81 ++++ .../src/req_coverage/check_lock.py | 140 +++++++ .../src/req_coverage/check_lock_test.py | 157 ++++++++ .../src/req_coverage/compute_lock.py | 187 +++++++++ .../src/req_coverage/compute_lock_test.py | 143 +++++++ .../rules_score/src/req_coverage/conftest.py | 79 ++++ .../src/req_coverage/lobster_generator.py | 151 +++++++ .../req_coverage/lobster_generator_test.py | 192 +++++++++ .../src/req_coverage/read_gtest_lobster.py | 238 +++++++++++ .../req_coverage/read_gtest_lobster_test.py | 376 ++++++++++++++++++ .../src/req_coverage/test_runner.py | 135 +++++++ .../src/req_coverage/test_runner_test.py | 125 ++++++ .../src/req_coverage/update_runner.py | 116 ++++++ .../src/req_coverage/update_runner_test.py | 93 +++++ bazel/rules/rules_score/test/BUILD | 17 + .../coverage_lock_test/coverage.lock.yaml | 14 + .../rules_score/test/unit_component_test.bzl | 37 ++ bazel/rules/rules_score/trlc/config/BUILD | 4 +- .../rules/rules_score/trlc/config/tools.trlc | 5 + 36 files changed, 3163 insertions(+), 6 deletions(-) create mode 100644 bazel/rules/rules_score/docs/tool_reference/req_coverage.rst create mode 100644 bazel/rules/rules_score/docs/user_guide/validation.md create mode 100644 bazel/rules/rules_score/examples/seooc/coverage.lock.yaml create mode 100644 bazel/rules/rules_score/src/req_coverage/BUILD create mode 100644 bazel/rules/rules_score/src/req_coverage/__init__.py create mode 100644 bazel/rules/rules_score/src/req_coverage/assets/component_diagram.puml create mode 100644 bazel/rules/rules_score/src/req_coverage/check_lock.py create mode 100644 bazel/rules/rules_score/src/req_coverage/check_lock_test.py create mode 100644 bazel/rules/rules_score/src/req_coverage/compute_lock.py create mode 100644 bazel/rules/rules_score/src/req_coverage/compute_lock_test.py create mode 100644 bazel/rules/rules_score/src/req_coverage/conftest.py create mode 100644 bazel/rules/rules_score/src/req_coverage/lobster_generator.py create mode 100644 bazel/rules/rules_score/src/req_coverage/lobster_generator_test.py create mode 100644 bazel/rules/rules_score/src/req_coverage/read_gtest_lobster.py create mode 100644 bazel/rules/rules_score/src/req_coverage/read_gtest_lobster_test.py create mode 100644 bazel/rules/rules_score/src/req_coverage/test_runner.py create mode 100644 bazel/rules/rules_score/src/req_coverage/test_runner_test.py create mode 100644 bazel/rules/rules_score/src/req_coverage/update_runner.py create mode 100644 bazel/rules/rules_score/src/req_coverage/update_runner_test.py create mode 100644 bazel/rules/rules_score/test/fixtures/coverage_lock_test/coverage.lock.yaml diff --git a/MODULE.bazel b/MODULE.bazel index ee1b738e..968faeb6 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -349,7 +349,7 @@ bazel_dep(name = "trlc", version = "3.0.0") bazel_dep(name = "lobster", version = "1.0.3") git_override( module_name = "lobster", - commit = "07382c33f95e22992e0f8788c0537ec06877e51f", + commit = "247927a438647f68c70a61f9fdde155f5aee145d", remote = "https://github.com/bmw-software-engineering/lobster.git", ) diff --git a/bazel/rules/rules_score/docs/requirements/tool_requirements.trlc b/bazel/rules/rules_score/docs/requirements/tool_requirements.trlc index 0af48c4c..f9c1828e 100644 --- a/bazel/rules/rules_score/docs/requirements/tool_requirements.trlc +++ b/bazel/rules/rules_score/docs/requirements/tool_requirements.trlc @@ -176,4 +176,105 @@ section "Tool Requirements" { } + section "Requirement Coverage" { + + ToolQualification.ToolRequirement Lock_File_As_Spec_And_Approval { + description = ''' + The tool shall use one YAML lock file as both the declaration of + which requirements are claimed to be completely covered and as the + committed approval artifact. + ''' + derived_from = [Declare_Complete_Coverage] + satisfied_by = Tools.ReqCoverage + } + + ToolQualification.ToolRequirement Requirement_IDs_From_TRLC { + description = ''' + The tool shall extract requirement identifiers automatically from + the TrlcProviderInfo provider of the requirements target attribute. + ''' + derived_from = [Semi_Automated_Lock_Update, Declare_Complete_Coverage] + satisfied_by = Tools.ReqCoverage + } + + ToolQualification.ToolRequirement Parse_GWT_Annotations { + description = ''' + The tool shall parse GTest XML result files, extracting + RecordProperty values for the keys lobster-tracing, given, when, + and then on a per-TEST, per-TEST_F, and per-TEST_P basis. A test case + without a given, when, or then property shall be included with an + empty string for each missing field. + ''' + derived_from = [Detect_Specification_Drift, Detect_Coverage_Set_Changes] + satisfied_by = Tools.ReqCoverage + } + + ToolQualification.ToolRequirement Multi_Requirement_Linking { + description = ''' + A test case that carries lobster-tracing values matching more than + one requirement identifier shall appear in the lock file entry of + every matched requirement. + ''' + derived_from = [Declare_Complete_Coverage] + satisfied_by = Tools.ReqCoverage + } + + ToolQualification.ToolRequirement Lock_File_Content { + description = ''' + The lock file shall contain, per requirement: the requirement + identifier and an ordered list of linked test cases. Each test + case entry shall contain: a uid in the format + "//bazel_package/SuiteName:TestName" (package-scoped gtest tag) + and the individual given, when, and then strings as separate + fields. + ''' + derived_from = [Detect_Specification_Drift, Declare_Complete_Coverage] + satisfied_by = Tools.ReqCoverage + } + + ToolQualification.ToolRequirement Lock_File_Ordering { + description = ''' + Test cases within a requirement entry shall be ordered + lexicographically by uid to produce a deterministic, + diff-friendly lock file. + ''' + derived_from = [Detect_Specification_Drift, Detect_Coverage_Set_Changes] + satisfied_by = Tools.ReqCoverage + } + + ToolQualification.ToolRequirement Update_Command { + description = ''' + The .update Bazel run target shall extract requirement identifiers + from the TrlcProviderInfo target, parse GTest XML result files for + lobster-tracing-linked test cases, build the spec from + given/when/then, and overwrite the committed lock file. + ''' + derived_from = [Semi_Automated_Lock_Update] + satisfied_by = Tools.ReqCoverage + } + + ToolQualification.ToolRequirement Test_Command_Verification { + description = ''' + The Bazel test target shall parse GTest XML result files, rebuild + the spec for each linked test, and compare uid and spec against + the committed lock file, failing if they differ. On failure it + shall print a human-readable diff showing which test cases changed + and the exact bazel run command needed to refresh the lock file. + ''' + derived_from = [Detect_Specification_Drift, Detect_Coverage_Set_Changes] + satisfied_by = Tools.ReqCoverage + } + + ToolQualification.ToolRequirement LOBSTER_Covered_Attribute { + description = ''' + The test target shall emit a lobster-act-trace JSON artifact that + marks each linked test case as "ok" when its uid exists in the + committed lock file and its spec matches, and "fail" otherwise. + ''' + derived_from = [Generate_Covered_Traceability_Artifact] + satisfied_by = Tools.ReqCoverage + } + + } + } diff --git a/bazel/rules/rules_score/docs/requirements/use_cases.trlc b/bazel/rules/rules_score/docs/requirements/use_cases.trlc index 96f310ed..0a2fa24a 100644 --- a/bazel/rules/rules_score/docs/requirements/use_cases.trlc +++ b/bazel/rules/rules_score/docs/requirements/use_cases.trlc @@ -173,4 +173,51 @@ section "Use Cases" { affected_tools = [Tools.Verifier, Tools.PlantumlParser, Tools.LibClang, Tools.Bazel] } + ToolQualification.UseCase Declare_Complete_Coverage { + description = ''' + As a safety engineer I want to declare that a set of requirements + is completely covered by automated test cases so that the coverage + claim is explicit, auditable, and version-controlled. + ''' + affected_tools = [Tools.ReqCoverage] + } + + ToolQualification.UseCase Detect_Specification_Drift { + description = ''' + In the CI pipeline I want to detect automatically when a test's + given/when/then specification has changed without re-approval so + that stale coverage claims are caught before release. + ''' + affected_tools = [Tools.ReqCoverage] + } + + ToolQualification.UseCase Detect_Coverage_Set_Changes { + description = ''' + In the CI pipeline I want to detect when a test case is added to or + removed from the set of tests linked to a requirement so that + undeclared coverage changes cannot pass silently. + ''' + affected_tools = [Tools.ReqCoverage] + } + + ToolQualification.UseCase Generate_Covered_Traceability_Artifact { + description = ''' + As a project integrator I want the tool to emit a LOBSTER-compatible + artifact that marks each linked test case as covered or not so that + complete requirement coverage is visible in the LOBSTER dashboard. + ''' + affected_tools = [Tools.ReqCoverage, Tools.Lobster] + } + + ToolQualification.UseCase Semi_Automated_Lock_Update { + description = ''' + As an engineer I want a single bazel run command that reads + requirement identifiers from the TRLC requirements target, + discovers all linked test cases in the GTest XML results, and + rewrites the lock file, so that I only need to review the diff + before committing my approval. + ''' + affected_tools = [Tools.ReqCoverage, Tools.Bazel] + } + } diff --git a/bazel/rules/rules_score/docs/tool_reference/index.rst b/bazel/rules/rules_score/docs/tool_reference/index.rst index 13e4b10f..81f9cf67 100644 --- a/bazel/rules/rules_score/docs/tool_reference/index.rst +++ b/bazel/rules/rules_score/docs/tool_reference/index.rst @@ -48,3 +48,4 @@ of truth. lobster_bazel lobster + req_coverage diff --git a/bazel/rules/rules_score/docs/tool_reference/req_coverage.rst b/bazel/rules/rules_score/docs/tool_reference/req_coverage.rst new file mode 100644 index 00000000..2bc533ff --- /dev/null +++ b/bazel/rules/rules_score/docs/tool_reference/req_coverage.rst @@ -0,0 +1,137 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Requirement Coverage (req_coverage) +==================================== + +``req_coverage`` verifies that a declared set of requirements is **completely** +covered by automated test cases. Coverage is locked and confirmed by a committed YAML file; + +Requirement IDs are extracted automatically and test records are read from the ``gtest.lobster`` file. + +.. note:: + + For how to annotate tests with ``lobster-tracing`` see + :doc:`../user_guide/validation`. + +The ``component`` rule integrates requirement coverage when ``coverage_lock`` +is provided: + +.. code-block:: starlark + + component( + name = "my_component", + requirements = [":my_component_requirements"], + components = [":unit_a", ":unit_b"], + coverage_lock = "coverage.lock.yaml", + ) + +When the build reports a req_coverage drift failure, run (here on my_component): + + .. code-block:: bash + + bazel run //:.update + +Two phases +---------- + +``req_coverage`` has two distinct operational phases: + +**Test phase** — ``bazel build //...`` / ``bazel test //...`` + + ``test_runner`` is invoked as a **build action** inside the + ``dependable_element`` rule for each component that has ``coverage_lock`` + set. The effective enforcement stringency is controlled by the enclosing + ``dependable_element``'s ``maturity`` attribute: + + * ``maturity = "development"`` — ``--allow-check-failures`` is passed so + the ``.lobster`` artifact is always produced even when coverage drifts. + Both lock-drift failures **and** missing-GWT-annotation errors are + downgraded to warnings; + * ``maturity = "release"`` — no ``--allow-check-failures``; a drift or + missing GWT annotation fails the Bazel build action directly. The + ``.lobster`` artifact is physically written before the action exits, but + Bazel marks the action as failed. + +**Run phase** — ``bazel run //:.update`` + + Computes the lock from the current ``gtest.lobster`` and **overwrites** + the committed ``coverage.lock.yaml`` in the workspace. Committing the + result constitutes approval of requirement coverage. + +Architecture +------------ + +.. code-block:: text + + bazel/rules/rules_score/src/req_coverage/ + ├── read_gtest_lobster.py # reads gtest.lobster; also resolve_path, scan, req_ids + ├── compute_lock.py # uid/spec serialisation + YAML lock + ├── check_lock.py # committed vs computed comparison + ├── lobster_generator.py # emits LOBSTER activity artifact + ├── update_runner.py # entry point: bazel run .update + └── test_runner.py # entry point: bazel test //... + +Data flow +--------- + +.. code-block:: text + + Requirements targets (.lobster) ──► extract requirement IDs + subrule_lobster_gtest ───────────► gtest.lobster + │ + read_gtest_lobster + (uid, req refs, spec text) + │ + compute_lock + ┌─────────┴──────────────┐ + RUN phase TEST phase + (bazel run .update) (bazel test) + │ │ + rewrite lock compare uid+spec + file in WS vs committed lock + │ + lobster_generator + (ok / fail per test) + +Lock file format +---------------- + +Auto-generated by the ``.update`` target. The engineer commits it after +reviewing ``git diff``. + +.. code-block:: yaml + + schema_version: 3 + requirements: + - id: MessagePassing.OsIpcFaultHandling + test_cases: + - uid: "//score/message_passing/ConnectionSuite:OsIpcFaultHandlingTest" + given: a connected client + when: the OS IPC call fails + then: the client receives an error + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Field + - Notes + * - ``requirements[].id`` + - Matches ``lobster-tracing`` values; extracted from requirements targets + * - ``test_cases[].uid`` + - ``//bazel_package/SuiteName:TestName`` — package-scoped gtest tag + * - ``test_cases[].given`` / ``when`` / ``then`` + - Individual GWT fields from ``RecordProperty`` annotations; any change + makes the lock stale diff --git a/bazel/rules/rules_score/docs/user_guide/index.rst b/bazel/rules/rules_score/docs/user_guide/index.rst index 20158d22..8bddedd0 100644 --- a/bazel/rules/rules_score/docs/user_guide/index.rst +++ b/bazel/rules/rules_score/docs/user_guide/index.rst @@ -23,3 +23,4 @@ User Guide architectural_design unit_design dependability_analysis + validation diff --git a/bazel/rules/rules_score/docs/user_guide/validation.md b/bazel/rules/rules_score/docs/user_guide/validation.md new file mode 100644 index 00000000..d605dc2f --- /dev/null +++ b/bazel/rules/rules_score/docs/user_guide/validation.md @@ -0,0 +1,73 @@ + + +# Validation + +## Unit Tests + +Unit tests in `rules_score` components are written with **GoogleTest** and built +with `cc_test`. Each test case that covers a component requirement must carry +a `lobster-tracing` annotation so that the build can link the test back to the +requirement. + +### Annotating Tests + +Call `RecordProperty` inside the test body next to the respective code blocks: + +```cpp +TEST_F(MyFixture, DoesXWhenY) { + ::testing::Test::RecordProperty("lobster-tracing", "MessagePassing.OsIpcFaultHandling"); + ::testing::Test::RecordProperty("given", "a connected client"); + ::testing::Test::RecordProperty("when", "the OS IPC call fails"); + ::testing::Test::RecordProperty("then", "the client receives an error"); +} +``` + +| Property | Required | Description | +|---|---|---| +| `lobster-tracing` | yes | Comma-separated requirement IDs; links the test to one or more `CompReq` records | +| `given` | no | Initial state / precondition | +| `when` | no | Action or event under test | +| `then` | no | Expected outcome | + +A test without `lobster-tracing` has no traceability and is not included in +coverage tracking. + +### Stating Coverage for a requirement + +Coverage is declared through a committed `coverage.lock.yaml` file that lists, +per requirement, every test case (uid + given/when/then) that covers it. +Committing the file is stating the coverage claim. + +In Bazel the yaml file can be linked to the `component` macro via the `coverage_lock` attribute: + +```starlark +component( + name = "my_component", + requirements = [":my_component_requirements"], + components = [":unit_a", ":unit_b"], + coverage_lock = "coverage.lock.yaml", +) +``` + +Two complementary workflows keep the lock file in sync: + +- **`bazel run …update`** — reads the current test results and **rewrites** + `coverage.lock.yaml` in the source tree. +- **`bazel test //…`** — the build action recomputes coverage from the same + test results and **compares** it against the committed lock. Any drift (new + test, removed test, changed GWT text, version bump) fails the build until the + lock is refreshed and re-committed. + +For the full tool description — lock file format, update workflow, design +decisions — see {doc}`../tool_reference/req_coverage`. diff --git a/bazel/rules/rules_score/examples/seooc/BUILD b/bazel/rules/rules_score/examples/seooc/BUILD index fd0653ae..e3501d07 100644 --- a/bazel/rules/rules_score/examples/seooc/BUILD +++ b/bazel/rules/rules_score/examples/seooc/BUILD @@ -33,9 +33,11 @@ component( "//bazel/rules/rules_score/examples/seooc/unit_1:unit_1", "//bazel/rules/rules_score/examples/seooc/unit_2:unit_2", ], + coverage_lock = "coverage.lock.yaml", requirements = [ "//bazel/rules/rules_score/examples/seooc/docs/requirements:component_requirements", "//bazel/rules/rules_score/examples/seooc/docs/requirements:component_requirements_extra", + "//bazel/rules/rules_score/examples/seooc/docs/requirements:feature_requirements", ], tags = ["manual"], tests = [], @@ -60,6 +62,7 @@ dependable_element( ], glossary = ["//bazel/rules/rules_score/examples/seooc/docs:glossary"], integrity_level = "B", + maturity = "development", requirements = [ "//bazel/rules/rules_score/examples/seooc/docs/requirements:feature_requirements", ], diff --git a/bazel/rules/rules_score/examples/seooc/coverage.lock.yaml b/bazel/rules/rules_score/examples/seooc/coverage.lock.yaml new file mode 100644 index 00000000..e0a7e34c --- /dev/null +++ b/bazel/rules/rules_score/examples/seooc/coverage.lock.yaml @@ -0,0 +1,72 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +# coverage.lock.yaml — auto-generated by req_coverage +# Run `bazel run @@//bazel/rules/rules_score/examples/seooc:component_example.update.update` to refresh. +schema_version: 3 +requirements: + - id: SampleComponent.REQ_COMP_001 + version: '1' + description: The numeric value management interface shall provide a read operation that returns a uint8_t value + test_cases: + - uid: //bazel/rules/rules_score/examples/seooc/Foo:GetNumber + given: a default-constructed Foo instance + when: GetNumber is called + then: it returns 42 + - id: SampleComponent.REQ_COMP_002 + version: '2' + description: The numeric value management interface shall not be extensible by external code + test_cases: + - uid: //bazel/rules/rules_score/examples/seooc/Foo:IsFinal + given: the Foo class definition + when: checking whether the class is extensible + then: it is declared final, preventing any subclassing + - id: SampleComponent.REQ_COMP_003 + version: '1' + description: The numeric value validator shall return a boolean result indicating whether the managed numeric value equals 42 + test_cases: + - uid: //bazel/rules/rules_score/examples/seooc/Bar:AssertNumber + given: a Bar instance owning a Foo with value 4 + when: AssertNumber is called + then: it returns true + - id: SampleComponent.REQ_COMP_004 + version: '1' + description: The numeric value validator shall accept a numeric value manager instance as its sole constructor argument + test_cases: + - uid: //bazel/rules/rules_score/examples/seooc/Bar:AssertNumber + given: a Bar instance owning a Foo with value 42 + when: AssertNumber is called + then: it returns true + - id: SampleComponent.REQ_COMP_005 + version: '1' + description: The numeric value validator shall take exclusive ownership of the received numeric value manager instance + test_cases: + - uid: //bazel/rules/rules_score/examples/seooc/Bar:AssertNumber + given: a Bar instance owning a Foo with value 42 + when: AssertNumber is called + then: it returns true + - id: SampleComponentExtra.REQ_COMP_EXTRA_001 + version: '1' + description: The numeric value management interface shall initialize its internal value to zero upon construction + test_cases: + - uid: //bazel/rules/rules_score/examples/seooc/Foo:InitializesToKnownValue + given: a default-constructed Foo instance + when: GetNumber is called for the first time + then: it returns 42 + - id: SampleComponentExtra.REQ_COMP_EXTRA_002 + version: '1' + description: The numeric value management interface shall guarantee that the stored value is never modified after construction + test_cases: + - uid: //bazel/rules/rules_score/examples/seooc/Foo:ValueConsistentAcrossReads + given: a const Foo instance + when: GetNumber is called multiple times + then: the same value is returned on each call diff --git a/bazel/rules/rules_score/examples/seooc/unit_1/foo_test.cpp b/bazel/rules/rules_score/examples/seooc/unit_1/foo_test.cpp index 42e9c575..62f50876 100644 --- a/bazel/rules/rules_score/examples/seooc/unit_1/foo_test.cpp +++ b/bazel/rules/rules_score/examples/seooc/unit_1/foo_test.cpp @@ -18,6 +18,10 @@ TEST(Foo, GetNumber) { ::testing::Test::RecordProperty("lobster-tracing", "SampleComponent.REQ_COMP_001"); + ::testing::Test::RecordProperty("given", + "a default-constructed Foo instance"); + ::testing::Test::RecordProperty("when", "GetNumber is called"); + ::testing::Test::RecordProperty("then", "it returns 42"); unit_1::Foo unit{}; @@ -27,6 +31,11 @@ TEST(Foo, GetNumber) { TEST(Foo, IsFinal) { ::testing::Test::RecordProperty("lobster-tracing", "SampleComponent.REQ_COMP_002"); + ::testing::Test::RecordProperty("given", "the Foo class definition"); + ::testing::Test::RecordProperty("when", + "checking whether the class is extensible"); + ::testing::Test::RecordProperty( + "then", "it is declared final, preventing any subclassing"); // Foo is declared final; extensibility is enforced at compile time. SUCCEED(); } @@ -34,6 +43,11 @@ TEST(Foo, IsFinal) { TEST(Foo, InitializesToKnownValue) { ::testing::Test::RecordProperty("lobster-tracing", "SampleComponentExtra.REQ_COMP_EXTRA_001"); + ::testing::Test::RecordProperty("given", + "a default-constructed Foo instance"); + ::testing::Test::RecordProperty("when", + "GetNumber is called for the first time"); + ::testing::Test::RecordProperty("then", "it returns 42"); unit_1::Foo unit{}; EXPECT_EQ(unit.GetNumber(), 42u); @@ -42,6 +56,10 @@ TEST(Foo, InitializesToKnownValue) { TEST(Foo, ValueConsistentAcrossReads) { ::testing::Test::RecordProperty("lobster-tracing", "SampleComponentExtra.REQ_COMP_EXTRA_002"); + ::testing::Test::RecordProperty("given", "a const Foo instance"); + ::testing::Test::RecordProperty("when", "GetNumber is called multiple times"); + ::testing::Test::RecordProperty("then", + "the same value is returned on each call"); const unit_1::Foo unit{}; EXPECT_EQ(unit.GetNumber(), unit.GetNumber()); diff --git a/bazel/rules/rules_score/examples/seooc/unit_2/bar_test.cpp b/bazel/rules/rules_score/examples/seooc/unit_2/bar_test.cpp index 17334981..dadff493 100644 --- a/bazel/rules/rules_score/examples/seooc/unit_2/bar_test.cpp +++ b/bazel/rules/rules_score/examples/seooc/unit_2/bar_test.cpp @@ -19,6 +19,10 @@ TEST(Bar, AssertNumber) { ::testing::Test::RecordProperty( "lobster-tracing", "SampleComponent.REQ_COMP_003,SampleComponent.REQ_" "COMP_004,SampleComponent.REQ_COMP_005"); + ::testing::Test::RecordProperty("given", + "a Bar instance owning a Foo with value 42"); + ::testing::Test::RecordProperty("when", "AssertNumber is called"); + ::testing::Test::RecordProperty("then", "it returns true"); unit_2::Bar unit{std::make_unique()}; EXPECT_TRUE(unit.AssertNumber()); diff --git a/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl b/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl index b4ccfd6b..f23db6ed 100644 --- a/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl +++ b/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl @@ -13,6 +13,11 @@ activity "Unit Test" { {UNIT_TEST_SOURCES} } +activity "Requirement Coverage" { +{COVERAGE_SOURCES} + trace to: "Component Requirements"; +} + implementation "Architecture" { {ARCH_SOURCES} trace to: "Component Requirements"; diff --git a/bazel/rules/rules_score/private/component.bzl b/bazel/rules/rules_score/private/component.bzl index 1d38e02f..cb993c79 100644 --- a/bazel/rules/rules_score/private/component.bzl +++ b/bazel/rules/rules_score/private/component.bzl @@ -20,11 +20,117 @@ with associated requirements and tests. """ load("@lobster//:lobster.bzl", "subrule_lobster_gtest", "subrule_lobster_html_report", "subrule_lobster_report") -load("//bazel/rules/rules_score:providers.bzl", "AssumedSystemRequirementsInfo", "CertifiedScope", "ComponentInfo", "ComponentRequirementsInfo", "FeatureRequirementsInfo", "SphinxSourcesInfo", "UnitInfo") +load("//bazel/rules/rules_score:providers.bzl", "AssumedSystemRequirementsInfo", "CertifiedScope", "ComponentCoverageInfo", "ComponentInfo", "ComponentRequirementsInfo", "FeatureRequirementsInfo", "SphinxSourcesInfo", "UnitInfo") load("//bazel/rules/rules_score/private:lobster_config.bzl", "format_lobster_sources") # ============================================================================ -# Private Rule Implementation +# Private Rule Implementation: Component .update target +# ============================================================================ + +def _collect_req_lobster_files(ctx): + """Collect CompReq .lobster files from requirements targets. + + Only ComponentRequirementsInfo targets contribute files; FeatReq and + AssumedSystemReq targets are intentionally excluded because the coverage + runner filters to ``kind == CompReq`` anyway. + """ + req_lobster_files = [] + for req_target in ctx.attr.requirements: + if ComponentRequirementsInfo in req_target: + req_lobster_files.extend(req_target[ComponentRequirementsInfo].srcs.to_list()) + return req_lobster_files + +def _component_update_impl(ctx): + """Implementation for component.update — rewrites coverage.lock.yaml. + + Runs ``update_runner`` as an executable, passing the gtest.lobster file + (produced by subrule_lobster_gtest) and the req lobster manifest via + environment variables. + """ + + # Collect req lobster files from requirements targets + req_lobster_files = _collect_req_lobster_files(ctx) + + # Collect unit test XML files from sub-components/units + unit_test_files = [] + for comp in ctx.attr.components: + if UnitInfo in comp: + unit_test_files.append(comp[UnitInfo].tests) + + # Produce gtest.lobster via subrule_lobster_gtest + gtest_lobster_file, _ = subrule_lobster_gtest(depset(transitive = unit_test_files).to_list()) + + # Write a runfiles manifest (short paths) — used by the runner at bazel run time. + runfiles_manifest = ctx.actions.declare_file("{}_req_lobster_runfiles_manifest.txt".format(ctx.label.name)) + ctx.actions.write( + output = runfiles_manifest, + content = "\n".join([f.short_path for f in req_lobster_files]) + "\n", + ) + + # The lock file is written back to the source tree by update_runner using + # $BUILD_WORKSPACE_DIRECTORY. Pass only the package-relative path; no + # symlink into runfiles is needed (the runner does not read the lock). + lock_file = ctx.file.coverage_lock + + # Symlink the update_runner binary as the rule's executable. + executable = ctx.actions.declare_file("{}_update_runner_bin".format(ctx.label.name)) + ctx.actions.symlink( + output = executable, + target_file = ctx.executable._update_runner, + is_executable = True, + ) + + all_inputs = req_lobster_files + [gtest_lobster_file, runfiles_manifest] + runfiles = ctx.runfiles(files = all_inputs) + runfiles = runfiles.merge(ctx.attr._update_runner[DefaultInfo].default_runfiles) + + return [ + DefaultInfo( + executable = executable, + runfiles = runfiles, + ), + RunEnvironmentInfo(environment = { + "REQ_COVERAGE_LOBSTER_MANIFEST": runfiles_manifest.short_path, + "REQ_COVERAGE_GTEST_LOBSTER": gtest_lobster_file.short_path, + # Package-relative path; update_runner prepends $BUILD_WORKSPACE_DIRECTORY. + "REQ_COVERAGE_LOCK_FILE": ctx.label.package + "/" + lock_file.basename, + "REQ_COVERAGE_LABEL": str(ctx.label), + "REQ_COVERAGE_PACKAGE": "//" + ctx.label.package, + }), + ] + +_component_update = rule( + implementation = _component_update_impl, + doc = "Refreshes component coverage.lock.yaml with current test coverage.", + attrs = { + "requirements": attr.label_list( + default = [], + providers = [[ComponentRequirementsInfo], [FeatureRequirementsInfo], [AssumedSystemRequirementsInfo]], + doc = "Requirements targets forwarded from the component() macro. Only ComponentRequirementsInfo targets contribute files; FeatReq and AssumedSystemReq labels are accepted so the macro can pass its full requirements list through without filtering.", + ), + "components": attr.label_list( + default = [], + providers = [[ComponentInfo], [UnitInfo]], + doc = "Sub-component/unit targets (provides unit test XML via UnitInfo)", + ), + "coverage_lock": attr.label( + allow_single_file = True, + mandatory = True, + doc = "Committed coverage.lock.yaml file to overwrite.", + ), + "_update_runner": attr.label( + doc = "req_coverage update runner executable.", + default = Label("//bazel/rules/rules_score/src/req_coverage:update_runner"), + executable = True, + cfg = "exec", + ), + }, + executable = True, + subrules = [subrule_lobster_gtest], +) + +# ============================================================================ +# Private Rule Implementation: Component main rule # ============================================================================ def _component_impl(ctx): @@ -211,6 +317,12 @@ def _component_impl(ctx): ), dependent_labels = depset(transitive = collected_dependencies), ), + # ComponentCoverageInfo: coverage-lock data consumed by dependable_element + # (only emitted when coverage_lock is set; presence signals the DE to run the check). + ComponentCoverageInfo( + gtest_lobster_file = gtest_lobster_file, + coverage_lock_file = ctx.file.coverage_lock, + ), # SphinxSourcesInfo: RST sources from component requirements + transitive sources from sub-components/units SphinxSourcesInfo( srcs = req_sphinx_depset, @@ -240,6 +352,10 @@ _component_test = rule( default = [], doc = "Component-level integration test targets", ), + "coverage_lock": attr.label( + allow_single_file = True, + doc = "Optional committed coverage.lock.yaml file for coverage validation", + ), "_lobster_ci_report": attr.label( default = "@lobster//:lobster-ci-report", executable = True, @@ -270,6 +386,7 @@ def component( tests = [], requirements = None, components = [], + coverage_lock = None, testonly = True, **kwargs): """Define a software component following S-CORE process guidelines. @@ -279,6 +396,7 @@ def component( - Component requirements: Requirements specification for the component - Components: Nested components (for hierarchical structures) - Tests: Integration tests that verify the component as a whole + - Coverage: Optional requirement coverage traceability (if coverage_lock provided) Args: name: The name of the component. Used as the target name. @@ -294,6 +412,10 @@ def component( component structures). tests: List of labels to Bazel test targets that verify the component integration. + coverage_lock: Optional label to committed coverage.lock.yaml file. + If provided, the component rule generates coverage.lobster by comparing + computed test coverage against the committed lock. Enables `.update` + target for refreshing coverage. testonly: If true, only testonly targets can depend on this component. visibility: Bazel visibility specification for the component target. @@ -304,6 +426,7 @@ def component( requirements = [":kvs_component_requirements"], components = [":kvs_unit1", ":kvs_unit2"], tests = ["//persistency/kvs/tests:score_kvs_component_integration_tests"], + coverage_lock = "coverage.lock.yaml", visibility = ["//visibility:public"], ) ``` @@ -314,6 +437,20 @@ def component( requirements = requirements, components = components, tests = tests, + coverage_lock = coverage_lock, testonly = testonly, **kwargs ) + + # Create .update target if coverage_lock is provided + if coverage_lock: + # Only forward visibility — _component_update is an executable rule + # and does not accept the generic build attrs (tags, deprecation, etc.). + _component_update( + name = name + ".update", + requirements = requirements or [], + components = components, + coverage_lock = coverage_lock, + testonly = testonly, + visibility = kwargs.get("visibility"), + ) diff --git a/bazel/rules/rules_score/private/dependable_element.bzl b/bazel/rules/rules_score/private/dependable_element.bzl index 6536efd8..d77a6d8d 100644 --- a/bazel/rules/rules_score/private/dependable_element.bzl +++ b/bazel/rules/rules_score/private/dependable_element.bzl @@ -32,6 +32,7 @@ load( "AssumedSystemRequirementsInfo", "AssumptionsOfUseInfo", "CertifiedScope", + "ComponentCoverageInfo", "ComponentInfo", "DependabilityAnalysisInfo", "DependableElementInfo", @@ -1065,6 +1066,61 @@ def _dependable_element_index_impl(ctx): comp_test_lobster_depset = depset(transitive = comp_test_lobster_files) comp_arch_lobster_depset = depset(transitive = comp_arch_lobster_files) + # Coverage lock check: run test_runner once per component that has a + # coverage_lock_file. Maturity controls whether drift is an error or a + # warning — matching the existing certified-scope / validation pattern. + # ComponentCoverageInfo is only present on components that declared coverage_lock. + # ComponentInfo.requirements is CompReq-only (FeatReq/AssumedSystemReq are + # kept in a separate depset by _component_impl and not included here). + coverage_lobster_files = [] + for comp_target in ctx.attr.components: + if ComponentCoverageInfo not in comp_target: + continue + cov_info = comp_target[ComponentCoverageInfo] + if not cov_info.coverage_lock_file: + continue + + comp_info = comp_target[ComponentInfo] + req_files = comp_info.requirements.to_list() + gtest_lobster = cov_info.gtest_lobster_file + lock_file = cov_info.coverage_lock_file + comp_name = comp_target.label.name + + # Write req lobster manifest (exec paths — used inside the build action sandbox) + cov_req_manifest = ctx.actions.declare_file( + "{}/coverage_req_manifest_{}.txt".format(ctx.label.name, comp_name), + ) + ctx.actions.write( + output = cov_req_manifest, + content = "\n".join([f.path for f in req_files]) + "\n", + ) + + cov_lobster = ctx.actions.declare_file( + "{}/coverage_{}.lobster".format(ctx.label.name, comp_name), + ) + + cov_args = [] + if ctx.attr.maturity == "development": + cov_args.append("--allow-check-failures") + + ctx.actions.run( + executable = ctx.executable._test_runner, + inputs = depset(req_files + [gtest_lobster, cov_req_manifest, lock_file]), + outputs = [cov_lobster], + arguments = cov_args, + env = { + "REQ_COVERAGE_LOBSTER_MANIFEST": cov_req_manifest.path, + "REQ_COVERAGE_GTEST_LOBSTER": gtest_lobster.path, + "REQ_COVERAGE_LOCK_FILE": lock_file.path, + "REQ_COVERAGE_LABEL": str(comp_target.label), + "REQ_COVERAGE_PACKAGE": "//" + comp_target.label.package, + "REQ_COVERAGE_LOBSTER_OUTPUT": cov_lobster.path, + }, + mnemonic = "ComponentCoverageLockCheck", + progress_message = "Checking coverage lock for {}".format(comp_target.label), + ) + coverage_lobster_files.append(cov_lobster) + # Collect safety analysis lobster files from dependability_analysis targets sa_lobster_files = {} # canonical name -> File, merged from all DA targets for da_target in ctx.attr.dependability_analysis: @@ -1154,6 +1210,7 @@ def _dependable_element_index_impl(ctx): "{COMP_REQ_TRACE}": comp_req_trace_lines, "{ARCH_SOURCES}": format_lobster_sources(comp_arch_list), "{UNIT_TEST_SOURCES}": format_lobster_sources(comp_test_list), + "{COVERAGE_SOURCES}": format_lobster_sources(coverage_lobster_files), "{PUBLIC_API_SOURCES}": format_lobster_sources(interface_req_list), "{FM_SOURCES}": format_lobster_sources(fm_list), "{CM_SOURCES}": format_lobster_sources(cm_list), @@ -1161,7 +1218,7 @@ def _dependable_element_index_impl(ctx): }, ) - all_lobster_inputs = feat_req_list + comp_req_list + comp_arch_list + comp_test_list + interface_req_list + fm_list + cm_list + rc_list + received_aou_list + all_lobster_inputs = feat_req_list + comp_req_list + comp_arch_list + comp_test_list + interface_req_list + fm_list + cm_list + rc_list + received_aou_list + coverage_lobster_files lobster_report_file = subrule_lobster_report(all_lobster_inputs, lobster_config) lobster_html_report = subrule_lobster_html_report(lobster_report_file) @@ -1302,6 +1359,12 @@ def _dependable_element_index_attrs(): cfg = "exec", doc = "Tool for filtering received AoU lobster entries based on chain-forwarding YAML.", ), + "_test_runner": attr.label( + default = Label("//bazel/rules/rules_score/src/req_coverage:test_runner"), + executable = True, + cfg = "exec", + doc = "req_coverage test runner — checks coverage.lock.yaml for each component; maturity controls error vs. warning.", + ), } attrs.update(VALIDATION_ATTRS) attrs.update(VERBOSITY_ATTR) diff --git a/bazel/rules/rules_score/providers.bzl b/bazel/rules/rules_score/providers.bzl index 0bf78451..58613cac 100644 --- a/bazel/rules/rules_score/providers.bzl +++ b/bazel/rules/rules_score/providers.bzl @@ -147,7 +147,7 @@ ComponentInfo = provider( doc = "Provider for component artifacts.", fields = { "name": "Name of the component target.", - "requirements": "Depset of requirement traceability files (.lobster) collected from requirements targets, including transitive files from nested components.", + "requirements": "Depset of component requirement traceability files (.lobster) collected from ComponentRequirementsInfo targets only (CompReq kind). Does not include feature or assumed-system requirement files.", "components": "Depset of nested component and/or unit Targets that comprise this component.", "tests": "Depset of test traceability files (.lobster) generated from unit test results, collected transitively from all nested components and units.", "architecture": "Depset of architecture traceability files (.lobster) generated from unit architectural designs, collected transitively from all nested components and units.", @@ -155,6 +155,21 @@ ComponentInfo = provider( }, ) +ComponentCoverageInfo = provider( + doc = """Internal provider carrying coverage-lock data for a component. + + Returned alongside ComponentInfo when ``coverage_lock`` is set on a + ``component()`` target. Consumed by ``dependable_element`` to run the + per-component ``ComponentCoverageLockCheck`` build action. This is an + implementation detail of the ``component`` / ``dependable_element`` + interaction; do not depend on it from external rules. + """, + fields = { + "gtest_lobster_file": "The component's own gtest.lobster File produced by lobster-gtest. Used by the dependable_element to run the coverage lock check.", + "coverage_lock_file": "The committed coverage.lock.yaml File, or None if no coverage_lock is declared on the component.", + }, +) + CcDependencyInfo = provider( doc = """Provider for collecting transitive dependencies from C/C++ targets. diff --git a/bazel/rules/rules_score/src/req_coverage/BUILD b/bazel/rules/rules_score/src/req_coverage/BUILD new file mode 100644 index 00000000..d608e579 --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/BUILD @@ -0,0 +1,172 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("@pip_lobster_bazel//:requirements.bzl", lobster_requirement = "requirement") +load("@pip_tooling//:requirements.bzl", tooling_requirement = "requirement") +load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") + +# --------------------------------------------------------------------------- +# Core libraries +# --------------------------------------------------------------------------- + +py_library( + name = "read_gtest_lobster", + srcs = ["read_gtest_lobster.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [ + ":compute_lock", + "@lobster//lobster/common", + ], +) + +py_library( + name = "compute_lock", + srcs = ["compute_lock.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [lobster_requirement("pyyaml")], +) + +py_library( + name = "check_lock", + srcs = ["check_lock.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [":compute_lock"], +) + +py_library( + name = "lobster_generator", + srcs = ["lobster_generator.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [ + ":compute_lock", + "@lobster//lobster/common", + ], +) + +py_library( + name = "update_runner_lib", + srcs = ["update_runner.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [ + ":check_lock", + ":compute_lock", + ":read_gtest_lobster", + ], +) + +py_binary( + name = "update_runner", + srcs = ["update_runner.py"], + imports = [".."], + main = "update_runner.py", + visibility = ["//visibility:public"], + deps = [":update_runner_lib"], +) + +py_library( + name = "test_runner_lib", + srcs = ["test_runner.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [ + ":check_lock", + ":compute_lock", + ":lobster_generator", + ":read_gtest_lobster", + ], +) + +py_binary( + name = "test_runner", + srcs = ["test_runner.py"], + imports = [".."], + main = "test_runner.py", + visibility = ["//visibility:public"], + deps = [":test_runner_lib"], +) + +# --------------------------------------------------------------------------- +# Unit tests +# --------------------------------------------------------------------------- + +py_test( + name = "read_gtest_lobster_test", + srcs = ["read_gtest_lobster_test.py"], + imports = [".."], + deps = [ + ":read_gtest_lobster", + tooling_requirement("pytest"), + ], +) + +py_test( + name = "compute_lock_test", + srcs = ["compute_lock_test.py"], + imports = [".."], + deps = [ + ":compute_lock", + ":read_gtest_lobster", + tooling_requirement("pytest"), + ], +) + +py_test( + name = "check_lock_test", + srcs = ["check_lock_test.py"], + imports = [".."], + deps = [ + ":check_lock", + tooling_requirement("pytest"), + ], +) + +py_test( + name = "lobster_generator_test", + srcs = ["lobster_generator_test.py"], + imports = [".."], + deps = [ + ":lobster_generator", + tooling_requirement("pytest"), + ], +) + +py_test( + name = "update_runner_test", + srcs = [ + "conftest.py", + "update_runner_test.py", + ], + imports = [".."], + deps = [ + ":update_runner_lib", + tooling_requirement("pytest"), + ], +) + +py_test( + name = "test_runner_test", + srcs = [ + "conftest.py", + "test_runner_test.py", + ], + imports = [".."], + deps = [ + ":test_runner_lib", + tooling_requirement("pytest"), + ], +) diff --git a/bazel/rules/rules_score/src/req_coverage/__init__.py b/bazel/rules/rules_score/src/req_coverage/__init__.py new file mode 100644 index 00000000..4301179a --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/__init__.py @@ -0,0 +1,20 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Requirement coverage lock mechanism and traceability.""" + +__all__ = [ + "compute_lock", + "read_gtest_lobster", + "check_lock", + "lobster_generator", +] diff --git a/bazel/rules/rules_score/src/req_coverage/assets/component_diagram.puml b/bazel/rules/rules_score/src/req_coverage/assets/component_diagram.puml new file mode 100644 index 00000000..dfd440ae --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/assets/component_diagram.puml @@ -0,0 +1,81 @@ +' ******************************************************************************* +' Copyright (c) 2026 Contributors to the Eclipse Foundation +' +' See the NOTICE file(s) distributed with this work for additional +' information regarding copyright ownership. +' +' This program and the accompanying materials are made available under the +' terms of the Apache License Version 2.0 which is available at +' https://www.apache.org/licenses/LICENSE-2.0 +' +' SPDX-License-Identifier: Apache-2.0 +' ******************************************************************************* + +@startuml req_coverage Architecture +skinparam componentStyle rectangle +skinparam linetype ortho +left to right direction + +package "Inputs" { + [TRLC .lobster] as trlc + [GTest binary] as gtest_bin + [coverage.lock.yaml\n(committed)] as lock_committed +} + +package "component rule\n(component.bzl)" { + [subrule_lobster_gtest] as subrule + [_component_test\n(coverage_lock attr)] as rule_test + [_component_update\n(.update target)] as rule_update +} + +gtest_bin --> subrule : XML results +subrule --> rule_test : gtest.lobster +subrule --> rule_update : gtest.lobster +trlc --> rule_test : action manifest\n(exec paths) +trlc --> rule_update : runfiles manifest\n(short paths) + +package "Python modules\n(src/req_coverage/)" { + [read_gtest_lobster\n+ scan + resolve + req_ids] as reader + [compute_lock] as compute + [check_lock] as check + [lobster_generator] as gen +} + +package "Test phase (bazel test //...)" #LightBlue { + [test_runner] as test_r +} + +package "Run phase (bazel run .update)" #LightYellow { + [update_runner] as update_r +} + +rule_test ..> test_r : ctx.actions.run\n(--allow-check-failures) +rule_update ..> update_r : bazel run + +lock_committed --> check : committed lock +reader --> compute +compute --> check +compute --> gen +check --> test_r +gen --> test_r +reader --> update_r +compute --> update_r + +package "Outputs" { + [coverage.lobster\nartifact] as lobster_out + [coverage.lock.yaml\n(refreshed)] as lock_out +} + +test_r --> lobster_out : always emitted +update_r --> lock_out : overwrites committed + +note bottom of rule_test + User API: + component( + name = "my_comp", + coverage_lock = "coverage.lock.yaml", + ... + ) +end note + +@enduml diff --git a/bazel/rules/rules_score/src/req_coverage/check_lock.py b/bazel/rules/rules_score/src/req_coverage/check_lock.py new file mode 100644 index 00000000..db806775 --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/check_lock.py @@ -0,0 +1,140 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Compare a committed lock file against a freshly computed one.""" + +from __future__ import annotations + +from pathlib import Path + +from req_coverage.compute_lock import ( + LockFile, + RequirementEntry, + TestCase, + load_lock_file, +) + + +def _req_map_by_uid(lock: LockFile) -> dict[str, dict[str, TestCase]]: + """Index a LockFile as {req_id: {uid: TestCase}}.""" + return {req.id: {tc.uid: tc for tc in req.test_cases} for req in lock.requirements} + + +def _req_version_map(lock: LockFile) -> dict[str, str]: + """Index a LockFile as {req_id: version}.""" + return {req.id: req.version for req in lock.requirements} + + +def compare_lock_files( + committed: LockFile, + computed: LockFile, +) -> tuple[bool, list[str]]: + """Return (ok, diff_lines) comparing committed vs computed lock. + + Checks (in order per requirement): + 1. Requirement added / removed. + 2. Version changed (triggers re-approval). + 3. Test cases added / removed / GWT changed. + + Description is intentionally excluded from comparison (display only). + diff_lines is empty when ok is True. + """ + diff: list[str] = [] + + committed_map = _req_map_by_uid(committed) + computed_map = _req_map_by_uid(computed) + committed_ver = _req_version_map(committed) + computed_ver = _req_version_map(computed) + + all_req_ids = sorted(set(committed_map) | set(computed_map)) + + for req_id in all_req_ids: + if req_id not in committed_map: + diff.append(f" + requirement added: {req_id}") + continue + if req_id not in computed_map: + diff.append(f" - requirement removed: {req_id}") + continue + + # Version check — a version bump needs explicit re-approval. + comm_ver = committed_ver.get(req_id, "") + comp_ver = computed_ver.get(req_id, "") + if comm_ver != comp_ver: + diff.append(f" ~ [{req_id}] version changed: {comm_ver!r} → {comp_ver!r}") + + comm_tests = committed_map[req_id] + comp_tests = computed_map[req_id] + all_uids = sorted(set(comm_tests) | set(comp_tests)) + + for uid in all_uids: + if uid not in comm_tests: + diff.append(f" + [{req_id}] test added: {uid}") + elif uid not in comp_tests: + diff.append(f" - [{req_id}] test removed: {uid}") + else: + comm_tc = comm_tests[uid] + comp_tc = comp_tests[uid] + for field_name, comm_val, comp_val in ( + ("given", comm_tc.given, comp_tc.given), + ("when", comm_tc.when, comp_tc.when), + ("then", comm_tc.then, comp_tc.then), + ): + if comm_val != comp_val: + diff.append( + f" ~ [{req_id}] {field_name} changed: {uid}\n" + f" lockfile: {comm_val!r}\n" + f" testcase: {comp_val!r}" + ) + + return len(diff) == 0, diff + + +def validate_specs(computed: LockFile) -> tuple[bool, list[str]]: + """Fail if any test case in *computed* is missing a GWT spec. + + A missing spec means the developer did not add ``given``/``when``/``then`` + ``RecordProperty`` annotations to the test. Coverage without a spec + cannot be meaningfully reviewed or approved. + + Returns (ok, issue_lines). + """ + issues: list[str] = [] + for req in computed.requirements: + for tc in req.test_cases: + missing = [ + name + for name, val in ( + ("given", tc.given), + ("when", tc.when), + ("then", tc.then), + ) + if not val.strip() + ] + if missing: + issues.append( + f" ! [{req.id}] test missing GWT fields ({', '.join(missing)}): {tc.uid}\n" + f" Add RecordProperty('given'/'when'/'then') annotations." + ) + return len(issues) == 0, issues + + +def evaluate_lock(committed_path: Path, computed: LockFile) -> tuple[bool, list[str]]: + """Load the committed lock file and compare against *computed*. + + Returns (ok, messages). On file-not-found or parse error, returns + (False, [error_message]). + """ + try: + committed = load_lock_file(committed_path) + except ValueError as exc: + return False, [str(exc)] + return compare_lock_files(committed, computed) diff --git a/bazel/rules/rules_score/src/req_coverage/check_lock_test.py b/bazel/rules/rules_score/src/req_coverage/check_lock_test.py new file mode 100644 index 00000000..fcc1392e --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/check_lock_test.py @@ -0,0 +1,157 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from req_coverage.check_lock import compare_lock_files, validate_specs +from req_coverage.compute_lock import LockFile, RequirementEntry, TestCase + + +def _lock(*reqs): + return LockFile(schema_version=2, requirements=list(reqs)) + + +def _req(req_id, *test_cases, version="1"): + return RequirementEntry(id=req_id, version=version, test_cases=list(test_cases)) + + +def _tc(uid="Suite:TestName", given="g", when="w", then="t"): + return TestCase(uid=uid, given=given, when=when, then=then) + + +def test_identical_locks_pass(): + lock = _lock(_req("P.R", _tc())) + ok, diff = compare_lock_files(lock, lock) + assert ok + assert diff == [] + + +def test_test_case_added_fails(): + committed = _lock(_req("P.R", _tc(uid="Suite:A"))) + computed = _lock(_req("P.R", _tc(uid="Suite:A"), _tc(uid="Suite:B"))) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("Suite:B" in line and "added" in line for line in diff) + + +def test_test_case_removed_fails(): + committed = _lock(_req("P.R", _tc(uid="Suite:A"), _tc(uid="Suite:B"))) + computed = _lock(_req("P.R", _tc(uid="Suite:A"))) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("Suite:B" in line and "removed" in line for line in diff) + + +def test_spec_changed_fails(): + tc_old = TestCase(uid="Suite:A", given="g", when="w", then="old") + tc_new = TestCase(uid="Suite:A", given="g", when="w", then="new") + committed = _lock(_req("P.R", tc_old)) + computed = _lock(_req("P.R", tc_new)) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("Suite:A" in line and "changed" in line for line in diff) + + +def test_requirement_added_fails(): + committed = _lock(_req("P.A")) + computed = _lock(_req("P.A"), _req("P.B")) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("P.B" in line and "added" in line for line in diff) + + +def test_requirement_removed_fails(): + committed = _lock(_req("P.A"), _req("P.B")) + computed = _lock(_req("P.A")) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("P.B" in line and "removed" in line for line in diff) + + +def test_version_changed_fails(): + committed = _lock(_req("P.R", _tc(), version="1")) + computed = _lock(_req("P.R", _tc(), version="2")) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("version changed" in line and "P.R" in line for line in diff) + + +def test_version_unchanged_passes(): + committed = _lock(_req("P.R", _tc(), version="3")) + computed = _lock(_req("P.R", _tc(), version="3")) + ok, diff = compare_lock_files(committed, computed) + assert ok + + +def test_description_change_does_not_fail(): + """Description is display-only — a change must not affect the drift verdict.""" + committed = _lock( + RequirementEntry(id="P.R", version="1", description="old", test_cases=[_tc()]) + ) + computed = _lock( + RequirementEntry(id="P.R", version="1", description="new", test_cases=[_tc()]) + ) + ok, diff = compare_lock_files(committed, computed) + assert ok + + +def test_empty_vs_empty_passes(): + ok, diff = compare_lock_files(_lock(), _lock()) + assert ok + assert diff == [] + + +# --------------------------------------------------------------------------- +# validate_specs tests +# --------------------------------------------------------------------------- + + +def test_validate_specs_passes_with_spec(): + lock = _lock(_req("P.R", _tc())) + ok, issues = validate_specs(lock) + assert ok + assert issues == [] + + +def test_validate_specs_fails_with_empty_spec(): + lock = _lock(_req("P.R", _tc(given="", when="", then=""))) + ok, issues = validate_specs(lock) + assert not ok + assert any("Suite:TestName" in line for line in issues) + assert any("P.R" in line for line in issues) + + +def test_validate_specs_fails_with_whitespace_only_spec(): + lock = _lock(_req("P.R", _tc(given=" ", when=" ", then=" "))) + ok, issues = validate_specs(lock) + assert not ok + + +def test_validate_specs_reports_all_missing(): + lock = _lock( + _req("P.A", _tc(uid="Suite:A", given="", when="", then="")), + _req("P.B", _tc(uid="Suite:B", given="", when="", then="")), + ) + ok, issues = validate_specs(lock) + assert not ok + assert len(issues) == 2 + + +def test_validate_specs_empty_lock_passes(): + ok, issues = validate_specs(_lock()) + assert ok + assert issues == [] + + +if __name__ == "__main__": + import sys + import pytest + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/src/req_coverage/compute_lock.py b/bazel/rules/rules_score/src/req_coverage/compute_lock.py new file mode 100644 index 00000000..aabaecd1 --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/compute_lock.py @@ -0,0 +1,187 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""YAML lock-file serialization/deserialization and lock computation.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +# Version of the lock-file schema produced by this module. +SCHEMA_VERSION: int = 3 + + +@dataclass +class TestCase: + uid: str # "Suite:TestName" — matches gtest.lobster tag (without namespace) + given: str = "" # :Given: text from RecordProperty annotation + when: str = "" # :When: text + then: str = "" # :Then: text + + +@dataclass +class RequirementMeta: + """Metadata for a single requirement read from a lobster-req-trace file.""" + + id: str # noqa: A003 e.g. "SampleComponent.REQ_COMP_001" + version: str = "" # from TRLC tag @version suffix, e.g. "1" + description: str = "" # from TRLC text field; stored for display only + + +@dataclass +class RequirementEntry: + id: str # noqa: A003 + version: str = "" # checked on drift — version bump requires re-approval + description: str = "" # display only; not compared in lock checks + test_cases: list[TestCase] = field(default_factory=list) + + +@dataclass +class LockFile: + schema_version: int + requirements: list[RequirementEntry] = field(default_factory=list) + + def to_yaml(self) -> str: + """Serialize to YAML string.""" + return serialize(self) + + @staticmethod + def from_yaml(data: dict[str, Any]) -> "LockFile": + """Deserialize from YAML dict.""" + return _from_dict(data) + + +# --------------------------------------------------------------------------- +# Serialization +# --------------------------------------------------------------------------- + + +def _tc_to_dict(tc: TestCase) -> dict[str, Any]: + return { + "uid": tc.uid, + "given": tc.given, + "when": tc.when, + "then": tc.then, + } + + +def _req_to_dict(req: RequirementEntry) -> dict[str, Any]: + d: dict[str, Any] = {"id": req.id} + if req.version: + d["version"] = req.version + if req.description: + d["description"] = req.description + d["test_cases"] = [_tc_to_dict(tc) for tc in req.test_cases] + return d + + +def serialize(lock: LockFile) -> str: + """Serialize a LockFile to a YAML string.""" + data: dict[str, Any] = { + "schema_version": lock.schema_version, + "requirements": [_req_to_dict(r) for r in lock.requirements], + } + return yaml.dump( + data, default_flow_style=False, allow_unicode=True, sort_keys=False + ) + + +# --------------------------------------------------------------------------- +# Deserialization +# --------------------------------------------------------------------------- + + +def _tc_from_dict(d: dict[str, Any]) -> TestCase: + return TestCase( + uid=str(d.get("uid", "")), + given=str(d.get("given", "")), + when=str(d.get("when", "")), + then=str(d.get("then", "")), + ) + + +def _req_from_dict(d: dict[str, Any]) -> RequirementEntry: + return RequirementEntry( + id=str(d["id"]), + version=str(d.get("version", "")), + description=str(d.get("description", "")), + test_cases=[_tc_from_dict(tc) for tc in d.get("test_cases", [])], + ) + + +def _from_dict(data: dict[str, Any]) -> LockFile: + """Deserialize from YAML dict.""" + if not isinstance(data, dict): + raise ValueError("Lock file must be a YAML mapping") + + schema_version = int(data.get("schema_version", 0)) + requirements = [_req_from_dict(r) for r in data.get("requirements", [])] + return LockFile(schema_version=schema_version, requirements=requirements) + + +def deserialize(yaml_str: str) -> LockFile: + """Deserialize a YAML string into a LockFile.""" + try: + data = yaml.safe_load(yaml_str) + except yaml.YAMLError as exc: + raise ValueError(f"Invalid lock file YAML: {exc}") from exc + + return _from_dict(data) + + +def load_lock_file(path: Path) -> LockFile: + """Read and deserialize a lock file from disk.""" + try: + return deserialize(path.read_text(encoding="utf-8")) + except OSError as exc: + raise ValueError(f"Cannot read lock file {path}: {exc}") from exc + + +# --------------------------------------------------------------------------- +# Computation from scan results +# --------------------------------------------------------------------------- + + +def compute_lock( + req_metadata: list["RequirementMeta"], + test_cases_by_req: dict[str, list], # list[TestRecord] from read_gtest_lobster +) -> LockFile: + """Build a LockFile from scanned test records, sorted deterministically. + + Each RequirementEntry stores version (checked on drift) and description + (display only). Each TestCase stores uid and GWT fields. + """ + requirements: list[RequirementEntry] = [] + for meta in req_metadata: + records = test_cases_by_req.get(meta.id, []) + test_cases: list[TestCase] = [ + TestCase(uid=rec.uid, given=rec.given, when=rec.when, then=rec.then) + for rec in records + ] + # Sort lexicographically by uid for determinism + test_cases.sort(key=lambda tc: tc.uid) + requirements.append( + RequirementEntry( + id=meta.id, + version=meta.version, + description=meta.description, + test_cases=test_cases, + ) + ) + + # Sort requirements by ID for determinism + requirements.sort(key=lambda r: r.id) + return LockFile(schema_version=SCHEMA_VERSION, requirements=requirements) diff --git a/bazel/rules/rules_score/src/req_coverage/compute_lock_test.py b/bazel/rules/rules_score/src/req_coverage/compute_lock_test.py new file mode 100644 index 00000000..18440711 --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/compute_lock_test.py @@ -0,0 +1,143 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from req_coverage.compute_lock import ( + LockFile, + RequirementEntry, + RequirementMeta, + TestCase, + compute_lock, + deserialize, + serialize, +) +from req_coverage.read_gtest_lobster import TestRecord + + +def _meta(req_id: str, version: str = "1", description: str = "") -> RequirementMeta: + return RequirementMeta(id=req_id, version=version, description=description) + + +def test_testcase_uid_and_spec(): + """Test that TestCase stores uid and given/when/then correctly.""" + tc = TestCase( + uid="Suite:TestName", + given="g", + when="w", + then="t", + ) + assert tc.uid == "Suite:TestName" + assert tc.given == "g" + assert tc.when == "w" + assert tc.then == "t" + + +def test_serialize_roundtrip(): + lock = LockFile( + schema_version=3, + requirements=[ + RequirementEntry( + id="Pkg.Req", + version="2", + description="Some description", + test_cases=[ + TestCase( + uid="Suite:Case", + given="g", + when="w", + then="t", + ) + ], + ) + ], + ) + yaml_str = serialize(lock) + restored = deserialize(yaml_str) + assert restored.schema_version == 3 + assert len(restored.requirements) == 1 + req = restored.requirements[0] + assert req.id == "Pkg.Req" + assert req.version == "2" + assert req.description == "Some description" + assert len(req.test_cases) == 1 + tc = req.test_cases[0] + assert tc.uid == "Suite:Case" + assert tc.given == "g" + assert tc.when == "w" + assert tc.then == "t" + + +def test_deserialize_missing_spec_defaults_to_empty(): + yaml_str = "schema_version: 3\nrequirements:\n - id: P.R\n test_cases:\n - uid: Suite:Test\n given: ''\n when: ''\n then: ''\n" + lock = deserialize(yaml_str) + tc = lock.requirements[0].test_cases[0] + assert tc.given == "" + assert tc.when == "" + assert tc.then == "" + + +def test_deserialize_invalid_yaml_raises(): + import pytest + + with pytest.raises(ValueError, match="Invalid lock file YAML"): + deserialize("{bad: [yaml") + + +def test_compute_lock_sorts_test_cases_by_uid(): + records = { + "P.R": [ + TestRecord(uid="Suite:B", lobster_traces=["P.R"]), + TestRecord(uid="Suite:A", lobster_traces=["P.R"]), + ] + } + lock = compute_lock([_meta("P.R")], records) + uids = [tc.uid for tc in lock.requirements[0].test_cases] + assert uids == ["Suite:A", "Suite:B"] + + +def test_compute_lock_sorts_requirements(): + lock = compute_lock([_meta("P.B"), _meta("P.A")], {"P.A": [], "P.B": []}) + ids = [r.id for r in lock.requirements] + assert ids == ["P.A", "P.B"] + + +def test_compute_lock_missing_req_yields_empty_test_cases(): + lock = compute_lock([_meta("P.R")], {}) + assert lock.requirements[0].test_cases == [] + + +def test_compute_lock_builds_gwt_from_record(): + records = { + "P.R": [ + TestRecord( + uid="Suite:T", lobster_traces=["P.R"], given="g", when="w", then="t" + ) + ] + } + lock = compute_lock([_meta("P.R")], records) + tc = lock.requirements[0].test_cases[0] + assert tc.given == "g" + assert tc.when == "w" + assert tc.then == "t" + + +def test_compute_lock_stores_version_and_description(): + lock = compute_lock([_meta("P.R", version="3", description="desc")], {}) + req = lock.requirements[0] + assert req.version == "3" + assert req.description == "desc" + + +if __name__ == "__main__": + import sys + import pytest + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/src/req_coverage/conftest.py b/bazel/rules/rules_score/src/req_coverage/conftest.py new file mode 100644 index 00000000..e0de8c8d --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/conftest.py @@ -0,0 +1,79 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Shared pytest helpers for req_coverage tests.""" + +from __future__ import annotations + +from pathlib import Path + +from lobster.common.io import lobster_write +from lobster.common.items import Activity, Requirement, Tracing_Tag +from lobster.common.location import Void_Reference + +collect_ignore = ["test_runner.py", "update_runner.py"] + +_DEFAULT_SPEC = ":Given: g\n:When: w\n:Then: t" + + +def write_req_lobster( + path: Path, + req_ids: list[str], + *, + version: str = "1", + kind: str = "CompReq", + text: str = "", +) -> None: + """Write a minimal lobster-req-trace JSON file to *path*. + + Args: + req_ids: Requirement identifiers. + version: Version suffix appended to every tag (``@version``). + kind: TRLC requirement kind written to the ``kind`` field. + Defaults to ``"CompReq"``; use ``"FeatReq"`` to write + feature requirements that should be filtered out. + text: Description text written to the ``text`` field. + """ + items = [ + Requirement( + tag=Tracing_Tag("req", r, version=version), + location=Void_Reference(), + framework="TRLC", + kind=kind, + name=r, + text=text or None, + ) + for r in req_ids + ] + with path.open("w", encoding="utf-8") as fd: + lobster_write(fd, Requirement, "t", items) + + +def write_gtest_lobster( + path: Path, + uid: str, + req_id: str, + spec: str = _DEFAULT_SPEC, +) -> None: + """Write a minimal lobster-act-trace JSON file to *path*.""" + item = Activity( + tag=Tracing_Tag("gtest", uid), + location=Void_Reference(), + framework="GoogleTest", + kind="test", + text=spec or None, + status="ok", + ) + item.add_tracing_target(Tracing_Tag("req", req_id)) + with path.open("w", encoding="utf-8") as fd: + lobster_write(fd, Activity, "lobster_gtest", [item]) diff --git a/bazel/rules/rules_score/src/req_coverage/lobster_generator.py b/bazel/rules/rules_score/src/req_coverage/lobster_generator.py new file mode 100644 index 00000000..f337e2d9 --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/lobster_generator.py @@ -0,0 +1,151 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Emit a lobster-act-trace artifact using the lobster library. + +Each *requirement* in the lock file becomes one ``Activity`` item, grouping +all test cases that cover that requirement. The ``status`` field carries the +coverage verdict: + + - ``"ok"`` — every test case for this requirement matches the committed lock + - ``"fail"`` — at least one test case is new, removed, or its GWT has changed + +The artifact is always written, even when verification fails, so that the +LOBSTER dashboard remains current during CI failures (D-6). + +Reuses ``Activity``, ``Tracing_Tag``, ``Void_Reference``, and +``lobster_write`` from the lobster library (no manual JSON construction). +""" + +from __future__ import annotations + +from pathlib import Path + +from lobster.common.io import lobster_write +from lobster.common.items import Activity, Tracing_Tag +from lobster.common.location import Void_Reference + +from req_coverage.compute_lock import LockFile, TestCase + +GENERATOR = "req_coverage" + + +def _committed_index( + committed: LockFile | None, +) -> dict[str, tuple[str, dict[str, tuple[str, str, str]]]]: + """Build {req_id: (version, {uid: (given, when, then)})} from committed lock.""" + if committed is None: + return {} + return { + req.id: ( + req.version, + {tc.uid: (tc.given, tc.when, tc.then) for tc in req.test_cases}, + ) + for req in committed.requirements + } + + +def _make_activity( + req_id: str, + req_version: str, + test_cases: list[TestCase], + committed_version: str, + committed_map: dict[str, tuple[str, str, str]], + label: str, +) -> Activity: + """Build one Activity representing coverage of *req_id* by all its test cases. + + ``covered`` is True only when the committed version matches, every test + case matches the committed GWT, and no committed test has been removed. + Text concatenates the GWT block of each test case, separated by a blank line. + """ + tag = Tracing_Tag("req_coverage", req_id) + + blocks = [] + for tc in test_cases: + parts = [ + f":{k.capitalize()}: {v}" + for k, v in (("given", tc.given), ("when", tc.when), ("then", tc.then)) + if v + ] + if parts: + blocks.append("\n".join(parts)) + text = "\n\n".join(blocks) if blocks else None + + computed_uids = {tc.uid for tc in test_cases} + covered = ( + # version must match the committed approval + req_version == committed_version + # every computed test matches the committed GWT + and all( + committed_map.get(tc.uid) == (tc.given, tc.when, tc.then) + for tc in test_cases + ) + # AND no committed test has been removed from computed + and set(committed_map.keys()) <= computed_uids + ) + + item = Activity( + tag=tag, + location=Void_Reference(), + framework=GENERATOR, + kind="test", + text=text, + status="ok" if covered else "fail", + ) + item.add_tracing_target(Tracing_Tag("req", req_id)) + + if not covered: + item.messages.append("requirement coverage not confirmed") + if req_version != committed_version: + item.messages.append( + f"Requirement Version updated: {committed_version!r} → {req_version!r}" + ) + + return item + + +def generate_lobster( + computed: LockFile, + committed: LockFile | None, + label: str, + output_path: Path, +) -> None: + """Write a lobster-act-trace file for *computed*, validated against *committed*. + + Args: + computed: Freshly computed lock (from current test XML scan). + committed: Committed lock file content, or None if missing. + label: Bazel label of the req_coverage target (for context). + output_path: Where to write the .lobster JSON file. + """ + committed_index = _committed_index(committed) + items: list[Activity] = [] + + for req in computed.requirements: + if not req.test_cases: + continue + committed_version, committed_map = committed_index.get(req.id, ("", {})) + items.append( + _make_activity( + req_id=req.id, + req_version=req.version, + test_cases=req.test_cases, + committed_version=committed_version, + committed_map=committed_map, + label=label, + ) + ) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8") as fd: + lobster_write(fd, Activity, GENERATOR, items) diff --git a/bazel/rules/rules_score/src/req_coverage/lobster_generator_test.py b/bazel/rules/rules_score/src/req_coverage/lobster_generator_test.py new file mode 100644 index 00000000..80596247 --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/lobster_generator_test.py @@ -0,0 +1,192 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from pathlib import Path + +from req_coverage.compute_lock import LockFile, RequirementEntry, TestCase +from req_coverage.lobster_generator import generate_lobster + + +def _make_lock(*reqs): + return LockFile(schema_version=3, requirements=list(reqs)) + + +def _req(req_id, *tcs, version="1"): + return RequirementEntry(id=req_id, version=version, test_cases=list(tcs)) + + +def _tc(uid="Suite:A", given="g", when="w", then="t"): + return TestCase(uid=uid, given=given, when=when, then=then) + + +def test_generates_valid_lobster_json(tmp_path): + computed = _make_lock(_req("P.R", _tc())) + committed = _make_lock(_req("P.R", _tc())) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + data = json.loads(out.read_text()) + assert data["schema"] == "lobster-act-trace" + assert data["version"] == 3 + assert data["generator"] == "req_coverage" + assert len(data["data"]) == 1 + + +def test_one_activity_per_requirement(tmp_path): + """Each requirement produces exactly one Activity, even if the same test covers both.""" + tc = _tc(uid="S:T") + computed = _make_lock(_req("P.A", tc), _req("P.B", tc)) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + data = json.loads(out.read_text())["data"] + tags = [item["tag"] for item in data] + assert len(tags) == 2 + assert set(tags) == {"req_coverage P.A", "req_coverage P.B"} + + +def test_covered_test_has_status_ok(tmp_path): + computed = _make_lock(_req("P.R", _tc())) + committed = _make_lock(_req("P.R", _tc())) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "ok" + + +def test_uncovered_test_has_drift_message(tmp_path): + tc_new = TestCase(uid="Suite:A", given="g", when="w", then="new") + tc_old = TestCase(uid="Suite:A", given="g", when="w", then="old") + computed = _make_lock(_req("P.R", tc_new)) + committed = _make_lock(_req("P.R", tc_old)) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "fail" + assert any("Requirement Coverage not confirmed" in m for m in item["messages"]) + + +def test_new_test_not_in_committed_has_status_fail(tmp_path): + computed = _make_lock(_req("P.R", _tc())) + committed = _make_lock(_req("P.R")) # no test cases + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "fail" + + +def test_emitted_even_with_none_committed(tmp_path): + computed = _make_lock(_req("P.R", _tc())) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + assert out.exists() + data = json.loads(out.read_text()) + assert data["data"][0]["status"] == "fail" + + +def test_tracing_ref_points_to_requirement(tmp_path): + computed = _make_lock(_req("P.R", _tc("S.A"))) + committed = _make_lock(_req("P.R", _tc("S.A"))) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert "req P.R" in item["refs"] + + +def test_text_contains_gwt(tmp_path): + computed = _make_lock(_req("P.R", _tc(given="g1", when="w1", then="t1"))) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert "g1" in item.get("text", "") + assert "w1" in item.get("text", "") + assert "t1" in item.get("text", "") + + +def test_empty_gwt_omits_text(tmp_path): + computed = _make_lock(_req("P.R", _tc(given="", when="", then=""))) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item.get("text") is None + + +def test_requirement_with_no_test_cases_produces_no_activity(tmp_path): + computed = _make_lock(_req("P.R")) # no test cases + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + assert json.loads(out.read_text())["data"] == [] + + +def test_text_contains_all_test_gwt_blocks(tmp_path): + computed = _make_lock( + _req( + "P.R", + _tc(uid="S:A", given="g1", when="w1", then="t1"), + _tc(uid="S:B", given="g2", when="w2", then="t2"), + ) + ) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + text = json.loads(out.read_text())["data"][0].get("text", "") + assert "g1" in text and "g2" in text + + +def test_removed_test_case_causes_fail(tmp_path): + """Committed lock has 2 tests; computed has only 1 (tracing tag removed) → PARTIAL.""" + tc_a = _tc(uid="S:A") + tc_b = _tc(uid="S:B") + computed = _make_lock(_req("P.R", tc_a)) # S:B disappeared + committed = _make_lock(_req("P.R", tc_a, tc_b)) # S:B was committed + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "fail" + assert any("requirement coverage not confirmed" in m for m in item["messages"]) + + +def test_version_changed_causes_fail(tmp_path): + """Requirement version bump in computed but committed lock still has old version → fail.""" + tc = _tc() + computed = _make_lock(_req("P.R", tc, version="2")) + committed = _make_lock(_req("P.R", tc, version="1")) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "fail" + assert any("requirement coverage not confirmed" in m for m in item["messages"]) + assert any("Requirement Version updated" in m for m in item["messages"]) + + +def test_same_version_and_gwt_is_ok(tmp_path): + tc = _tc() + lock = _make_lock(_req("P.R", tc, version="3")) + out = tmp_path / "out.lobster" + generate_lobster(lock, lock, "//pkg:cov", out) + assert json.loads(out.read_text())["data"][0]["status"] == "ok" + + """Activity is ok only when every test case in the req matches committed.""" + tc_ok = _tc(uid="S:A", given="g", when="w", then="t") + tc_drift = TestCase(uid="S:B", given="g", when="w", then="new") + tc_old = TestCase(uid="S:B", given="g", when="w", then="old") + computed = _make_lock(_req("P.R", tc_ok, tc_drift)) + committed = _make_lock(_req("P.R", tc_ok, tc_old)) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "fail" + + +if __name__ == "__main__": + import sys + import pytest + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/src/req_coverage/read_gtest_lobster.py b/bazel/rules/rules_score/src/req_coverage/read_gtest_lobster.py new file mode 100644 index 00000000..1a4eb997 --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/read_gtest_lobster.py @@ -0,0 +1,238 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Read test records from lobster-gtest JSON artifacts. + +Provides all lobster-file reading utilities used by both runners: + +* :func:`read_gtest_lobster` — parse a single gtest.lobster file +* :func:`scan_gtest_lobster` — group test records by requirement ID +* :func:`read_req_metadata_from_lobster_files` — extract CompReq metadata from a manifest +* :func:`resolve_path` — resolve Bazel-relative path in action/runfiles contexts +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field, replace +from pathlib import Path + +from lobster.common.errors import LOBSTER_Error, Message_Handler +from lobster.common.io import lobster_read +from lobster.common.items import Activity, Requirement + +from req_coverage.compute_lock import RequirementMeta + + +@dataclass +class TestRecord: + """Metadata extracted from a single gtest.lobster item.""" + + uid: str # "Suite:TestName" (gtest tag without "gtest " prefix) + lobster_traces: list[str] = field(default_factory=list) # requirement IDs + given: str = "" # :Given: text from RecordProperty annotation + when: str = "" # :When: text + then: str = "" # :Then: text + + +def _parse_gwt(text: str) -> tuple[str, str, str]: + """Parse a ``:Given:/:When:/:Then:`` text field into its three components. + + lobster-gtest capitalises RecordProperty keys, so the expected format is:: + + :Given: value + :When: value + :Then: value + """ + given = when = then = "" + for line in text.splitlines(): + if line.startswith(":Given: "): + given = line[len(":Given: ") :] + elif line.startswith(":When: "): + when = line[len(":When: ") :] + elif line.startswith(":Then: "): + then = line[len(":Then: ") :] + return given, when, then + + +def read_gtest_lobster(lobster_path: Path) -> list[TestRecord]: + """Parse a single gtest.lobster JSON file and return test records. + + Only items with at least one ``req`` reference are returned; items that + carry no tracing annotation are silently skipped. + + Args: + lobster_path: Path to a ``lobster-act-trace`` JSON file produced by + ``lobster-gtest``. + + Raises: + ValueError: if the file cannot be read or parsed. + """ + mh = Message_Handler() + items: dict = {} + try: + lobster_read(mh, str(lobster_path), "act", items) + except (OSError, LOBSTER_Error) as exc: + raise ValueError(f"Cannot parse gtest.lobster {lobster_path}: {exc}") from exc + + records: list[TestRecord] = [] + for item in items.values(): + if not isinstance(item, Activity): + continue + if item.tag.namespace != "gtest": + continue + + refs = [ + ref.tag + for ref in item.unresolved_references + if ref.namespace == "req" + ] + if not refs: + continue + + gwt = _parse_gwt(item.text or "") + records.append( + TestRecord( + uid=item.tag.tag, + lobster_traces=refs, + given=gwt[0], + when=gwt[1], + then=gwt[2], + ) + ) + + return records + + +# --------------------------------------------------------------------------- +# Path resolution (action-sandbox vs runfiles contexts) +# --------------------------------------------------------------------------- + + +def resolve_path(raw: str) -> Path: + """Resolve a Bazel-relative path to an absolute filesystem path. + + Resolution order: + + 1. Absolute path that exists — returned as-is. + 2. Relative path that exists from CWD — works in Bazel action sandboxes + where CWD is the execroot and short paths are available. + 3. ``$RUNFILES_DIR/`` — used by ``bazel run`` executables whose CWD + is ``$BUILD_WORKSPACE_DIRECTORY``, not the execroot. + 4. Raw path returned unchanged as a last resort (caller will get a clear + ``FileNotFoundError`` rather than a silent wrong-path write). + """ + candidate = Path(raw) + if candidate.is_absolute() and candidate.exists(): + return candidate + if not candidate.is_absolute() and candidate.exists(): + return candidate.resolve() + runfiles_dir = os.environ.get("RUNFILES_DIR") + if runfiles_dir and not candidate.is_absolute(): + via_runfiles = Path(runfiles_dir) / raw + if via_runfiles.exists(): + return via_runfiles + return candidate + + +# --------------------------------------------------------------------------- +# Req-ID extraction from lobster-req-trace manifests +# --------------------------------------------------------------------------- + +# Only these TRLC requirement kinds belong in the coverage lock file. +# Feature requirements and assumed-system requirements are excluded — they +# are traceability targets, not directly testable component-level items. +_COMP_REQ_KINDS: frozenset[str] = frozenset({"CompReq"}) + + +def read_req_metadata_from_lobster_files( + lobster_manifest_path: Path, +) -> list[RequirementMeta]: + """Parse a single-column manifest of lobster paths and return CompReq metadata. + + Each line is a path to a ``lobster-req-trace`` JSON file. Only items + with ``kind == "CompReq"`` are included; FeatReq and AssumedSystemReq + items are silently skipped. + + Returns a deduplicated list sorted by requirement ID. Each entry carries: + + * ``id`` — requirement identifier (tag without ``@version``) + * ``version`` — version string from the ``@version`` tag suffix (e.g. ``"1"``) + * ``description`` — requirement text from the ``text`` field + """ + metadata: list[RequirementMeta] = [] + seen: set[str] = set() + + for line in lobster_manifest_path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + lobster_path = resolve_path(line) + mh = Message_Handler() + items: dict = {} + try: + lobster_read(mh, str(lobster_path), "req", items) + except (OSError, LOBSTER_Error) as exc: + raise ValueError(f"Cannot read lobster file {lobster_path}: {exc}") from exc + + for item in items.values(): + if not isinstance(item, Requirement): + continue + if item.kind not in _COMP_REQ_KINDS: + continue # skip FeatReq, AssumedSystemReq, etc. + if item.tag.namespace != "req": + continue + req_id = item.tag.tag + if req_id in seen: + continue + seen.add(req_id) + metadata.append( + RequirementMeta( + id=req_id, + version=str(item.tag.version) if item.tag.version is not None else "", + description=str(item.text or ""), + ) + ) + + return sorted(metadata, key=lambda m: m.id) + + +# --------------------------------------------------------------------------- +# Gtest scan — group records by requirement ID +# --------------------------------------------------------------------------- + + +def scan_gtest_lobster( + gtest_lobster_path: Path, + req_ids: list[str], + package: str = "", +) -> dict[str, list[TestRecord]]: + """Read a gtest.lobster file and group test records by requirement ID. + + Only records whose ``lobster_traces`` intersect with *req_ids* are kept. + A record that covers multiple requirements appears in all their lists. + + The ``package`` (e.g. ``//score/message_passing``) is prepended to each + uid to make it globally unique across components. Pass + ``"//" + ctx.label.package`` from the Bazel rule via the + ``REQ_COVERAGE_PACKAGE`` environment variable. + """ + all_records = read_gtest_lobster(gtest_lobster_path) + req_id_set = set(req_ids) + by_req: dict[str, list[TestRecord]] = {rid: [] for rid in req_ids} + for record in all_records: + if package: + record = replace(record, uid=f"{package}/{record.uid}") + for trace in record.lobster_traces: + if trace in req_id_set: + by_req[trace].append(record) + return by_req diff --git a/bazel/rules/rules_score/src/req_coverage/read_gtest_lobster_test.py b/bazel/rules/rules_score/src/req_coverage/read_gtest_lobster_test.py new file mode 100644 index 00000000..16c63559 --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/read_gtest_lobster_test.py @@ -0,0 +1,376 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from pathlib import Path + +import pytest + +from req_coverage.read_gtest_lobster import ( + TestRecord, + read_gtest_lobster, + read_gtest_lobster_files, + read_req_metadata_from_lobster_files, + resolve_path, + scan_gtest_lobster, +) + + +def _write_gtest_lobster(path: Path, items: list[tuple[str, list[str], str]]) -> Path: + """Write a minimal gtest.lobster JSON file. + + Args: + path: Directory to write the file in. + items: List of (uid, req_ids, text) tuples. + """ + data = { + "schema": "lobster-act-trace", + "version": 3, + "generator": "lobster_gtest", + "data": [ + { + "tag": f"gtest {uid}", + "location": {"kind": "void"}, + "name": uid, + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [f"req {r}" for r in req_ids], + "framework": "GoogleTest", + "kind": "test", + "text": text or None, + "status": "ok", + } + for uid, req_ids, text in items + ], + } + p = path / "gtest.lobster" + p.write_text(json.dumps(data), encoding="utf-8") + return p + + +def test_single_item_basic(tmp_path): + p = _write_gtest_lobster( + tmp_path, [("Suite:TestName", ["Req.A"], ":Given: g\n:When: w\n:Then: t")] + ) + records = read_gtest_lobster(p) + assert len(records) == 1 + r = records[0] + assert r.uid == "Suite:TestName" + assert r.lobster_traces == ["Req.A"] + assert r.given == "g" + assert r.when == "w" + assert r.then == "t" + + +def test_item_with_no_req_refs_is_skipped(tmp_path): + p = _write_gtest_lobster(tmp_path, [("Suite:TestName", [], "")]) + records = read_gtest_lobster(p) + assert records == [] + + +def test_non_gtest_tags_are_skipped(tmp_path): + data = { + "schema": "lobster-act-trace", + "version": 3, + "generator": "test", + "data": [ + { + "tag": "req SomeReq", + "location": {"kind": "void"}, + "name": "SomeReq", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [], + "framework": "trlc", + "kind": "requirement", + "text": None, + "status": "ok", + } + ], + } + p = tmp_path / "mixed.lobster" + p.write_text(json.dumps(data), encoding="utf-8") + records = read_gtest_lobster(p) + assert records == [] + + +def test_multiple_req_refs(tmp_path): + p = _write_gtest_lobster(tmp_path, [("S:T", ["Req.A", "Req.B"], "")]) + records = read_gtest_lobster(p) + assert len(records) == 1 + assert records[0].lobster_traces == ["Req.A", "Req.B"] + + +def test_empty_spec_stored_as_empty_string(tmp_path): + p = _write_gtest_lobster(tmp_path, [("S:T", ["Req.A"], "")]) + records = read_gtest_lobster(p) + assert records[0].given == "" + assert records[0].when == "" + assert records[0].then == "" + + +def test_none_text_stored_as_empty_string(tmp_path): + """Items with text=null (no GWT annotation) store given/when/then as empty strings.""" + data = { + "schema": "lobster-act-trace", + "version": 3, + "generator": "test", + "data": [ + { + "tag": "gtest S:T", + "location": {"kind": "void"}, + "name": "S:T", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": ["req Req.A"], + "framework": "GoogleTest", + "kind": "test", + "text": None, + "status": "ok", + } + ], + } + p = tmp_path / "g.lobster" + p.write_text(json.dumps(data), encoding="utf-8") + records = read_gtest_lobster(p) + assert records[0].given == "" + assert records[0].when == "" + assert records[0].then == "" + + +def test_invalid_json_raises_value_error(tmp_path): + p = tmp_path / "bad.lobster" + p.write_text("{invalid json", encoding="utf-8") + with pytest.raises(ValueError, match="Cannot parse gtest.lobster"): + read_gtest_lobster(p) + + +def test_read_gtest_lobster_files_concatenates(tmp_path): + dir_a = tmp_path / "a" + dir_a.mkdir() + p1 = _write_gtest_lobster(dir_a, [("S:T1", ["Req.A"], "")]) + dir_b = tmp_path / "b" + dir_b.mkdir() + p2 = _write_gtest_lobster(dir_b, [("S:T2", ["Req.B"], "")]) + records = read_gtest_lobster_files([p1, p2]) + uids = {r.uid for r in records} + assert uids == {"S:T1", "S:T2"} + + +# --------------------------------------------------------------------------- +# resolve_path +# --------------------------------------------------------------------------- + + +def test_resolve_path_absolute_existing(tmp_path): + f = tmp_path / "x.txt" + f.write_text("hi") + assert resolve_path(str(f)) == f + + +def test_resolve_path_relative_existing(tmp_path, monkeypatch): + f = tmp_path / "x.txt" + f.write_text("hi") + monkeypatch.chdir(tmp_path) + result = resolve_path("x.txt") + assert result.exists() + + +def test_resolve_path_nonexistent_returns_path(): + p = resolve_path("/no/such/file.lobster") + assert str(p) == "/no/such/file.lobster" + + +# --------------------------------------------------------------------------- +# read_req_ids_from_lobster_files +# --------------------------------------------------------------------------- + + +def _write_req_lobster( + path: Path, + req_ids: list[str], + kind: str = "CompReq", + version: str = "1", + text: str = "", +) -> Path: + data = { + "schema": "lobster-req-trace", + "version": 4, + "generator": "test", + "data": [ + { + "tag": f"req {rid}@{version}", + "name": rid, + "location": {"kind": "void"}, + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "TRLC", + "kind": kind, + "text": text or None, + "status": None, + } + for rid in req_ids + ], + } + p = path / "reqs.lobster" + p.write_text(json.dumps(data), encoding="utf-8") + return p + + +def _write_manifest(tmp_path: Path, lobster_path: Path) -> Path: + m = tmp_path / "manifest.txt" + m.write_text(str(lobster_path) + "\n", encoding="utf-8") + return m + + +def test_read_req_ids_basic(tmp_path): + lobster = _write_req_lobster(tmp_path, ["P.A", "P.B"]) + manifest = _write_manifest(tmp_path, lobster) + metadata = read_req_metadata_from_lobster_files(manifest) + assert [m.id for m in metadata] == ["P.A", "P.B"] + + +def test_read_req_ids_sorted(tmp_path): + lobster = _write_req_lobster(tmp_path, ["P.Z", "P.A"]) + manifest = _write_manifest(tmp_path, lobster) + assert [m.id for m in read_req_metadata_from_lobster_files(manifest)] == [ + "P.A", + "P.Z", + ] + + +def test_read_req_ids_deduplicated(tmp_path): + lobster = _write_req_lobster(tmp_path, ["P.A", "P.A"]) + manifest = _write_manifest(tmp_path, lobster) + assert len(read_req_metadata_from_lobster_files(manifest)) == 1 + + +def test_read_req_version_extracted(tmp_path): + lobster = _write_req_lobster(tmp_path, ["P.A"], version="3") + manifest = _write_manifest(tmp_path, lobster) + meta = read_req_metadata_from_lobster_files(manifest) + assert meta[0].version == "3" + + +def test_read_req_description_extracted(tmp_path): + lobster = _write_req_lobster(tmp_path, ["P.A"], text="The system shall do X.") + manifest = _write_manifest(tmp_path, lobster) + meta = read_req_metadata_from_lobster_files(manifest) + assert meta[0].description == "The system shall do X." + + +def test_feat_req_items_excluded(tmp_path): + """FeatReq items in the lobster file must not appear in the output.""" + lobster = _write_req_lobster(tmp_path, ["Feat.001"], kind="FeatReq") + manifest = _write_manifest(tmp_path, lobster) + assert read_req_metadata_from_lobster_files(manifest) == [] + + +def test_mixed_kinds_only_comp_req_returned(tmp_path): + """CompReq returned, FeatReq silently skipped.""" + import json as _json + + data = { + "schema": "lobster-req-trace", + "version": 4, + "generator": "test", + "data": [ + { + "tag": "req C.001@1", + "name": "C.001", + "location": {"kind": "void"}, + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "TRLC", + "kind": "CompReq", + "text": None, + "status": None, + }, + { + "tag": "req F.001@1", + "name": "F.001", + "location": {"kind": "void"}, + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "TRLC", + "kind": "FeatReq", + "text": None, + "status": None, + }, + ], + } + p = tmp_path / "mixed.lobster" + p.write_text(_json.dumps(data), encoding="utf-8") + manifest = tmp_path / "manifest.txt" + manifest.write_text(str(p) + "\n", encoding="utf-8") + meta = read_req_metadata_from_lobster_files(manifest) + assert [m.id for m in meta] == ["C.001"] + + +# --------------------------------------------------------------------------- +# scan_gtest_lobster +# --------------------------------------------------------------------------- + + +def test_scan_groups_by_req(tmp_path): + gtest = _write_gtest_lobster( + tmp_path, [("Suite:T", ["P.R"], ":Given: g\n:When: w\n:Then: t")] + ) + by_req = scan_gtest_lobster(gtest, ["P.R"]) + assert len(by_req["P.R"]) == 1 + rec = by_req["P.R"][0] + assert rec.given == "g" + assert rec.when == "w" + assert rec.then == "t" + + +def test_scan_uid_no_package(tmp_path): + gtest = _write_gtest_lobster(tmp_path, [("Suite:T", ["P.R"], "")]) + by_req = scan_gtest_lobster(gtest, ["P.R"]) + assert by_req["P.R"][0].uid == "Suite:T" + + +def test_scan_uid_with_package(tmp_path): + gtest = _write_gtest_lobster(tmp_path, [("Suite:T", ["P.R"], "")]) + by_req = scan_gtest_lobster(gtest, ["P.R"], package="//score/msg") + assert by_req["P.R"][0].uid == "//score/msg/Suite:T" + + +def test_scan_skips_unknown_reqs(tmp_path): + gtest = _write_gtest_lobster(tmp_path, [("Suite:T", ["P.Unknown"], "")]) + by_req = scan_gtest_lobster(gtest, ["P.R"]) + assert by_req["P.R"] == [] + + +def test_scan_multi_trace_appears_in_each(tmp_path): + gtest = _write_gtest_lobster(tmp_path, [("Suite:T", ["P.A", "P.B"], "")]) + by_req = scan_gtest_lobster(gtest, ["P.A", "P.B"]) + assert len(by_req["P.A"]) == 1 + assert len(by_req["P.B"]) == 1 + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/src/req_coverage/test_runner.py b/bazel/rules/rules_score/src/req_coverage/test_runner.py new file mode 100644 index 00000000..2482eceb --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/test_runner.py @@ -0,0 +1,135 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Test runner: verify coverage.lock.yaml is current and emit LOBSTER artifact. + +Environment variables +--------------------- +REQ_COVERAGE_LOBSTER_MANIFEST + Path to a newline-delimited file listing lobster-req-trace JSON paths. +REQ_COVERAGE_GTEST_LOBSTER + Path to the gtest.lobster file produced by subrule_lobster_gtest. +REQ_COVERAGE_LOCK_FILE + Path to the committed lock YAML (short_path in runfiles). +REQ_COVERAGE_LABEL + Bazel label of the req_coverage target. +REQ_COVERAGE_LOBSTER_OUTPUT + Path where the .lobster artifact should be written. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +from req_coverage.check_lock import evaluate_lock, validate_specs +from req_coverage.compute_lock import compute_lock, load_lock_file +from req_coverage.lobster_generator import generate_lobster +from req_coverage.read_gtest_lobster import ( + read_req_metadata_from_lobster_files, + resolve_path, + scan_gtest_lobster, +) + + +def _require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + print(f"ERROR: {name} is required", file=sys.stderr) + sys.exit(1) + return value + + +def main(argv: list[str] | None = None) -> None: + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--allow-check-failures", + action="store_true", + help="Return success even when lock checks fail (used for LOBSTER build action).", + ) + args = parser.parse_args(argv) + + lobster_manifest = resolve_path(_require_env("REQ_COVERAGE_LOBSTER_MANIFEST")) + gtest_lobster_path = resolve_path(_require_env("REQ_COVERAGE_GTEST_LOBSTER")) + committed_lock_path = resolve_path(_require_env("REQ_COVERAGE_LOCK_FILE")) + label = os.environ.get("REQ_COVERAGE_LABEL", "") + package = os.environ.get("REQ_COVERAGE_PACKAGE", "") + lobster_output_raw = os.environ.get("REQ_COVERAGE_LOBSTER_OUTPUT") + + # Step 1: extract requirement metadata (CompReq only) + req_metadata = read_req_metadata_from_lobster_files(lobster_manifest) + + req_ids = [m.id for m in req_metadata] + + # Step 2: scan gtest.lobster for test records + by_req = scan_gtest_lobster(gtest_lobster_path, req_ids, package=package) + + # Warn about requirements with no linked tests + for req_id, records in by_req.items(): + if not records: + print( + f"WARNING: [{label}] Requirement {req_id!r} has no linked test cases " + "in the scanned XML files.", + file=sys.stderr, + ) + + # Step 3: compute lock + computed = compute_lock(req_metadata, by_req) + + # Step 3b: validate GWT specs (collect, don't exit yet — artifact must be written first) + spec_ok, spec_issues = validate_specs(computed) + if not spec_ok: + print( + "ERROR: One or more test cases are missing GWT spec annotations.", + file=sys.stderr, + ) + for line in spec_issues: + print(line, file=sys.stderr) + + # Step 4: load committed lock (may not exist) + try: + committed = load_lock_file(committed_lock_path) + except ValueError: + committed = None + + # Step 5: emit LOBSTER artifact — ALWAYS, before any sys.exit, so Bazel's + # declared output is produced even when the lock check fails (D-6). + if lobster_output_raw: + generate_lobster( + computed=computed, + committed=committed, + label=label, + output_path=Path(lobster_output_raw), + ) + + # Step 6: compare + lock_ok, diff_lines = evaluate_lock(committed_lock_path, computed) + if not lock_ok: + print("ERROR: Coverage lock file is out of date.", file=sys.stderr) + for line in diff_lines: + print(line, file=sys.stderr) + print( + f"\nRun `bazel run {label}.update` to refresh the lock file.", + file=sys.stderr, + ) + + # Step 7: exit if any check failed (after artifact is written) + if not args.allow_check_failures: + if not spec_ok or not lock_ok: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/bazel/rules/rules_score/src/req_coverage/test_runner_test.py b/bazel/rules/rules_score/src/req_coverage/test_runner_test.py new file mode 100644 index 00000000..6ddaea53 --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/test_runner_test.py @@ -0,0 +1,125 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from pathlib import Path + +import pytest + +from req_coverage.compute_lock import LockFile, RequirementEntry, TestCase, serialize +from req_coverage.test_runner import main as runner_main +from req_coverage.conftest import write_req_lobster, write_gtest_lobster + +_SPEC = ":Given: g\n:When: w\n:Then: t" +_UID = "Suite:T" # raw uid in gtest.lobster tag +_LABEL = "//pkg:cov" # Bazel label set in all tests +_PACKAGE = "//pkg" # package extracted from _LABEL + + +def _setup(tmp_path: Path, req_ids, uid=_UID): + req_lobster = tmp_path / "reqs.lobster" + write_req_lobster(req_lobster, req_ids) + lm = tmp_path / "lobster_manifest.txt" + lm.write_text(str(req_lobster) + "\n") + + gtest = tmp_path / "gtest.lobster" + write_gtest_lobster(gtest, uid, req_ids[0] if req_ids else "X") + + # uid in lock must include package prefix (as scan_gtest_lobster will produce) + scoped_uid = f"{_PACKAGE}/{uid}" + lock_content = serialize( + LockFile( + schema_version=3, + requirements=[ + RequirementEntry( + id=req_ids[0] if req_ids else "X", + version="1", + test_cases=[ + TestCase(uid=scoped_uid, given="g", when="w", then="t") + ], + ) + ], + ) + ) + lock = tmp_path / "coverage.lock.yaml" + lock.write_text(lock_content) + + lobster_out = tmp_path / "out.lobster" + return lm, gtest, lock, lobster_out + + +def test_clean_pass(tmp_path, monkeypatch): + lm, gtest, lock, lobster_out = _setup(tmp_path, ["P.R"]) + monkeypatch.setenv("REQ_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("REQ_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("REQ_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("REQ_COVERAGE_LABEL", _LABEL) + monkeypatch.setenv("REQ_COVERAGE_PACKAGE", _PACKAGE) + monkeypatch.setenv("REQ_COVERAGE_LOBSTER_OUTPUT", str(lobster_out)) + runner_main([]) # must not raise + + +def test_drift_fails(tmp_path, monkeypatch): + lm, gtest, lock, lobster_out = _setup(tmp_path, ["P.R"]) + # Tamper with the gtest.lobster so spec differs from committed + write_gtest_lobster(gtest, _UID, "P.R", spec=":Given: CHANGED\n:When: w\n:Then: t") + monkeypatch.setenv("REQ_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("REQ_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("REQ_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("REQ_COVERAGE_LABEL", _LABEL) + monkeypatch.setenv("REQ_COVERAGE_PACKAGE", _PACKAGE) + monkeypatch.setenv("REQ_COVERAGE_LOBSTER_OUTPUT", str(lobster_out)) + with pytest.raises(SystemExit) as exc_info: + runner_main([]) + assert exc_info.value.code == 1 + + +def test_drift_still_emits_lobster(tmp_path, monkeypatch): + lm, gtest, lock, lobster_out = _setup(tmp_path, ["P.R"]) + write_gtest_lobster(gtest, _UID, "P.R", spec=":Given: CHANGED\n:When: w\n:Then: t") + monkeypatch.setenv("REQ_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("REQ_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("REQ_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("REQ_COVERAGE_LABEL", _LABEL) + monkeypatch.setenv("REQ_COVERAGE_PACKAGE", _PACKAGE) + monkeypatch.setenv("REQ_COVERAGE_LOBSTER_OUTPUT", str(lobster_out)) + try: + runner_main([]) + except SystemExit: + pass + assert lobster_out.exists() + data = json.loads(lobster_out.read_text()) + assert data["data"][0]["status"] == "fail" + + +def test_failure_prints_remediation(tmp_path, monkeypatch, capsys): + lm, gtest, lock, lobster_out = _setup(tmp_path, ["P.R"]) + write_gtest_lobster(gtest, _UID, "P.R", spec=":Given: CHANGED\n:When: w\n:Then: t") + monkeypatch.setenv("REQ_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("REQ_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("REQ_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("REQ_COVERAGE_LABEL", _LABEL) + monkeypatch.setenv("REQ_COVERAGE_PACKAGE", _PACKAGE) + monkeypatch.delenv("REQ_COVERAGE_LOBSTER_OUTPUT", raising=False) + try: + runner_main([]) + except SystemExit: + pass + err = capsys.readouterr().err + assert "bazel run" in err + assert ".update" in err + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/src/req_coverage/update_runner.py b/bazel/rules/rules_score/src/req_coverage/update_runner.py new file mode 100644 index 00000000..3fe51f90 --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/update_runner.py @@ -0,0 +1,116 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Update runner: refresh coverage.lock.yaml from current gtest.lobster output. + +Environment variables +--------------------- +REQ_COVERAGE_LOBSTER_MANIFEST + Path to a newline-delimited file listing lobster-req-trace JSON paths + (one per line). Requirement IDs are extracted from these files. +REQ_COVERAGE_GTEST_LOBSTER + Path to the gtest.lobster file produced by subrule_lobster_gtest. +REQ_COVERAGE_LOCK_FILE + Workspace-relative path of the committed lock YAML file to overwrite. +REQ_COVERAGE_LABEL + Bazel label of the req_coverage target (used in log output only). +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +from req_coverage.compute_lock import compute_lock, serialize +from req_coverage.check_lock import validate_specs +from req_coverage.read_gtest_lobster import ( + read_req_metadata_from_lobster_files, + resolve_path, + scan_gtest_lobster, +) + + +def _require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + print(f"ERROR: {name} is required", file=sys.stderr) + sys.exit(1) + return value + + +def main() -> None: + lobster_manifest = resolve_path(_require_env("REQ_COVERAGE_LOBSTER_MANIFEST")) + gtest_lobster_path = resolve_path(_require_env("REQ_COVERAGE_GTEST_LOBSTER")) + # The lock file must be resolved to the source tree so that ``bazel run`` + # overwrites the committed file. ``$BUILD_WORKSPACE_DIRECTORY`` is set + # by Bazel for all ``bazel run`` invocations; the env var carries the + # package-relative path (e.g. ``score/message_passing/coverage.lock.yaml``). + _raw_lock = _require_env("REQ_COVERAGE_LOCK_FILE") + build_workspace = os.environ.get("BUILD_WORKSPACE_DIRECTORY") + if build_workspace: + lock_file_path = Path(build_workspace) / _raw_lock + else: + lock_file_path = resolve_path(_raw_lock) + label = os.environ.get("REQ_COVERAGE_LABEL", "") + package = os.environ.get("REQ_COVERAGE_PACKAGE", "") + + # Step 1: extract requirement metadata from TRLC lobster output (CompReq only) + req_metadata = read_req_metadata_from_lobster_files(lobster_manifest) + if not req_metadata: + print( + f"WARNING: [{label}] No CompReq requirement IDs found in lobster manifest.", + file=sys.stderr, + ) + + req_ids = [m.id for m in req_metadata] + + # Step 2: scan gtest.lobster for test records + by_req = scan_gtest_lobster(gtest_lobster_path, req_ids, package=package) + + # Warn about requirements with no linked tests (D-7) + for req_id, records in by_req.items(): + if not records: + print( + f"WARNING: [{label}] Requirement {req_id!r} has no linked test cases " + "in the scanned XML files.", + file=sys.stderr, + ) + + # Step 3: compute lock + lock = compute_lock(req_metadata, by_req) + + # Step 3b: warn about missing GWT specs (informational on update) + spec_ok, spec_issues = validate_specs(lock) + if not spec_ok: + print( + "WARNING: One or more test cases are missing GWT spec annotations:", + file=sys.stderr, + ) + for line in spec_issues: + print(line, file=sys.stderr) + + # Step 4: write lock file + yaml_content = serialize(lock) + header = ( + f"# coverage.lock.yaml — auto-generated by req_coverage\n" + f"# Run `bazel run {label}.update` to refresh.\n" + ) + lock_file_path.write_text(header + yaml_content, encoding="utf-8") + + print(f"[{label}] Lock file updated: {lock_file_path}") + for req in lock.requirements: + print(f" {req.id}: {len(req.test_cases)} test case(s)") + + +if __name__ == "__main__": + main() diff --git a/bazel/rules/rules_score/src/req_coverage/update_runner_test.py b/bazel/rules/rules_score/src/req_coverage/update_runner_test.py new file mode 100644 index 00000000..3dc4cf85 --- /dev/null +++ b/bazel/rules/rules_score/src/req_coverage/update_runner_test.py @@ -0,0 +1,93 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from pathlib import Path + +import pytest + +from req_coverage.compute_lock import deserialize +from req_coverage.update_runner import main as update_main +from req_coverage.conftest import write_req_lobster, write_gtest_lobster + +_SPEC = ":Given: g\n:When: w\n:Then: t" +_UID = "Suite:T" + + +def _setup(tmp_path: Path, req_ids, uid=_UID, trace=None): + req_lobster = tmp_path / "reqs.lobster" + write_req_lobster(req_lobster, req_ids) + + lm = tmp_path / "lobster_manifest.txt" + lm.write_text(str(req_lobster) + "\n", encoding="utf-8") + + gtest = tmp_path / "gtest.lobster" + traced_req = trace or (req_ids[0] if req_ids else "X") + write_gtest_lobster(gtest, uid, traced_req) + + lock = tmp_path / "coverage.lock.yaml" + return lm, gtest, lock + + +def test_update_writes_lock_file(tmp_path, monkeypatch): + lm, gtest, lock = _setup(tmp_path, ["P.R"]) + monkeypatch.setenv("REQ_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("REQ_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("REQ_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("REQ_COVERAGE_LABEL", "//pkg:cov") + monkeypatch.setenv("REQ_COVERAGE_PACKAGE", "//pkg") + update_main() + assert lock.exists() + parsed = deserialize(lock.read_text()) + assert parsed.requirements[0].id == "P.R" + assert parsed.requirements[0].version == "1" + assert parsed.requirements[0].test_cases[0].uid == f"//pkg/{_UID}" + + +def test_update_warns_on_zero_linked_tests(tmp_path, monkeypatch, capsys): + lm, gtest, lock = _setup(tmp_path, ["P.R", "P.NoTest"], trace="P.R") + monkeypatch.setenv("REQ_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("REQ_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("REQ_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("REQ_COVERAGE_LABEL", "//pkg:cov") + monkeypatch.setenv("REQ_COVERAGE_PACKAGE", "//pkg") + update_main() # must not raise + captured = capsys.readouterr() + assert "P.NoTest" in captured.err + assert "WARNING" in captured.err + + +def test_update_exits_0_on_zero_linked_tests(tmp_path, monkeypatch): + lm, gtest, lock = _setup(tmp_path, ["P.NoTest"], trace="NOMATCH") + monkeypatch.setenv("REQ_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("REQ_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("REQ_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("REQ_COVERAGE_LABEL", "//pkg:cov") + monkeypatch.setenv("REQ_COVERAGE_PACKAGE", "//pkg") + update_main() # should not raise SystemExit + + +def test_update_missing_env_exits_1(monkeypatch): + for var in ( + "REQ_COVERAGE_LOBSTER_MANIFEST", + "REQ_COVERAGE_GTEST_LOBSTER", + "REQ_COVERAGE_LOCK_FILE", + ): + monkeypatch.delenv(var, raising=False) + with pytest.raises(SystemExit) as exc_info: + update_main() + assert exc_info.value.code == 1 + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/test/BUILD b/bazel/rules/rules_score/test/BUILD index e00a0c6d..516323b1 100644 --- a/bazel/rules/rules_score/test/BUILD +++ b/bazel/rules/rules_score/test/BUILD @@ -88,6 +88,7 @@ load( ) load( ":unit_component_test.bzl", + "component_coverage_lock_test", "component_excludes_feature_req_docs_test", "component_provider_test", "component_sphinx_sources_test", @@ -667,6 +668,17 @@ unit_sphinx_sources_test( target_under_test = ":test_unit", ) +# Test Component with coverage lock — verifies analysis phase succeeds when +# coverage_lock is provided. +component( + name = "test_component_with_coverage_lock", + testonly = True, + components = [":test_unit"], + coverage_lock = "fixtures/coverage_lock_test/coverage.lock.yaml", + requirements = [":comp_req"], + tests = [], +) + # Component tests component_provider_test( name = "component_provider_test", @@ -683,6 +695,11 @@ component_excludes_feature_req_docs_test( target_under_test = ":test_component", ) +component_coverage_lock_test( + name = "component_coverage_lock_test", + target_under_test = ":test_component_with_coverage_lock", +) + # Unit, Component, and Dependable Element test suite unit_component_test_suite(name = "unit_component_tests") diff --git a/bazel/rules/rules_score/test/fixtures/coverage_lock_test/coverage.lock.yaml b/bazel/rules/rules_score/test/fixtures/coverage_lock_test/coverage.lock.yaml new file mode 100644 index 00000000..efd58f06 --- /dev/null +++ b/bazel/rules/rules_score/test/fixtures/coverage_lock_test/coverage.lock.yaml @@ -0,0 +1,14 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +schema_version: 3 +requirements: [] diff --git a/bazel/rules/rules_score/test/unit_component_test.bzl b/bazel/rules/rules_score/test/unit_component_test.bzl index b51c0141..9090c9a8 100644 --- a/bazel/rules/rules_score/test/unit_component_test.bzl +++ b/bazel/rules/rules_score/test/unit_component_test.bzl @@ -178,6 +178,42 @@ def _component_excludes_feature_req_docs_test_impl(ctx): component_excludes_feature_req_docs_test = analysistest.make(_component_excludes_feature_req_docs_test_impl) +# ============================================================================ +# Coverage Lock Tests +# ============================================================================ + +def _component_coverage_lock_test_impl(ctx): + """Test that a component with coverage_lock still provides ComponentInfo and + DefaultInfo with output files (analysis phase must succeed).""" + env = analysistest.begin(ctx) + target_under_test = analysistest.target_under_test(env) + + asserts.true( + env, + ComponentInfo in target_under_test, + "Component with coverage_lock should provide ComponentInfo", + ) + + # DefaultInfo.files must contain the lobster report (non-empty). + output_files = target_under_test[DefaultInfo].files.to_list() + asserts.true( + env, + len(output_files) > 0, + "Component with coverage_lock should declare output files (expected lobster report); got: %s" % output_files, + ) + + # At least one output file should be named like the lobster report. + lobster_report_present = any([f.basename.endswith(".lobster_report") or f.extension == "html" or f.basename.endswith(".lobster") for f in output_files]) + asserts.true( + env, + lobster_report_present, + "Component with coverage_lock should declare a lobster report output; got: %s" % [f.basename for f in output_files], + ) + + return analysistest.end(env) + +component_coverage_lock_test = analysistest.make(_component_coverage_lock_test_impl) + # ============================================================================ # Dependable Element Tests # ============================================================================ @@ -202,5 +238,6 @@ def unit_component_test_suite(name): ":component_provider_test", ":component_sphinx_sources_test", ":component_excludes_feature_req_docs_test", + ":component_coverage_lock_test", ], ) diff --git a/bazel/rules/rules_score/trlc/config/BUILD b/bazel/rules/rules_score/trlc/config/BUILD index 4bbdbb0c..7b9e3853 100644 --- a/bazel/rules/rules_score/trlc/config/BUILD +++ b/bazel/rules/rules_score/trlc/config/BUILD @@ -45,7 +45,9 @@ trlc_requirements_test( trlc_requirements( name = "tools", - srcs = ["tools.trlc"], + srcs = [ + "tools.trlc", + ], spec = [":qualification_model"], visibility = ["//visibility:public"], ) diff --git a/bazel/rules/rules_score/trlc/config/tools.trlc b/bazel/rules/rules_score/trlc/config/tools.trlc index c3bf19c3..c807b458 100644 --- a/bazel/rules/rules_score/trlc/config/tools.trlc +++ b/bazel/rules/rules_score/trlc/config/tools.trlc @@ -79,4 +79,9 @@ section "Tools" { description = "Manual Analysis Framework" classification = ToolQualification.Classification.None } + + ToolQualification.Tool ReqCoverage { + description = "Requirement Coverage Lock — verifies complete test coverage of requirements via lobster-tracing annotations" + classification = ToolQualification.Classification.None + } }