Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,7 @@ bazel_dep(name = "trlc", version = "3.0.0")
bazel_dep(name = "lobster", version = "1.0.3")
git_override(
module_name = "lobster",
commit = "07382c33f95e22992e0f8788c0537ec06877e51f",
commit = "247927a438647f68c70a61f9fdde155f5aee145d",
remote = "https://github.com/bmw-software-engineering/lobster.git",
)

Expand Down
101 changes: 101 additions & 0 deletions bazel/rules/rules_score/docs/requirements/tool_requirements.trlc
Original file line number Diff line number Diff line change
Expand Up @@ -176,4 +176,105 @@ section "Tool Requirements" {

}

section "Requirement Coverage" {

ToolQualification.ToolRequirement Lock_File_As_Spec_And_Approval {
description = '''
The tool shall use one YAML lock file as both the declaration of
which requirements are claimed to be completely covered and as the
committed approval artifact.
'''
derived_from = [Declare_Complete_Coverage]
satisfied_by = Tools.ReqCoverage
}

ToolQualification.ToolRequirement Requirement_IDs_From_TRLC {
description = '''
The tool shall extract requirement identifiers automatically from
the TrlcProviderInfo provider of the requirements target attribute.
'''
derived_from = [Semi_Automated_Lock_Update, Declare_Complete_Coverage]
satisfied_by = Tools.ReqCoverage
}

ToolQualification.ToolRequirement Parse_GWT_Annotations {
description = '''
The tool shall parse GTest XML result files, extracting
RecordProperty values for the keys lobster-tracing, given, when,
and then on a per-TEST, per-TEST_F, and per-TEST_P basis. A test case
without a given, when, or then property shall be included with an
empty string for each missing field.
'''
derived_from = [Detect_Specification_Drift, Detect_Coverage_Set_Changes]
satisfied_by = Tools.ReqCoverage
}

ToolQualification.ToolRequirement Multi_Requirement_Linking {
description = '''
A test case that carries lobster-tracing values matching more than
one requirement identifier shall appear in the lock file entry of
every matched requirement.
'''
derived_from = [Declare_Complete_Coverage]
satisfied_by = Tools.ReqCoverage
}

ToolQualification.ToolRequirement Lock_File_Content {
description = '''
The lock file shall contain, per requirement: the requirement
identifier and an ordered list of linked test cases. Each test
case entry shall contain: a uid in the format
"//bazel_package/SuiteName:TestName" (package-scoped gtest tag)
and the individual given, when, and then strings as separate
fields.
'''
derived_from = [Detect_Specification_Drift, Declare_Complete_Coverage]
satisfied_by = Tools.ReqCoverage
}

ToolQualification.ToolRequirement Lock_File_Ordering {
description = '''
Test cases within a requirement entry shall be ordered
lexicographically by uid to produce a deterministic,
diff-friendly lock file.
'''
derived_from = [Detect_Specification_Drift, Detect_Coverage_Set_Changes]
satisfied_by = Tools.ReqCoverage
}

ToolQualification.ToolRequirement Update_Command {
description = '''
The .update Bazel run target shall extract requirement identifiers
from the TrlcProviderInfo target, parse GTest XML result files for
lobster-tracing-linked test cases, build the spec from
given/when/then, and overwrite the committed lock file.
'''
derived_from = [Semi_Automated_Lock_Update]
satisfied_by = Tools.ReqCoverage
}

ToolQualification.ToolRequirement Test_Command_Verification {
description = '''
The Bazel test target shall parse GTest XML result files, rebuild
the spec for each linked test, and compare uid and spec against
the committed lock file, failing if they differ. On failure it
shall print a human-readable diff showing which test cases changed
and the exact bazel run command needed to refresh the lock file.
'''
derived_from = [Detect_Specification_Drift, Detect_Coverage_Set_Changes]
satisfied_by = Tools.ReqCoverage
}

ToolQualification.ToolRequirement LOBSTER_Covered_Attribute {
description = '''
The test target shall emit a lobster-act-trace JSON artifact that
marks each linked test case as "ok" when its uid exists in the
committed lock file and its spec matches, and "fail" otherwise.
'''
derived_from = [Generate_Covered_Traceability_Artifact]
satisfied_by = Tools.ReqCoverage
}

}

}
47 changes: 47 additions & 0 deletions bazel/rules/rules_score/docs/requirements/use_cases.trlc
Original file line number Diff line number Diff line change
Expand Up @@ -173,4 +173,51 @@ section "Use Cases" {
affected_tools = [Tools.Verifier, Tools.PlantumlParser, Tools.LibClang, Tools.Bazel]
}

ToolQualification.UseCase Declare_Complete_Coverage {
description = '''
As a safety engineer I want to declare that a set of requirements
is completely covered by automated test cases so that the coverage
claim is explicit, auditable, and version-controlled.
'''
affected_tools = [Tools.ReqCoverage]
}

ToolQualification.UseCase Detect_Specification_Drift {
description = '''
In the CI pipeline I want to detect automatically when a test's
given/when/then specification has changed without re-approval so
that stale coverage claims are caught before release.
'''
affected_tools = [Tools.ReqCoverage]
}

ToolQualification.UseCase Detect_Coverage_Set_Changes {
description = '''
In the CI pipeline I want to detect when a test case is added to or
removed from the set of tests linked to a requirement so that
undeclared coverage changes cannot pass silently.
'''
affected_tools = [Tools.ReqCoverage]
}

ToolQualification.UseCase Generate_Covered_Traceability_Artifact {
description = '''
As a project integrator I want the tool to emit a LOBSTER-compatible
artifact that marks each linked test case as covered or not so that
complete requirement coverage is visible in the LOBSTER dashboard.
'''
affected_tools = [Tools.ReqCoverage, Tools.Lobster]
}

ToolQualification.UseCase Semi_Automated_Lock_Update {
description = '''
As an engineer I want a single bazel run command that reads
requirement identifiers from the TRLC requirements target,
discovers all linked test cases in the GTest XML results, and
rewrites the lock file, so that I only need to review the diff
before committing my approval.
'''
affected_tools = [Tools.ReqCoverage, Tools.Bazel]
}

}
1 change: 1 addition & 0 deletions bazel/rules/rules_score/docs/tool_reference/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,4 @@ of truth.

lobster_bazel
lobster
req_coverage
137 changes: 137 additions & 0 deletions bazel/rules/rules_score/docs/tool_reference/req_coverage.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
..
# *******************************************************************************
# Copyright (c) 2026 Contributors to the Eclipse Foundation
#
# See the NOTICE file(s) distributed with this work for additional
# information regarding copyright ownership.
#
# This program and the accompanying materials are made available under the
# terms of the Apache License Version 2.0 which is available at
# https://www.apache.org/licenses/LICENSE-2.0
#
# SPDX-License-Identifier: Apache-2.0
# *******************************************************************************

Requirement Coverage (req_coverage)
====================================

``req_coverage`` verifies that a declared set of requirements is **completely**
covered by automated test cases. Coverage is locked and confirmed by a committed YAML file;

Requirement IDs are extracted automatically and test records are read from the ``gtest.lobster`` file.

.. note::

For how to annotate tests with ``lobster-tracing`` see
:doc:`../user_guide/validation`.

The ``component`` rule integrates requirement coverage when ``coverage_lock``
is provided:

.. code-block:: starlark

component(
name = "my_component",
requirements = [":my_component_requirements"],
components = [":unit_a", ":unit_b"],
coverage_lock = "coverage.lock.yaml",
)

When the build reports a req_coverage drift failure, run (here on my_component):

.. code-block:: bash

bazel run //<pkg>:<name>.update

Two phases
----------

``req_coverage`` has two distinct operational phases:

**Test phase** — ``bazel build //...`` / ``bazel test //...``

``test_runner`` is invoked as a **build action** inside the
``dependable_element`` rule for each component that has ``coverage_lock``
set. The effective enforcement stringency is controlled by the enclosing
``dependable_element``'s ``maturity`` attribute:

* ``maturity = "development"`` — ``--allow-check-failures`` is passed so
the ``.lobster`` artifact is always produced even when coverage drifts.
Both lock-drift failures **and** missing-GWT-annotation errors are
downgraded to warnings;
* ``maturity = "release"`` — no ``--allow-check-failures``; a drift or
missing GWT annotation fails the Bazel build action directly. The
``.lobster`` artifact is physically written before the action exits, but
Bazel marks the action as failed.

**Run phase** — ``bazel run //<pkg>:<name>.update``

Computes the lock from the current ``gtest.lobster`` and **overwrites**
the committed ``coverage.lock.yaml`` in the workspace. Committing the
result constitutes approval of requirement coverage.

Architecture
------------

.. code-block:: text

bazel/rules/rules_score/src/req_coverage/
├── read_gtest_lobster.py # reads gtest.lobster; also resolve_path, scan, req_ids
├── compute_lock.py # uid/spec serialisation + YAML lock
├── check_lock.py # committed vs computed comparison
├── lobster_generator.py # emits LOBSTER activity artifact
├── update_runner.py # entry point: bazel run <component>.update
└── test_runner.py # entry point: bazel test //...

Data flow
---------

.. code-block:: text

Requirements targets (.lobster) ──► extract requirement IDs
subrule_lobster_gtest ───────────► gtest.lobster
read_gtest_lobster
(uid, req refs, spec text)
compute_lock
┌─────────┴──────────────┐
RUN phase TEST phase
(bazel run .update) (bazel test)
│ │
rewrite lock compare uid+spec
file in WS vs committed lock
lobster_generator
(ok / fail per test)

Lock file format
----------------

Auto-generated by the ``.update`` target. The engineer commits it after
reviewing ``git diff``.

.. code-block:: yaml

schema_version: 3
requirements:
- id: MessagePassing.OsIpcFaultHandling
test_cases:
- uid: "//score/message_passing/ConnectionSuite:OsIpcFaultHandlingTest"
given: a connected client
when: the OS IPC call fails
then: the client receives an error

.. list-table::
:header-rows: 1
:widths: 30 70

* - Field
- Notes
* - ``requirements[].id``
- Matches ``lobster-tracing`` values; extracted from requirements targets
* - ``test_cases[].uid``
- ``//bazel_package/SuiteName:TestName`` — package-scoped gtest tag
* - ``test_cases[].given`` / ``when`` / ``then``
- Individual GWT fields from ``RecordProperty`` annotations; any change
makes the lock stale
1 change: 1 addition & 0 deletions bazel/rules/rules_score/docs/user_guide/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ User Guide
architectural_design
unit_design
dependability_analysis
validation
73 changes: 73 additions & 0 deletions bazel/rules/rules_score/docs/user_guide/validation.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
<!-- ----------------------------------------------------------------------------
Copyright (c) 2026 Contributors to the Eclipse Foundation

See the NOTICE file(s) distributed with this work for additional
information regarding copyright ownership.

This program and the accompanying materials are made available under the
terms of the Apache License Version 2.0 which is available at
https://www.apache.org/licenses/LICENSE-2.0

SPDX-License-Identifier: Apache-2.0
----------------------------------------------------------------------------- -->

# Validation

## Unit Tests

Unit tests in `rules_score` components are written with **GoogleTest** and built
with `cc_test`. Each test case that covers a component requirement must carry
a `lobster-tracing` annotation so that the build can link the test back to the
requirement.

### Annotating Tests

Call `RecordProperty` inside the test body next to the respective code blocks:

```cpp
TEST_F(MyFixture, DoesXWhenY) {
::testing::Test::RecordProperty("lobster-tracing", "MessagePassing.OsIpcFaultHandling");
::testing::Test::RecordProperty("given", "a connected client");
::testing::Test::RecordProperty("when", "the OS IPC call fails");
::testing::Test::RecordProperty("then", "the client receives an error");
}
```

| Property | Required | Description |
|---|---|---|
| `lobster-tracing` | yes | Comma-separated requirement IDs; links the test to one or more `CompReq` records |
| `given` | no | Initial state / precondition |
| `when` | no | Action or event under test |
| `then` | no | Expected outcome |

A test without `lobster-tracing` has no traceability and is not included in
coverage tracking.

### Stating Coverage for a requirement

Coverage is declared through a committed `coverage.lock.yaml` file that lists,
per requirement, every test case (uid + given/when/then) that covers it.
Committing the file is stating the coverage claim.

In Bazel the yaml file can be linked to the `component` macro via the `coverage_lock` attribute:

```starlark
component(
name = "my_component",
requirements = [":my_component_requirements"],
components = [":unit_a", ":unit_b"],
coverage_lock = "coverage.lock.yaml",
)
```

Two complementary workflows keep the lock file in sync:

- **`bazel run …update`** — reads the current test results and **rewrites**
`coverage.lock.yaml` in the source tree.
- **`bazel test //…`** — the build action recomputes coverage from the same
test results and **compares** it against the committed lock. Any drift (new
test, removed test, changed GWT text, version bump) fails the build until the
lock is refreshed and re-committed.

For the full tool description — lock file format, update workflow, design
decisions — see {doc}`../tool_reference/req_coverage`.
Loading
Loading