Bug 2052985 - feat: add build_signing transform and artifact signing specs

src/mozilla_taskgraph/transforms/build_signing.py

@@ -0,0 +1,86 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Product-neutral build-signing transforms.
+
+These shape a signing task from its primary dependency: they derive the signing
+index routes and turn a set of signing specs into the ``upstream-artifacts``
+that the ``scriptworker-signing`` payload builder consumes.
+
+The decision of *which* artifacts and formats to sign is product-specific (it
+depends on platforms, locales, installer variants, ...), so it is NOT made
+here. An earlier, project-specific transform is expected to populate
+``job["signing-artifacts"]`` with a list of specs, each of the form::
+
+    {"paths": ["<locale>/target.zip", ...], "formats": ["...", ...]}
+"""
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.dependencies import get_primary_dependency
+
+from mozilla_taskgraph.util.attributes import copy_attributes_from_dependent_job
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def add_signed_routes(config, jobs):
+    """Mirror the primary dependency's index routes, inserting a ``signed``
+    component after the route prefix.
+
+    The prefix is read from ``graph_config`` (``scriptworker.signed-route-prefix``,
+    e.g. ``index.gecko.v2``) so no project-specific value is baked in here. A
+    project that configures no prefix simply gets no signed routes.
+    """
+    route_prefix = config.graph_config["scriptworker"].get("signed-route-prefix")
+    prefix_len = len(route_prefix.split(".")) if route_prefix else 0
+
+    for job in jobs:
+        dep_job = get_primary_dependency(config, job)
+        enable_signing_routes = job.pop("enable-signing-routes", True)
+
+        job["routes"] = []
+        if route_prefix and enable_signing_routes:
+            for dep_route in dep_job.task.get("routes", []):
+                if not dep_route.startswith(route_prefix):
+                    continue
+                parts = dep_route.split(".")
+                branch = parts[prefix_len]
+                rest = ".".join(parts[prefix_len + 1 :])
+                job["routes"].append(f"{route_prefix}.{branch}.signed.{rest}")
+
+        yield job
+
+
+def _artifact_task_type(dep_kind):
+    """Notarization dependencies run on scriptworker; everything else is a build."""
+    return "scriptworker" if "notarization" in dep_kind else "build"
+
+
+@transforms.add
+def define_upstream_artifacts(config, jobs):
+    """Copy the curated attributes from the primary dependency and shape the
+    project-provided ``signing-artifacts`` specs into ``upstream-artifacts``."""
+    for job in jobs:
+        dep_job = get_primary_dependency(config, job)
+
+        attributes = job.setdefault("attributes", {})
+        attributes.update(copy_attributes_from_dependent_job(dep_job))
+        attributes["signed"] = True
+
+        specs = job.pop("signing-artifacts", [])
+        task_ref = {"task-reference": f"<{dep_job.kind}>"}
+        task_type = _artifact_task_type(dep_job.kind)
+
+        job["upstream-artifacts"] = [
+            {
+                "taskId": task_ref,
+                "taskType": task_type,
+                "paths": spec["paths"],
+                "formats": spec["formats"],
+            }
+            for spec in specs
+        ]
+
+        yield job

src/mozilla_taskgraph/worker_types.py

@@ -167,6 +167,23 @@ class ScriptworkerSigningSchema(Schema, forbid_unknown_fields=False, kw_only=Tru
 def build_signing_payload(config, task, task_def):
     worker = task["worker"]
 
+    # Allow a project to opt out of signing formats it doesn't use (e.g. Comm
+    # doesn't sign widevine or langpacks) via graph_config, instead of each
+    # consumer post-processing the upstream-artifacts to strip them. An artifact
+    # whose formats are *emptied* by the exclusion is dropped; one that arrived
+    # with no formats is left alone, since notarization kinds rely on that.
+    excluded_formats = set(
+        config.graph_config["scriptworker"].get("excluded-signing-formats", [])
+    )
+    if excluded_formats:
+        filtered = []
+        for artifact in worker["upstream-artifacts"]:
+            original = artifact.get("formats", [])
+            formats = [f for f in original if f not in excluded_formats]
+            if formats or not original:
+                filtered.append({**artifact, "formats": formats})
+        worker["upstream-artifacts"] = filtered
+
     task_def["payload"] = {
         "upstreamArtifacts": worker["upstream-artifacts"],
     }

test/test_worker_types.py

@@ -1519,3 +1519,102 @@ def test_get_release_config_partials(make_release_config):
         partial_updates=partial_updates,
     )
     assert release_config["partial_versions"] == "70.0build1, 69.0build3"
+
+
+def test_build_signing_payload_excluded_formats(
+    make_graph_config, make_transform_config, parameters
+):
+    graph_config = make_graph_config(
+        extra_config={
+            "scriptworker": {
+                "scope-prefix": "foo",
+                "excluded-signing-formats": ["gcp_prod_autograph_widevine"],
+            }
+        },
+    )
+    worker = {
+        "implementation": "scriptworker-signing",
+        "signing-type": "release",
+        "upstream-artifacts": [
+            {
+                "taskId": "abc",
+                "taskType": "build",
+                "paths": ["foo/target.zip"],
+                "formats": [
+                    "gcp_prod_autograph_authenticode_202412",
+                    "gcp_prod_autograph_widevine",
+                ],
+            },
+            {
+                "taskId": "abc",
+                "taskType": "build",
+                "paths": ["foo/target.langpack.xpi"],
+                "formats": ["gcp_prod_autograph_widevine"],
+            },
+        ],
+    }
+    task = {"worker": worker}
+    task_def = {"tags": {}}
+    config = make_transform_config(params=parameters, graph_cfg=graph_config)
+
+    payload_builder = payload_builders["scriptworker-signing"]
+    validate_schema(payload_builder.schema, worker, "schema error")
+    payload_builder.builder(config, task, task_def)
+
+    # widevine stripped everywhere; the artifact that had *only* widevine
+    # (the langpack) is dropped. This is what lets Comm delete its
+    # remove_widevine / no_sign_langpacks cleanup transforms.
+    assert task_def["payload"]["upstreamArtifacts"] == [
+        {
+            "taskId": "abc",
+            "taskType": "build",
+            "paths": ["foo/target.zip"],
+            "formats": ["gcp_prod_autograph_authenticode_202412"],
+        }
+    ]
+
+
+def test_build_signing_payload_keeps_formatless_artifact(
+    make_graph_config, make_transform_config, parameters
+):
+    """Notarization kinds emit specs with no formats at all (see
+    gecko_taskgraph.util.signed_artifacts, where langpack_formats stays empty on
+    the notarization path). Those must survive the exclusion filter untouched --
+    only artifacts the filter itself empties get dropped.
+    """
+    graph_config = make_graph_config(
+        extra_config={
+            "scriptworker": {
+                "scope-prefix": "foo",
+                "excluded-signing-formats": ["gcp_prod_autograph_widevine"],
+            }
+        },
+    )
+    worker = {
+        "implementation": "scriptworker-signing",
+        "signing-type": "release",
+        "upstream-artifacts": [
+            {
+                "taskId": "abc",
+                "taskType": "signing",
+                "paths": ["ja-JP-mac/target.langpack.xpi"],
+                "formats": [],
+            },
+        ],
+    }
+    task = {"worker": worker}
+    task_def = {"tags": {}}
+    config = make_transform_config(params=parameters, graph_cfg=graph_config)
+
+    payload_builder = payload_builders["scriptworker-signing"]
+    validate_schema(payload_builder.schema, worker, "schema error")
+    payload_builder.builder(config, task, task_def)
+
+    assert task_def["payload"]["upstreamArtifacts"] == [
+        {
+            "taskId": "abc",
+            "taskType": "signing",
+            "paths": ["ja-JP-mac/target.langpack.xpi"],
+            "formats": [],
+        }
+    ]

test/transforms/test_build_signing.py

@@ -0,0 +1,119 @@
+from mozilla_taskgraph.transforms.build_signing import (
+    transforms as build_signing_transforms,
+)
+
+from ..conftest import make_task
+
+add_signed_routes = build_signing_transforms._transforms[0]
+define_upstream_artifacts = build_signing_transforms._transforms[1]
+
+
+def _dep(routes=None, attributes=None, kind="build"):
+    return make_task(
+        "dep-label",
+        kind=kind,
+        task_def={"routes": routes or []},
+        attributes=attributes or {},
+    )
+
+
+def _job(dep, **extra):
+    job = {"attributes": {"primary-dependency-label": dep.label}}
+    job.update(extra)
+    return job
+
+
+def test_add_signed_routes(make_transform_config, make_graph_config):
+    dep = _dep(
+        routes=[
+            "index.gecko.v2.mozilla-central.latest.firefox.win64",
+            "tc-treeherder.v2.mozilla-central.abcdef",
+        ]
+    )
+    graph_config = make_graph_config(
+        extra_config={"scriptworker": {"signed-route-prefix": "index.gecko.v2"}}
+    )
+    config = make_transform_config(
+        kind_dependencies_tasks={dep.label: dep}, graph_cfg=graph_config
+    )
+
+    [job] = list(add_signed_routes(config, [_job(dep)]))
+
+    # Only the index route is mirrored, with `.signed` inserted; the
+    # non-index route is left alone.
+    assert job["routes"] == [
+        "index.gecko.v2.mozilla-central.signed.latest.firefox.win64"
+    ]
+
+
+def test_add_signed_routes_disabled(make_transform_config, make_graph_config):
+    dep = _dep(routes=["index.gecko.v2.mozilla-central.latest.firefox.win64"])
+    graph_config = make_graph_config(
+        extra_config={"scriptworker": {"signed-route-prefix": "index.gecko.v2"}}
+    )
+    config = make_transform_config(
+        kind_dependencies_tasks={dep.label: dep}, graph_cfg=graph_config
+    )
+
+    [job] = list(
+        add_signed_routes(config, [_job(dep, **{"enable-signing-routes": False})])
+    )
+    assert job["routes"] == []
+
+
+def test_add_signed_routes_no_prefix(make_transform_config, make_graph_config):
+    dep = _dep(routes=["index.gecko.v2.mozilla-central.latest.firefox.win64"])
+    graph_config = make_graph_config(extra_config={"scriptworker": {}})
+    config = make_transform_config(
+        kind_dependencies_tasks={dep.label: dep}, graph_cfg=graph_config
+    )
+
+    [job] = list(add_signed_routes(config, [_job(dep)]))
+    assert job["routes"] == []
+
+
+def test_define_upstream_artifacts(make_transform_config):
+    dep = _dep(attributes={"build_platform": "win64-shippable", "shippable": True})
+    config = make_transform_config(kind_dependencies_tasks={dep.label: dep})
+
+    job = _job(
+        dep,
+        **{
+            "signing-artifacts": [
+                {
+                    "paths": ["public/build/target.zip"],
+                    "formats": ["autograph_authenticode"],
+                }
+            ]
+        },
+    )
+    [job] = list(define_upstream_artifacts(config, [job]))
+
+    assert job["upstream-artifacts"] == [
+        {
+            "taskId": {"task-reference": "<build>"},
+            "taskType": "build",
+            "paths": ["public/build/target.zip"],
+            "formats": ["autograph_authenticode"],
+        }
+    ]
+    # Curated attributes copied from the dependency, plus signed marker.
+    assert job["attributes"]["build_platform"] == "win64-shippable"
+    assert job["attributes"]["shippable"] is True
+    assert job["attributes"]["signed"] is True
+
+
+def test_define_upstream_artifacts_notarization_task_type(make_transform_config):
+    dep = _dep(attributes={"build_platform": "macosx64"}, kind="mac-notarization")
+    config = make_transform_config(kind_dependencies_tasks={dep.label: dep})
+
+    job = _job(
+        dep,
+        **{"signing-artifacts": [{"paths": ["a/target.dmg"], "formats": ["apple"]}]},
+    )
+    [job] = list(define_upstream_artifacts(config, [job]))
+
+    assert job["upstream-artifacts"][0]["taskType"] == "scriptworker"
+    assert job["upstream-artifacts"][0]["taskId"] == {
+        "task-reference": "<mac-notarization>"
+    }