From 6e8ce3accf9a8a2bc396b3443411b8080401d1f8 Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 6 Jul 2026 13:33:25 +0200 Subject: [PATCH] support custom specs for compilers --- docs/recipes.md | 26 +++++++++++++++- stackinator/recipe.py | 23 +++++++++----- stackinator/schema/compilers.json | 45 ++++++++++++++++++++++++--- unittests/test_recipe.py | 49 +++++++++++++++++++++++++++++- unittests/test_schema.py | 12 +++++--- unittests/yaml/compilers.full.yaml | 1 + 6 files changed, 137 insertions(+), 19 deletions(-) diff --git a/docs/recipes.md b/docs/recipes.md index e24ead8a..d1e7037c 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -120,7 +120,31 @@ The first step - building `gcc` - is required, so that the simplest stack will p * `nvhpc:version:"21.7"` generates `nvhpc@21.7 ~mpi~blas~lapack` * `llvm:version:"14"` generates `llvm@14 +clang ~gold` - * `gcc:version:"13"` generates `gcc@13 build_type=Release +profiled +strip +bootstrap` + * `gcc:version:"13"` generates `gcc@13 +bootstrap` + +The default variants can be customised by setting the optional `spec` field on a compiler, which **replaces** the default variants for that compiler: + +```yaml title="compilers.yaml with a custom gcc spec" +gcc: + version: "13" + spec: "~bootstrap+nvptx" # generates gcc@13 ~bootstrap+nvptx +llvm: + version: "16" # generates the default llvm@16 +clang ~gold +``` + +The `spec` string is appended verbatim to `name@version`. +Because it replaces the defaults, restate any default variants that should be kept. + +!!! example "`gcc` with support for openmp offload on GPU" + ```yaml title='cuda offload' + gcc: + version: "15" # compatible with cuda@13.x + speac: "+nvptx~bootstrap" + ``` + Bootstrapping needs to be explicitly disabled to compile gcc. + +!!! note + The `spec` field is ignored when `gcc:version` is `system`. ## Environments diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 78381d90..88e95811 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -497,19 +497,26 @@ def generate_compiler_specs(self, raw): gcc_version = raw["gcc"]["version"] if gcc_version == "system": + if raw["gcc"].get("spec"): + self._logger.warning("compilers.yaml: the 'spec' field is ignored when gcc version is 'system'") compilers["gcc"] = {"system": True} else: - compilers["gcc"] = {"specs": [f"gcc@{gcc_version} +bootstrap"], "version": gcc_version} - - for name, spec_template in [ - ("nvhpc", "nvhpc@{version} ~mpi~blas~lapack"), - ("llvm", "llvm@{version} +clang ~gold"), - ("llvm-amdgpu", "llvm-amdgpu@{version}"), - ("intel-oneapi-compilers", "intel-oneapi-compilers@{version}"), + suffix = raw["gcc"].get("spec") or "+bootstrap" + compilers["gcc"] = {"specs": [f"gcc@{gcc_version} {suffix}"], "version": gcc_version} + + # the default spec (variants etc) for each compiler, used when the recipe + # does not provide an explicit 'spec' field + for name, default_suffix in [ + ("nvhpc", "~mpi~blas~lapack"), + ("llvm", "+clang ~gold"), + ("llvm-amdgpu", ""), + ("intel-oneapi-compilers", ""), ]: if raw.get(name) is not None: version = raw[name]["version"] - compilers[name] = {"specs": [spec_template.format(version=version)], "version": version} + suffix = raw[name].get("spec") or default_suffix + spec = f"{name}@{version} {suffix}".strip() + compilers[name] = {"specs": [spec], "version": version} self.compilers = compilers diff --git a/stackinator/schema/compilers.json b/stackinator/schema/compilers.json index 810ad0b7..64654628 100644 --- a/stackinator/schema/compilers.json +++ b/stackinator/schema/compilers.json @@ -8,7 +8,14 @@ "gcc": { "type": "object", "properties": { - "version": {"type": "string"} + "version": {"type": "string"}, + "spec": { + "oneOf": [ + {"type": "string"}, + {"type": "null"} + ], + "default": null + } }, "additionalProperties": false, "required": ["version"] @@ -18,7 +25,14 @@ { "type": "object", "properties": { - "version": {"type": "string"} + "version": {"type": "string"}, + "spec": { + "oneOf": [ + {"type": "string"}, + {"type": "null"} + ], + "default": null + } }, "additionalProperties": false, "required": ["version"] @@ -34,7 +48,14 @@ { "type": "object", "properties": { - "version": {"type": "string"} + "version": {"type": "string"}, + "spec": { + "oneOf": [ + {"type": "string"}, + {"type": "null"} + ], + "default": null + } }, "additionalProperties": false, "required": ["version"] @@ -50,7 +71,14 @@ { "type": "object", "properties": { - "version": {"type": "string"} + "version": {"type": "string"}, + "spec": { + "oneOf": [ + {"type": "string"}, + {"type": "null"} + ], + "default": null + } }, "additionalProperties": false, "required": ["version"] @@ -66,7 +94,14 @@ { "type": "object", "properties": { - "version": {"type": "string"} + "version": {"type": "string"}, + "spec": { + "oneOf": [ + {"type": "string"}, + {"type": "null"} + ], + "default": null + } }, "additionalProperties": false, "required": ["version"] diff --git a/unittests/test_recipe.py b/unittests/test_recipe.py index e8f09211..68208155 100644 --- a/unittests/test_recipe.py +++ b/unittests/test_recipe.py @@ -1,13 +1,16 @@ +import logging + import pytest from stackinator.recipe import Recipe from stackinator.spack_util import Version -def make_recipe(commit): +def make_recipe(commit=None): """A Recipe with only the spack config populated, bypassing the heavy __init__.""" recipe = Recipe.__new__(Recipe) recipe._config = {"spack": {"commit": commit}} + recipe._logger = logging.getLogger("test_recipe") return recipe @@ -39,3 +42,47 @@ def test_find_spack_version_develop_flag(): """The --develop flag targets the develop branch of spack, now at 1.2.""" recipe = make_recipe("releases/v1.0") assert recipe.find_spack_version(develop=True) == Version(1, 2) + + +def test_generate_compiler_specs_defaults(): + """Without a 'spec' field, each compiler gets its default variants.""" + recipe = make_recipe() + recipe.generate_compiler_specs( + { + "gcc": {"version": "13", "spec": None}, + "nvhpc": {"version": "25.1", "spec": None}, + "llvm": {"version": "16", "spec": None}, + "llvm-amdgpu": {"version": "6.0", "spec": None}, + "intel-oneapi-compilers": {"version": "2024.1", "spec": None}, + } + ) + assert recipe.compilers["gcc"]["specs"] == ["gcc@13 +bootstrap"] + assert recipe.compilers["nvhpc"]["specs"] == ["nvhpc@25.1 ~mpi~blas~lapack"] + assert recipe.compilers["llvm"]["specs"] == ["llvm@16 +clang ~gold"] + assert recipe.compilers["llvm-amdgpu"]["specs"] == ["llvm-amdgpu@6.0"] + assert recipe.compilers["intel-oneapi-compilers"]["specs"] == ["intel-oneapi-compilers@2024.1"] + assert not recipe.use_system_gcc + + +def test_generate_compiler_specs_custom_spec(): + """A 'spec' field replaces the default variants for that compiler.""" + recipe = make_recipe() + recipe.generate_compiler_specs( + { + "gcc": {"version": "13", "spec": "~bootstrap+nvptx"}, + "nvhpc": {"version": "25.1", "spec": None}, + "llvm": {"version": "16", "spec": "+clang +flang ~gold"}, + } + ) + assert recipe.compilers["gcc"]["specs"] == ["gcc@13 ~bootstrap+nvptx"] + # nvhpc keeps the default variants + assert recipe.compilers["nvhpc"]["specs"] == ["nvhpc@25.1 ~mpi~blas~lapack"] + assert recipe.compilers["llvm"]["specs"] == ["llvm@16 +clang +flang ~gold"] + + +def test_generate_compiler_specs_system_gcc(): + """A custom spec is ignored when the system gcc is used.""" + recipe = make_recipe() + recipe.generate_compiler_specs({"gcc": {"version": "system", "spec": "+nvptx"}}) + assert recipe.compilers["gcc"] == {"system": True} + assert recipe.use_system_gcc diff --git a/unittests/test_schema.py b/unittests/test_schema.py index 7c6b47a5..5af36a1a 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -117,15 +117,19 @@ def test_compilers_yaml(yaml_path): with open(yaml_path / "compilers.defaults.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) schema.CompilersValidator.validate(raw) - assert raw["gcc"] == {"version": "10.2"} + assert raw["gcc"] == {"version": "10.2", "spec": None} assert raw["llvm"] is None with open(yaml_path / "compilers.full.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) schema.CompilersValidator.validate(raw) - assert raw["gcc"] == {"version": "11"} - assert raw["llvm"] == {"version": "13"} - assert raw["nvhpc"] == {"version": "25.1"} + assert raw["gcc"] == {"version": "11", "spec": "~bootstrap+nvptx"} + assert raw["llvm"] == {"version": "13", "spec": None} + assert raw["nvhpc"] == {"version": "25.1", "spec": None} + + # spec must be a string (or null) + with pytest.raises(schema.ValidationError): + schema.CompilersValidator.validate({"gcc": {"version": "13", "spec": ["+nvptx"]}}) def test_recipe_compilers_yaml(recipe_paths): diff --git a/unittests/yaml/compilers.full.yaml b/unittests/yaml/compilers.full.yaml index 7ec9bf1a..afc9ee85 100644 --- a/unittests/yaml/compilers.full.yaml +++ b/unittests/yaml/compilers.full.yaml @@ -1,5 +1,6 @@ gcc: version: "11" + spec: "~bootstrap+nvptx" llvm: version: "13" nvhpc: