diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..703cced --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Ruff format +c69bbeff1d5171e230eb069112fdb65da1476a82 diff --git a/tox.ini b/tox.ini index 21b68e1..1037ce8 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ commands = mypy pytest {posargs} ruff check + ruff format --check [gh-actions] python = diff --git a/ymmsl/__init__.py b/ymmsl/__init__.py index a864c23..369f659 100644 --- a/ymmsl/__init__.py +++ b/ymmsl/__init__.py @@ -3,6 +3,7 @@ This package contains all the classes needed to represent a yMMSL file, as well as to read and write yMMSL files. """ + from importlib.metadata import version as package_version from ymmsl.conversion.converter import DowngradeError, convert_to @@ -13,10 +14,18 @@ from ymmsl.v0_2 import Operator, Settings __version__ = package_version("ymmsl") -__author__ = 'Lourens Veen' -__email__ = 'l.veen@esciencecenter.nl' +__author__ = "Lourens Veen" +__email__ = "l.veen@esciencecenter.nl" __all__ = [ - 'convert_to', 'Document', 'dump', 'DowngradeError', 'load', 'load_as', - 'Operator', 'save', 'Settings'] + "convert_to", + "Document", + "dump", + "DowngradeError", + "load", + "load_as", + "Operator", + "save", + "Settings", +] diff --git a/ymmsl/command_line.py b/ymmsl/command_line.py index 7b6346b..5b7fe58 100644 --- a/ymmsl/command_line.py +++ b/ymmsl/command_line.py @@ -13,9 +13,14 @@ def showwarning( - message: Union[Warning, str], category: Type[Warning], filename: str, - lineno: int, file: Optional[TextIO] = None, line: Optional[str] = None) -> None: - print(f'WARNING: {message}', file=file) + message: Union[Warning, str], + category: Type[Warning], + filename: str, + lineno: int, + file: Optional[TextIO] = None, + line: Optional[str] = None, +) -> None: + print(f"WARNING: {message}", file=file) warnings.showwarning = showwarning @@ -33,24 +38,37 @@ def ymmsl() -> None: _version_tag_to_type: Dict[str, Type] = { - 'v0.1': v0_1.PartialConfiguration, - 'v0.2': v0_2.Configuration, - } + "v0.1": v0_1.PartialConfiguration, + "v0.2": v0_2.Configuration, +} -@ymmsl.command(short_help='Convert a yMMSL file to a newer version') +@ymmsl.command(short_help="Convert a yMMSL file to a newer version") @click.argument( - 'input_file', default='-', type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, - resolve_path=False, allow_dash=True)) + "input_file", + default="-", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=False, + allow_dash=True, + ), +) @click.argument( - 'output_file', required=False, type=click.Path( - file_okay=True, dir_okay=True, writable=True, resolve_path=False, - allow_dash=True)) -@click.option( - '-t', '--to', default='v0.2', help='Version to convert to, e.g. "v0.2".') -def convert( - input_file: str, output_file: Optional[str], to: str) -> None: + "output_file", + required=False, + type=click.Path( + file_okay=True, + dir_okay=True, + writable=True, + resolve_path=False, + allow_dash=True, + ), +) +@click.option("-t", "--to", default="v0.2", help='Version to convert to, e.g. "v0.2".') +def convert(input_file: str, output_file: Optional[str], to: str) -> None: """Convert a yMMSL file to a later version When upgrading in place, and/or if an output file is specified and it exists, a @@ -98,16 +116,17 @@ def convert( won't work, because the shell will open the file to do the redirect, empty it, and then run ymmsl convert, which then fails because the input is empty. """ - if input_file != '-': - print(f'Converting {input_file}') + if input_file != "-": + print(f"Converting {input_file}") if output_file is None: output_file = input_file if to not in _version_tag_to_type: click.echo( - f'Invalid version {to} specified. Supported versions are v0.1 and v0.2', - err=True) + f"Invalid version {to} specified. Supported versions are v0.1 and v0.2", + err=True, + ) exit(1) try: @@ -116,22 +135,24 @@ def convert( except DowngradeError: click.echo() click.echo( - f'It seems that {input_file} is of a newer version than the target' - f' version {to}.') + f"It seems that {input_file} is of a newer version than the target" + f" version {to}." + ) click.echo() click.echo( - 'This tool can only upgrade files, not downgrade them, so that does' - ' not work.') + "This tool can only upgrade files, not downgrade them, so that does" + " not work." + ) exit(1) - if output_file != '-' and os.path.exists(output_file): - backup_file = output_file + '.bak' + if output_file != "-" and os.path.exists(output_file): + backup_file = output_file + ".bak" if not os.path.exists(backup_file): copyfile(output_file, backup_file) - print(f'Wrote backup file {backup_file}') + print(f"Wrote backup file {backup_file}") - with click.open_file(output_file, 'w') as output_stream: + with click.open_file(output_file, "w") as output_stream: save(document, output_stream) - if input_file != '-': - print('Conversion complete') + if input_file != "-": + print("Conversion complete") diff --git a/ymmsl/conversion/convert_v0_1_to_v0_2.py b/ymmsl/conversion/convert_v0_1_to_v0_2.py index e3f5a67..159cd3a 100644 --- a/ymmsl/conversion/convert_v0_1_to_v0_2.py +++ b/ymmsl/conversion/convert_v0_1_to_v0_2.py @@ -15,19 +15,20 @@ def convert_v0_1_to_v0_2(config: v0_1.PartialConfiguration) -> v0_2.Configuratio Returns: The corresponding configuration expressed in yMMSL v0.2. """ - description = 'Please add a description' + description = "Please add a description" if config.description: description = config.description models = [convert_model(config.model)] if config.model is not None else None settings = deepcopy(config.settings) programs = [ - convert_implementation(impl) for impl in config.implementations.values()] + convert_implementation(impl) for impl in config.implementations.values() + ] if programs: warnings.warn( - 'In yMMSL v0.2 implementations have become programs, and you can now' - ' specify the ports of a program in the yMMSL description. If your' - ' program has fixed ports then you should do this, because it will make' - ' incorrect wiring easier to debug. While there, add a description too!', + "In yMMSL v0.2 implementations have become programs, and you can now" + " specify the ports of a program in the yMMSL description. If your" + " program has fixed ports then you should do this, because it will make" + " incorrect wiring easier to debug. While there, add a description too!", stacklevel=2, ) @@ -36,27 +37,40 @@ def convert_v0_1_to_v0_2(config: v0_1.PartialConfiguration) -> v0_2.Configuratio resume = deepcopy(config.resume) warnings.warn( - 'Comments can unfortunately not be read by this converter, and so have been' - ' ignored. Please copy them into an appropriate description field.', + "Comments can unfortunately not be read by this converter, and so have been" + " ignored. Please copy them into an appropriate description field.", stacklevel=2, ) return v0_2.Configuration( - description, None, models, None, settings, programs, resources, checkpoints, - resume) + description, + None, + models, + None, + settings, + programs, + resources, + checkpoints, + resume, + ) def convert_component(component: v0_1.Component) -> v0_2.Component: """Convert a v0.1 Component object to a v0.2 Component.""" ports = component.ports if component.ports else v0_1.Ports() - description = 'Please add a description' + description = "Please add a description" implementation: Optional[str] = None if component.implementation is not None: implementation = str(component.implementation) return v0_2.Component( - str(component.name), convert_ports(ports), description, implementation, - False, component.multiplicity) + str(component.name), + convert_ports(ports), + description, + implementation, + False, + component.multiplicity, + ) def convert_conduit(conduit: v0_1.Conduit) -> v0_2.Conduit: @@ -77,27 +91,31 @@ def infer_ports(components: List[v0_2.Component], conduits: List[v0_2.Conduit]) for conduit in conduits: if conduit.sending_component() == component.name: component.ports[conduit.sending_port()] = v0_2.Port( - conduit.sending_port(), v0_2.Operator.O_F, - v0_2.Timeline('')) + conduit.sending_port(), v0_2.Operator.O_F, v0_2.Timeline("") + ) if component.name not in changed_components: changed_components.append(component.name) if conduit.receiving_component() == component.name: component.ports[conduit.receiving_port()] = v0_2.Port( - conduit.receiving_port(), v0_2.Operator.F_INIT, - v0_2.Timeline('')) + conduit.receiving_port(), + v0_2.Operator.F_INIT, + v0_2.Timeline(""), + ) if component.name not in changed_components: changed_components.append(component.name) if changed_components: - ch_comp_list = '\n - ' + '\n - '.join(map(str, changed_components)) + ch_comp_list = "\n - " + "\n - ".join(map(str, changed_components)) warnings.warn( - 'In yMMSL v0.2 components are required to declare their ports. The' - ' following components did not have a ports declaration, so one has' - ' been added based on the connected conduits. THIS MAY BE WRONG,' - ' because the operators have all been set to F_INIT and O_F, while they' - ' may really be O_I or S. Please check these components and adjust' - f' as needed: {ch_comp_list}', stacklevel=4) + "In yMMSL v0.2 components are required to declare their ports. The" + " following components did not have a ports declaration, so one has" + " been added based on the connected conduits. THIS MAY BE WRONG," + " because the operators have all been set to F_INIT and O_F, while they" + " may really be O_I or S. Please check these components and adjust" + f" as needed: {ch_comp_list}", + stacklevel=4, + ) def convert_model(model: v0_1.ModelReference) -> v0_2.Model: @@ -109,14 +127,15 @@ def convert_model(model: v0_1.ModelReference) -> v0_2.Model: Returns: The corresponding configuration expressed in yMMSL v0.2. """ - description = 'Please add a description' + description = "Please add a description" if isinstance(model, v0_1.Model): components = list(map(convert_component, model.components)) conduits = list(map(convert_conduit, model.conduits)) infer_ports(components, conduits) return v0_2.Model( - str(model.name), None, description, None, components, conduits) + str(model.name), None, description, None, components, conduits + ) else: return v0_2.Model(str(model.name), None, description, None, [], []) @@ -133,7 +152,7 @@ def convert_implementation(impl: v0_1.Implementation) -> v0_2.Program: Returns: The corresponding program expressed in yMMSL v0.2. """ - description = 'Please add a description' + description = "Please add a description" base_env: Optional[v0_1.BaseEnv] = impl.base_env env: Optional[Dict[str, str]] = impl.env @@ -146,9 +165,21 @@ def convert_implementation(impl: v0_1.Implementation) -> v0_2.Program: env = None return v0_2.Program( - str(impl.name), None, description, None, base_env, impl.modules, - impl.virtual_env, env, execution_model, impl.executable, impl.args, - impl.script, impl.can_share_resources, impl.keeps_state_for_next_use) + str(impl.name), + None, + description, + None, + base_env, + impl.modules, + impl.virtual_env, + env, + execution_model, + impl.executable, + impl.args, + impl.script, + impl.can_share_resources, + impl.keeps_state_for_next_use, + ) def convert_ports(ports: v0_1.Ports) -> v0_2.Ports: @@ -161,23 +192,26 @@ def convert_ports(ports: v0_1.Ports) -> v0_2.Ports: The corresponding ports expressed in yMMSL v0.2. """ return v0_2.Ports( - list(map(str, ports.f_init)), - list(map(str, ports.o_i)), - list(map(str, ports.s)), - list(map(str, ports.o_f))) + list(map(str, ports.f_init)), + list(map(str, ports.o_i)), + list(map(str, ports.s)), + list(map(str, ports.o_f)), + ) def convert_resources( - resources: MutableMapping[v0_1.Reference, v0_1.ResourceRequirements], - models: Optional[List[v0_2.Model]] - ) -> MutableMapping[v0_2.Reference, v0_2.ResourceRequirements]: + resources: MutableMapping[v0_1.Reference, v0_1.ResourceRequirements], + models: Optional[List[v0_2.Model]], +) -> MutableMapping[v0_2.Reference, v0_2.ResourceRequirements]: if not models: warnings.warn( - 'When specifying resources in yMMSL v0.2, component names must be' - ' prefixed with the name of the top (outermost) model, e.g. as' - ' my_model.my_component rather than just my_component. This file' - ' does not contain a model, so its name cannot be added automatically.' - ' Please add the model name yourself.', stacklevel=3) + "When specifying resources in yMMSL v0.2, component names must be" + " prefixed with the name of the top (outermost) model, e.g. as" + " my_model.my_component rather than just my_component. This file" + " does not contain a model, so its name cannot be added automatically." + " Please add the model name yourself.", + stacklevel=3, + ) return deepcopy(resources) else: mname = models[0].name diff --git a/ymmsl/conversion/converter.py b/ymmsl/conversion/converter.py index c9db1d9..0a16316 100644 --- a/ymmsl/conversion/converter.py +++ b/ymmsl/conversion/converter.py @@ -14,11 +14,12 @@ class DowngradeError(RuntimeError): _converters: Dict[Type[Document], Callable] = { - v0_1.PartialConfiguration: convert_v0_1_to_v0_2, - v0_1.Configuration: convert_v0_1_to_v0_2} + v0_1.PartialConfiguration: convert_v0_1_to_v0_2, + v0_1.Configuration: convert_v0_1_to_v0_2, +} -T = TypeVar('T', bound=Document) +T = TypeVar("T", bound=Document) def convert_to(to_type: Type[T], document: Document) -> T: @@ -53,15 +54,16 @@ def convert_to(to_type: Type[T], document: Document) -> T: cur_type = type(document) if cur_type not in _config_types: - raise TypeError(f'Unsupported document type {cur_type}') + raise TypeError(f"Unsupported document type {cur_type}") if to_type not in _config_types: - raise ValueError(f'Unsupported to_type {to_type}') + raise ValueError(f"Unsupported to_type {to_type}") while cur_type != to_type: if cur_type not in _converters: raise DowngradeError( - 'The requested version is not supported. Are you trying to downgrade?') + "The requested version is not supported. Are you trying to downgrade?" + ) document = _converters[cur_type](document) cur_type = type(document) diff --git a/ymmsl/conversion/tests/conftest.py b/ymmsl/conversion/tests/conftest.py index 3dbc428..f529475 100644 --- a/ymmsl/conversion/tests/conftest.py +++ b/ymmsl/conversion/tests/conftest.py @@ -15,52 +15,64 @@ def full_config() -> v0_1.PartialConfiguration: # TODO: return a Configuration once we have everything in here components = [ - v0_1.Component( - 'macro', 'macro_impl', 10, v0_1.Ports(o_i=['out'], s=['in'])), - v0_1.Component( - 'micro', 'micro_impl', 10, v0_1.Ports(f_init=['init'], o_f=['final']))] + v0_1.Component("macro", "macro_impl", 10, v0_1.Ports(o_i=["out"], s=["in"])), + v0_1.Component( + "micro", "micro_impl", 10, v0_1.Ports(f_init=["init"], o_f=["final"]) + ), + ] conduits = [ - v0_1.Conduit('macro.out', 'micro.init'), - v0_1.Conduit('micro.final', 'macro.in')] + v0_1.Conduit("macro.out", "micro.init"), + v0_1.Conduit("micro.final", "macro.in"), + ] - model = v0_1.Model('test_model', components, conduits) - settings = v0_1.Settings({'a': 42, 'b': 'Test'}) + model = v0_1.Model("test_model", components, conduits) + settings = v0_1.Settings({"a": 42, "b": "Test"}) implementations = [ - v0_1.Implementation( - v0_1.Reference('macro_impl'), - v0_1.BaseEnv.LOGIN, ['OpenMPI/4.1.0', 'FFTW/3.1.0'], - Path('/home/user/venv'), {'ENV1': '23'}, - v0_1.ExecutionModel.DIRECT, - Path('/home/user/models/macro'), ['-a', '-b'], - None), - v0_1.Implementation( - v0_1.Reference('micro_impl'), - script=(['#!/bin/bash', '/home/user/models/micro']), - can_share_resources=False, - keeps_state_for_next_use=v0_1.KeepsStateForNextUse.HELPFUL)] + v0_1.Implementation( + v0_1.Reference("macro_impl"), + v0_1.BaseEnv.LOGIN, + ["OpenMPI/4.1.0", "FFTW/3.1.0"], + Path("/home/user/venv"), + {"ENV1": "23"}, + v0_1.ExecutionModel.DIRECT, + Path("/home/user/models/macro"), + ["-a", "-b"], + None, + ), + v0_1.Implementation( + v0_1.Reference("micro_impl"), + script=(["#!/bin/bash", "/home/user/models/micro"]), + can_share_resources=False, + keeps_state_for_next_use=v0_1.KeepsStateForNextUse.HELPFUL, + ), + ] resources = [ - v0_1.ThreadedResReq(v0_1.Reference('A'), 8), - v0_1.MPICoresResReq(v0_1.Reference('B'), 4, 8)] + v0_1.ThreadedResReq(v0_1.Reference("A"), 8), + v0_1.MPICoresResReq(v0_1.Reference("B"), 4, 8), + ] - description = 'Testing a full configuration' + description = "Testing a full configuration" checkpoints = v0_1.Checkpoints( - True, [ - v0_1.CheckpointRangeRule(0.0, 7200.0, 900.0), - v0_1.CheckpointAtRule([10.0, 4.0]), - ], - [ - v0_1.CheckpointAtRule([3.0, 14.0]), - v0_1.CheckpointRangeRule(0.0, 720.0, 90.0) - ]) + True, + [ + v0_1.CheckpointRangeRule(0.0, 7200.0, 900.0), + v0_1.CheckpointAtRule([10.0, 4.0]), + ], + [ + v0_1.CheckpointAtRule([3.0, 14.0]), + v0_1.CheckpointRangeRule(0.0, 720.0, 90.0), + ], + ) resume = { - v0_1.Reference('A'): Path('/path/to/snapshot_A.pack'), - v0_1.Reference('B'): Path('/path/to/snapshot_B.pack')} + v0_1.Reference("A"): Path("/path/to/snapshot_A.pack"), + v0_1.Reference("B"): Path("/path/to/snapshot_B.pack"), + } return v0_1.PartialConfiguration( - model, settings, implementations, resources, description, checkpoints, - resume) + model, settings, implementations, resources, description, checkpoints, resume + ) diff --git a/ymmsl/conversion/tests/test_convert_v0_1_to_v0_2.py b/ymmsl/conversion/tests/test_convert_v0_1_to_v0_2.py index 17bc736..34806db 100644 --- a/ymmsl/conversion/tests/test_convert_v0_1_to_v0_2.py +++ b/ymmsl/conversion/tests/test_convert_v0_1_to_v0_2.py @@ -12,12 +12,12 @@ Ref2 = v0_2.Reference -@pytest.mark.filterwarnings('ignore:Comments.*') -@pytest.mark.filterwarnings('ignore:When specifying resources.*') +@pytest.mark.filterwarnings("ignore:Comments.*") +@pytest.mark.filterwarnings("ignore:When specifying resources.*") def test_convert_simple_config(empty_config: v0_1.PartialConfiguration) -> None: v2 = convert_v0_1_to_v0_2(empty_config) - assert v2.description == 'Please add a description' + assert v2.description == "Please add a description" assert isinstance(v2.settings, v0_2.Settings) assert len(v2.settings) == 0 assert v2.resources == {} @@ -28,53 +28,53 @@ def test_convert_simple_config(empty_config: v0_1.PartialConfiguration) -> None: assert v2.resume == {} -@pytest.mark.filterwarnings('ignore:Comments.*') -@pytest.mark.filterwarnings('ignore:.*implementations have become programs.*') +@pytest.mark.filterwarnings("ignore:Comments.*") +@pytest.mark.filterwarnings("ignore:.*implementations have become programs.*") def test_convert_full_config(full_config: v0_1.Configuration) -> None: v2 = convert_v0_1_to_v0_2(full_config) - assert v2.description == 'Testing a full configuration' + assert v2.description == "Testing a full configuration" assert v2.settings is not full_config.settings assert len(v2.settings) == 2 - assert v2.settings['a'] == 42 - assert v2.settings['b'] == 'Test' + assert v2.settings["a"] == 42 + assert v2.settings["b"] == "Test" assert len(v2.programs) == 2 - assert Ref2('macro_impl') in v2.programs + assert Ref2("macro_impl") in v2.programs - macro_prog = v2.programs[Ref2('macro_impl')] - assert macro_prog.name == Ref2('macro_impl') + macro_prog = v2.programs[Ref2("macro_impl")] + assert macro_prog.name == Ref2("macro_impl") assert macro_prog.base_env == v0_2.BaseEnv.LOGIN - assert macro_prog.modules == ['OpenMPI/4.1.0', 'FFTW/3.1.0'] - assert macro_prog.virtual_env == Path('/home/user/venv') - assert macro_prog.env == {'ENV1': '23'} + assert macro_prog.modules == ["OpenMPI/4.1.0", "FFTW/3.1.0"] + assert macro_prog.virtual_env == Path("/home/user/venv") + assert macro_prog.env == {"ENV1": "23"} assert macro_prog.execution_model == v0_2.ExecutionModel.DIRECT - assert macro_prog.executable == Path('/home/user/models/macro') - assert macro_prog.args == ['-a', '-b'] + assert macro_prog.executable == Path("/home/user/models/macro") + assert macro_prog.args == ["-a", "-b"] assert macro_prog.script is None assert macro_prog.can_share_resources is True assert macro_prog.keeps_state_for_next_use == v0_2.KeepsStateForNextUse.NECESSARY - assert Ref2('micro_impl') in v2.programs - micro_prog = v2.programs[Ref2('micro_impl')] - assert micro_prog.name == Ref2('micro_impl') - assert micro_prog.script == '#!/bin/bash\n/home/user/models/micro\n' + assert Ref2("micro_impl") in v2.programs + micro_prog = v2.programs[Ref2("micro_impl")] + assert micro_prog.name == Ref2("micro_impl") + assert micro_prog.script == "#!/bin/bash\n/home/user/models/micro\n" assert micro_prog.can_share_resources is False assert micro_prog.keeps_state_for_next_use == v0_2.KeepsStateForNextUse.HELPFUL assert v2.resources is not full_config.resources assert len(v2.resources) == 2 - ra = v2.resources[Ref2('test_model.A')] - assert ra is not full_config.resources[Ref1('A')] + ra = v2.resources[Ref2("test_model.A")] + assert ra is not full_config.resources[Ref1("A")] assert isinstance(ra, v0_2.ThreadedResReq) - assert ra.name == 'test_model.A' + assert ra.name == "test_model.A" assert ra.threads == 8 - rb = v2.resources[Ref2('test_model.B')] - assert rb is not full_config.resources[Ref1('B')] + rb = v2.resources[Ref2("test_model.B")] + assert rb is not full_config.resources[Ref1("B")] assert isinstance(rb, v0_2.MPICoresResReq) - assert rb.name == 'test_model.B' + assert rb.name == "test_model.B" assert rb.mpi_processes == 4 assert rb.threads_per_mpi_process == 8 @@ -105,7 +105,7 @@ def test_convert_full_config(full_config: v0_1.Configuration) -> None: assert st[1].every == 90.0 assert isinstance(v2.resume, dict) - assert v2.resume[Ref2('A')] is not full_config.resume[Ref1('A')] - assert v2.resume[Ref2('A')] == Path('/path/to/snapshot_A.pack') - assert v2.resume[Ref2('B')] is not full_config.resume[Ref1('B')] - assert v2.resume[Ref2('B')] == Path('/path/to/snapshot_B.pack') + assert v2.resume[Ref2("A")] is not full_config.resume[Ref1("A")] + assert v2.resume[Ref2("A")] == Path("/path/to/snapshot_A.pack") + assert v2.resume[Ref2("B")] is not full_config.resume[Ref1("B")] + assert v2.resume[Ref2("B")] == Path("/path/to/snapshot_B.pack") diff --git a/ymmsl/conversion/tests/test_converter.py b/ymmsl/conversion/tests/test_converter.py index 18d1ff6..d733b7d 100644 --- a/ymmsl/conversion/tests/test_converter.py +++ b/ymmsl/conversion/tests/test_converter.py @@ -9,13 +9,13 @@ def test_convert_to_no_change(full_config: v0_1.PartialConfiguration) -> None: new_config = convert_to(v0_1.PartialConfiguration, full_config) assert isinstance(new_config, v0_1.PartialConfiguration) # quick sanity check, conversion checks are specific to each version - assert new_config.settings['a'] == 42 + assert new_config.settings["a"] == 42 -@pytest.mark.filterwarnings('ignore:.*specify the ports.*') -@pytest.mark.filterwarnings('ignore:Comments.*') +@pytest.mark.filterwarnings("ignore:.*specify the ports.*") +@pytest.mark.filterwarnings("ignore:Comments.*") def test_convert_to(full_config: v0_1.PartialConfiguration) -> None: new_config = convert_to(v0_2.Configuration, full_config) assert isinstance(new_config, v0_2.Configuration) # quick sanity check, conversion checks are specific to each version - assert new_config.settings['a'] == 42 + assert new_config.settings["a"] == 42 diff --git a/ymmsl/io.py b/ymmsl/io.py index 64db167..5da03c8 100644 --- a/ymmsl/io.py +++ b/ymmsl/io.py @@ -1,4 +1,5 @@ """Loading and saving functions.""" + from pathlib import Path from typing import IO, Any, Type, TypeVar, Union @@ -13,22 +14,61 @@ from ymmsl.v0_2.model import MulticastConduit as v0_2_MulticastConduit _classes = ( - Document, - v0_1.BaseEnv, v0_1.CheckpointRangeRule, v0_1.CheckpointAtRule, - v0_1.CheckpointRule, v0_1.Checkpoints, v0_1.Component, v0_1.Conduit, - v0_1.Configuration, v0_1_Document, v0_1.ExecutionModel, v0_1.Identifier, - v0_1.Implementation, v0_1.KeepsStateForNextUse, v0_1.Model, v0_1.ModelReference, - v0_1.MPICoresResReq, v0_1.MPINodesResReq, v0_1.PartialConfiguration, v0_1.Ports, - v0_1.Reference, v0_1.ResourceRequirements, v0_1.Settings, v0_1.ThreadedResReq, - v0_1_MulticastConduit, - v0_2.BaseEnv, v0_2.CheckpointRangeRule, v0_2.CheckpointAtRule, - v0_2.CheckpointRule, v0_2.Checkpoints, v0_2.Component, v0_2.Conduit, - v0_2.ConduitFilter, v0_2.Configuration, v0_2.Document, v0_2.ExecutionModel, - v0_2.Identifier, v0_2.Implementation, v0_2.ImportKind, v0_2.ImportStatement, - v0_2.KeepsStateForNextUse, v0_2.Model, v0_2.MPICoresResReq, v0_2.MPINodesResReq, - v0_2_MulticastConduit, v0_2.Ports, v0_2.Program, v0_2.Reference, - v0_2.ResourceRequirements, v0_2.SettingType, v0_2.Settings, - v0_2.SupportedSetting, v0_2.SupportedSettings, v0_2.ThreadedResReq) + Document, + v0_1.BaseEnv, + v0_1.CheckpointRangeRule, + v0_1.CheckpointAtRule, + v0_1.CheckpointRule, + v0_1.Checkpoints, + v0_1.Component, + v0_1.Conduit, + v0_1.Configuration, + v0_1_Document, + v0_1.ExecutionModel, + v0_1.Identifier, + v0_1.Implementation, + v0_1.KeepsStateForNextUse, + v0_1.Model, + v0_1.ModelReference, + v0_1.MPICoresResReq, + v0_1.MPINodesResReq, + v0_1.PartialConfiguration, + v0_1.Ports, + v0_1.Reference, + v0_1.ResourceRequirements, + v0_1.Settings, + v0_1.ThreadedResReq, + v0_1_MulticastConduit, + v0_2.BaseEnv, + v0_2.CheckpointRangeRule, + v0_2.CheckpointAtRule, + v0_2.CheckpointRule, + v0_2.Checkpoints, + v0_2.Component, + v0_2.Conduit, + v0_2.ConduitFilter, + v0_2.Configuration, + v0_2.Document, + v0_2.ExecutionModel, + v0_2.Identifier, + v0_2.Implementation, + v0_2.ImportKind, + v0_2.ImportStatement, + v0_2.KeepsStateForNextUse, + v0_2.Model, + v0_2.MPICoresResReq, + v0_2.MPINodesResReq, + v0_2_MulticastConduit, + v0_2.Ports, + v0_2.Program, + v0_2.Reference, + v0_2.ResourceRequirements, + v0_2.SettingType, + v0_2.Settings, + v0_2.SupportedSetting, + v0_2.SupportedSettings, + v0_2.ThreadedResReq, +) _load = yatiml.load_function(*_classes) # type: ignore @@ -51,7 +91,7 @@ def load(source: Union[str, Path, IO[Any]]) -> Document: return _load(source) -T = TypeVar('T', bound=Document) +T = TypeVar("T", bound=Document) def load_as(as_type: Type[T], source: Union[str, Path, IO[Any]]) -> T: @@ -105,9 +145,7 @@ def dump(config: Document) -> str: _save = yatiml.dump_function(*_classes) -def save( - config: Document, target: Union[str, Path, IO[Any]] - ) -> None: +def save(config: Document, target: Union[str, Path, IO[Any]]) -> None: """Saves a yMMSL configuration to a file. The `config` argument should be either a v0_1.PartialConfiguration, a diff --git a/ymmsl/tests/conftest.py b/ymmsl/tests/conftest.py index c5631b2..25b75cf 100644 --- a/ymmsl/tests/conftest.py +++ b/ymmsl/tests/conftest.py @@ -30,327 +30,347 @@ @pytest.fixture def test_yaml1() -> str: - text = ('ymmsl_version: v0.1\n' - 'settings:\n' - ' test_str: value\n' - ' test_int: 13\n' - ' test_list: [12.3, 1.3]\n' - ' test_list_list:\n' - ' - [1.0, 2.0]\n' - ' - [3.0, 4.0]\n' - ) + text = ( + "ymmsl_version: v0.1\n" + "settings:\n" + " test_str: value\n" + " test_int: 13\n" + " test_list: [12.3, 1.3]\n" + " test_list_list:\n" + " - [1.0, 2.0]\n" + " - [3.0, 4.0]\n" + ) return text @pytest.fixture def test_config1() -> PartialConfiguration: - settings = Settings(OrderedDict([ - ('test_str', 'value'), - ('test_int', 13), - ('test_list', [12.3, 1.3]), - ('test_list_list', [[1.0, 2.0], [3.0, 4.0]]) - ])) + settings = Settings( + OrderedDict( + [ + ("test_str", "value"), + ("test_int", 13), + ("test_list", [12.3, 1.3]), + ("test_list_list", [[1.0, 2.0], [3.0, 4.0]]), + ] + ) + ) return PartialConfiguration(None, settings) @pytest.fixture def test_yaml2() -> str: - text = ('ymmsl_version: v0.1\n' - 'model:\n' - ' name: test_model\n' - ' components:\n' - ' ic: isr2d.initial_conditions\n' - ' smc: isr2d.smc\n' - ' bf: isr2d.blood_flow\n' - ' smc2bf: isr2d.smc2bf\n' - ' bf2smc: isr2d.bf2smc\n' - ' conduits:\n' - ' ic.out: smc.initial_state\n' - ' smc.cell_positions: smc2bf.in\n' - ' smc2bf.out: bf.initial_domain\n' - ' bf.wss_out: bf2smc.in\n' - ' bf2smc.out: smc.wss_in\n') + text = ( + "ymmsl_version: v0.1\n" + "model:\n" + " name: test_model\n" + " components:\n" + " ic: isr2d.initial_conditions\n" + " smc: isr2d.smc\n" + " bf: isr2d.blood_flow\n" + " smc2bf: isr2d.smc2bf\n" + " bf2smc: isr2d.bf2smc\n" + " conduits:\n" + " ic.out: smc.initial_state\n" + " smc.cell_positions: smc2bf.in\n" + " smc2bf.out: bf.initial_domain\n" + " bf.wss_out: bf2smc.in\n" + " bf2smc.out: smc.wss_in\n" + ) return text @pytest.fixture def test_config2() -> PartialConfiguration: model = Model( - 'test_model', - [ - Component('ic', 'isr2d.initial_conditions'), - Component('smc', 'isr2d.smc'), - Component('bf', 'isr2d.blood_flow'), - Component('smc2bf', 'isr2d.smc2bf'), - Component('bf2smc', 'isr2d.bf2smc')], - [ - Conduit('ic.out', 'smc.initial_state'), - Conduit('smc.cell_positions', 'smc2bf.in'), - Conduit('smc2bf.out', 'bf.initial_domain'), - Conduit('bf.wss_out', 'bf2smc.in'), - Conduit('bf2smc.out', 'smc.wss_in')]) + "test_model", + [ + Component("ic", "isr2d.initial_conditions"), + Component("smc", "isr2d.smc"), + Component("bf", "isr2d.blood_flow"), + Component("smc2bf", "isr2d.smc2bf"), + Component("bf2smc", "isr2d.bf2smc"), + ], + [ + Conduit("ic.out", "smc.initial_state"), + Conduit("smc.cell_positions", "smc2bf.in"), + Conduit("smc2bf.out", "bf.initial_domain"), + Conduit("bf.wss_out", "bf2smc.in"), + Conduit("bf2smc.out", "smc.wss_in"), + ], + ) return PartialConfiguration(model) @pytest.fixture def test_yaml3() -> str: - text = ('ymmsl_version: v0.1\n' - 'model:\n' - ' name: test_model\n') + text = "ymmsl_version: v0.1\nmodel:\n name: test_model\n" return text @pytest.fixture def test_config3() -> PartialConfiguration: - model = ModelReference('test_model') + model = ModelReference("test_model") return PartialConfiguration(model) @pytest.fixture def test_yaml4() -> str: - text = ('ymmsl_version: v0.1\n' - 'implementations:\n' - ' isr2d.initial_conditions: isr2d/bin/ic\n' - ' isr2d.smc: isr2d/bin/smc\n' - ' isr2d.blood_flow: isr2d/bin/bf\n' - ' isr2d.smc2bf: isr2d/bin/smc2bf.py\n' - ' isr2d.bf2smc: isr2d/bin/bf2smc.py\n' - 'resources:\n' - ' ic:\n' - ' threads: 4\n' - ' smc:\n' - ' threads: 4\n' - ' bf:\n' - ' mpi_processes: 4\n' - ' smc2bf:\n' - ' threads: 1\n' - ' bf2smc:\n' - ' threads: 1\n' - 'description: |-\n' - ' Multiline description for\n' - ' this workflow\n' - 'checkpoints:\n' - ' at_end: true\n' - ' wallclock_time:\n' - ' - every: 100\n' - ' - at:\n' - ' - 10\n' - ' - 20\n' - ' - 50\n' - ' simulation_time:\n' - ' - start: 0\n' - ' stop: 10\n' - ' every: 2\n' - ' - start: 10\n' - ' every: 5\n' - 'resume:\n' - ' ic: /path/to/snapshots/ic.pack\n' - ' smc: /path/to/snapshots/smc.pack\n' - ' bf: /path/to/snapshots/bf.pack\n' - ' smc2bf: /path/to/snapshots/smc2bf.pack\n' - ' bf2smc: /path/to/snapshots/bf2smc.pack\n') + text = ( + "ymmsl_version: v0.1\n" + "implementations:\n" + " isr2d.initial_conditions: isr2d/bin/ic\n" + " isr2d.smc: isr2d/bin/smc\n" + " isr2d.blood_flow: isr2d/bin/bf\n" + " isr2d.smc2bf: isr2d/bin/smc2bf.py\n" + " isr2d.bf2smc: isr2d/bin/bf2smc.py\n" + "resources:\n" + " ic:\n" + " threads: 4\n" + " smc:\n" + " threads: 4\n" + " bf:\n" + " mpi_processes: 4\n" + " smc2bf:\n" + " threads: 1\n" + " bf2smc:\n" + " threads: 1\n" + "description: |-\n" + " Multiline description for\n" + " this workflow\n" + "checkpoints:\n" + " at_end: true\n" + " wallclock_time:\n" + " - every: 100\n" + " - at:\n" + " - 10\n" + " - 20\n" + " - 50\n" + " simulation_time:\n" + " - start: 0\n" + " stop: 10\n" + " every: 2\n" + " - start: 10\n" + " every: 5\n" + "resume:\n" + " ic: /path/to/snapshots/ic.pack\n" + " smc: /path/to/snapshots/smc.pack\n" + " bf: /path/to/snapshots/bf.pack\n" + " smc2bf: /path/to/snapshots/smc2bf.pack\n" + " bf2smc: /path/to/snapshots/bf2smc.pack\n" + ) return text @pytest.fixture def test_config4() -> PartialConfiguration: implementations = [ - Implementation( - Reference('isr2d.initial_conditions'), script='isr2d/bin/ic'), - Implementation( - Reference('isr2d.smc'), script='isr2d/bin/smc'), - Implementation( - Reference('isr2d.blood_flow'), script='isr2d/bin/bf'), - Implementation( - Reference('isr2d.smc2bf'), script='isr2d/bin/smc2bf.py'), - Implementation( - Reference('isr2d.bf2smc'), script='isr2d/bin/bf2smc.py')] + Implementation(Reference("isr2d.initial_conditions"), script="isr2d/bin/ic"), + Implementation(Reference("isr2d.smc"), script="isr2d/bin/smc"), + Implementation(Reference("isr2d.blood_flow"), script="isr2d/bin/bf"), + Implementation(Reference("isr2d.smc2bf"), script="isr2d/bin/smc2bf.py"), + Implementation(Reference("isr2d.bf2smc"), script="isr2d/bin/bf2smc.py"), + ] resources = [ - ThreadedResReq(Reference('ic'), 4), - ThreadedResReq(Reference('smc'), 4), - MPICoresResReq(Reference('bf'), 4), - ThreadedResReq(Reference('smc2bf'), 1), - ThreadedResReq(Reference('bf2smc'), 1)] + ThreadedResReq(Reference("ic"), 4), + ThreadedResReq(Reference("smc"), 4), + MPICoresResReq(Reference("bf"), 4), + ThreadedResReq(Reference("smc2bf"), 1), + ThreadedResReq(Reference("bf2smc"), 1), + ] description = "Multiline description for\nthis workflow" checkpoints = Checkpoints( - True, - [CheckpointRangeRule(every=100), - CheckpointAtRule([10, 20, 50])], - [CheckpointRangeRule(start=0, stop=10, every=2), - CheckpointRangeRule(start=10, every=5)]) - resume = {Ref('ic'): Path('/path/to/snapshots/ic.pack'), - Ref('smc'): Path('/path/to/snapshots/smc.pack'), - Ref('bf'): Path('/path/to/snapshots/bf.pack'), - Ref('smc2bf'): Path('/path/to/snapshots/smc2bf.pack'), - Ref('bf2smc'): Path('/path/to/snapshots/bf2smc.pack')} + True, + [CheckpointRangeRule(every=100), CheckpointAtRule([10, 20, 50])], + [ + CheckpointRangeRule(start=0, stop=10, every=2), + CheckpointRangeRule(start=10, every=5), + ], + ) + resume = { + Ref("ic"): Path("/path/to/snapshots/ic.pack"), + Ref("smc"): Path("/path/to/snapshots/smc.pack"), + Ref("bf"): Path("/path/to/snapshots/bf.pack"), + Ref("smc2bf"): Path("/path/to/snapshots/smc2bf.pack"), + Ref("bf2smc"): Path("/path/to/snapshots/bf2smc.pack"), + } - return PartialConfiguration(None, None, implementations, resources, - description, checkpoints, resume) + return PartialConfiguration( + None, None, implementations, resources, description, checkpoints, resume + ) @pytest.fixture def test_yaml5() -> str: - text = ('ymmsl_version: v0.1\n' - 'model:\n' - ' name: test_model\n' - ' components:\n' - ' ic: isr2d.initial_conditions\n' - ' smc: isr2d.smc\n' - ' bf: isr2d.blood_flow\n' - ' smc2bf: isr2d.smc2bf\n' - ' bf2smc: isr2d.bf2smc\n' - ' conduits:\n' - ' ic.out: smc.initial_state\n' - ' smc.cell_positions: smc2bf.in\n' - ' smc2bf.out: bf.initial_domain\n' - ' bf.wss_out: bf2smc.in\n' - ' bf2smc.out: smc.wss_in\n' - 'implementations:\n' - ' isr2d.initial_conditions: isr2d/bin/ic\n' - ' isr2d.smc: isr2d/bin/smc\n' - ' isr2d.blood_flow: isr2d/bin/bf\n' - ' isr2d.smc2bf: isr2d/bin/smc2bf.py\n' - ' isr2d.bf2smc: isr2d/bin/bf2smc.py\n' - 'resources:\n' - ' ic:\n' - ' threads: 4\n' - ' smc:\n' - ' threads: 4\n' - ' bf:\n' - ' mpi_processes: 4\n' - ' smc2bf:\n' - ' threads: 1\n' - ' bf2smc:\n' - ' threads: 1\n') + text = ( + "ymmsl_version: v0.1\n" + "model:\n" + " name: test_model\n" + " components:\n" + " ic: isr2d.initial_conditions\n" + " smc: isr2d.smc\n" + " bf: isr2d.blood_flow\n" + " smc2bf: isr2d.smc2bf\n" + " bf2smc: isr2d.bf2smc\n" + " conduits:\n" + " ic.out: smc.initial_state\n" + " smc.cell_positions: smc2bf.in\n" + " smc2bf.out: bf.initial_domain\n" + " bf.wss_out: bf2smc.in\n" + " bf2smc.out: smc.wss_in\n" + "implementations:\n" + " isr2d.initial_conditions: isr2d/bin/ic\n" + " isr2d.smc: isr2d/bin/smc\n" + " isr2d.blood_flow: isr2d/bin/bf\n" + " isr2d.smc2bf: isr2d/bin/smc2bf.py\n" + " isr2d.bf2smc: isr2d/bin/bf2smc.py\n" + "resources:\n" + " ic:\n" + " threads: 4\n" + " smc:\n" + " threads: 4\n" + " bf:\n" + " mpi_processes: 4\n" + " smc2bf:\n" + " threads: 1\n" + " bf2smc:\n" + " threads: 1\n" + ) return text @pytest.fixture def test_config5() -> Configuration: model = Model( - 'test_model', - [ - Component('ic', 'isr2d.initial_conditions'), - Component('smc', 'isr2d.smc'), - Component('bf', 'isr2d.blood_flow'), - Component('smc2bf', 'isr2d.smc2bf'), - Component('bf2smc', 'isr2d.bf2smc')], - [ - Conduit('ic.out', 'smc.initial_state'), - Conduit('smc.cell_positions', 'smc2bf.in'), - Conduit('smc2bf.out', 'bf.initial_domain'), - Conduit('bf.wss_out', 'bf2smc.in'), - Conduit('bf2smc.out', 'smc.wss_in')]) + "test_model", + [ + Component("ic", "isr2d.initial_conditions"), + Component("smc", "isr2d.smc"), + Component("bf", "isr2d.blood_flow"), + Component("smc2bf", "isr2d.smc2bf"), + Component("bf2smc", "isr2d.bf2smc"), + ], + [ + Conduit("ic.out", "smc.initial_state"), + Conduit("smc.cell_positions", "smc2bf.in"), + Conduit("smc2bf.out", "bf.initial_domain"), + Conduit("bf.wss_out", "bf2smc.in"), + Conduit("bf2smc.out", "smc.wss_in"), + ], + ) implementations = [ - Implementation( - Reference('isr2d.initial_conditions'), script='isr2d/bin/ic'), - Implementation( - Reference('isr2d.smc'), script='isr2d/bin/smc'), - Implementation( - Reference('isr2d.blood_flow'), script='isr2d/bin/bf'), - Implementation( - Reference('isr2d.smc2bf'), script='isr2d/bin/smc2bf.py'), - Implementation( - Reference('isr2d.bf2smc'), script='isr2d/bin/bf2smc.py')] + Implementation(Reference("isr2d.initial_conditions"), script="isr2d/bin/ic"), + Implementation(Reference("isr2d.smc"), script="isr2d/bin/smc"), + Implementation(Reference("isr2d.blood_flow"), script="isr2d/bin/bf"), + Implementation(Reference("isr2d.smc2bf"), script="isr2d/bin/smc2bf.py"), + Implementation(Reference("isr2d.bf2smc"), script="isr2d/bin/bf2smc.py"), + ] resources = [ - ThreadedResReq(Reference('ic'), 4), - ThreadedResReq(Reference('smc'), 4), - MPICoresResReq(Reference('bf'), 4), - ThreadedResReq(Reference('smc2bf'), 1), - ThreadedResReq(Reference('bf2smc'), 1)] + ThreadedResReq(Reference("ic"), 4), + ThreadedResReq(Reference("smc"), 4), + MPICoresResReq(Reference("bf"), 4), + ThreadedResReq(Reference("smc2bf"), 1), + ThreadedResReq(Reference("bf2smc"), 1), + ] return Configuration(model, None, implementations, resources) @pytest.fixture def test_yaml6() -> str: - text = ('ymmsl_version: v0.1\n' - 'model:\n' - ' name: resources_test\n' - ' components:\n' - ' singlethreaded: a\n' - ' multithreaded: b\n' - ' mpi_cores1: c\n' - ' mpi_cores2: d\n' - ' mpi_nodes1: c\n' - ' mpi_nodes2: d\n' - 'implementations:\n' - ' a: /home/user/models/bin/modela\n' - ' b: /home/user/models/bin/modelb\n' - ' c:\n' - ' base_env: login\n' - ' modules:\n' - ' - gcc-6.3.0\n' - ' - openmpi-1.10\n' - ' execution_model: openmpi\n' - ' executable: /home/user/models/bin/modelc\n' - ' d:\n' - ' base_env: clean\n' - ' modules:\n' - ' - icc-18.0\n' - ' - IntelMPI-2021-3\n' - ' execution_model: intelmpi\n' - ' executable: /home/user/models/bin/modeld\n' - 'resources:\n' - ' singlethreaded:\n' - ' threads: 1\n' - ' multithreaded:\n' - ' threads: 8\n' - ' mpi_cores1:\n' - ' mpi_processes: 16\n' - ' mpi_cores2:\n' - ' mpi_processes: 4\n' - ' threads_per_mpi_process: 4\n' - ' mpi_nodes1:\n' - ' nodes: 10\n' - ' mpi_processes_per_node: 16\n' - ' mpi_nodes2:\n' - ' nodes: 10\n' - ' mpi_processes_per_node: 4\n' - ' threads_per_mpi_process: 4\n') + text = ( + "ymmsl_version: v0.1\n" + "model:\n" + " name: resources_test\n" + " components:\n" + " singlethreaded: a\n" + " multithreaded: b\n" + " mpi_cores1: c\n" + " mpi_cores2: d\n" + " mpi_nodes1: c\n" + " mpi_nodes2: d\n" + "implementations:\n" + " a: /home/user/models/bin/modela\n" + " b: /home/user/models/bin/modelb\n" + " c:\n" + " base_env: login\n" + " modules:\n" + " - gcc-6.3.0\n" + " - openmpi-1.10\n" + " execution_model: openmpi\n" + " executable: /home/user/models/bin/modelc\n" + " d:\n" + " base_env: clean\n" + " modules:\n" + " - icc-18.0\n" + " - IntelMPI-2021-3\n" + " execution_model: intelmpi\n" + " executable: /home/user/models/bin/modeld\n" + "resources:\n" + " singlethreaded:\n" + " threads: 1\n" + " multithreaded:\n" + " threads: 8\n" + " mpi_cores1:\n" + " mpi_processes: 16\n" + " mpi_cores2:\n" + " mpi_processes: 4\n" + " threads_per_mpi_process: 4\n" + " mpi_nodes1:\n" + " nodes: 10\n" + " mpi_processes_per_node: 16\n" + " mpi_nodes2:\n" + " nodes: 10\n" + " mpi_processes_per_node: 4\n" + " threads_per_mpi_process: 4\n" + ) return text @pytest.fixture def test_config6() -> Configuration: model = Model( - 'resources_test', - [ - Component('singlethreaded', 'a'), - Component('multithreaded', 'b'), - Component('mpi_cores1', 'c'), - Component('mpi_cores2', 'd'), - Component('mpi_nodes1', 'c'), - Component('mpi_nodes2', 'd')], - []) + "resources_test", + [ + Component("singlethreaded", "a"), + Component("multithreaded", "b"), + Component("mpi_cores1", "c"), + Component("mpi_cores2", "d"), + Component("mpi_nodes1", "c"), + Component("mpi_nodes2", "d"), + ], + [], + ) implementations = [ - Implementation( - Reference('a'), script='/home/user/models/bin/modela'), - Implementation( - Reference('b'), script='/home/user/models/bin/modelb'), - Implementation( - Reference('c'), - base_env=BaseEnv.LOGIN, - modules=['gcc-6.3.0', 'openmpi-1.10'], - execution_model=ExecutionModel.OPENMPI, - executable=Path('/home/user/models/bin/modelc')), - Implementation( - Reference('d'), - base_env=BaseEnv.CLEAN, - modules=['icc-18.0', 'IntelMPI-2021-3'], - execution_model=ExecutionModel.INTELMPI, - executable=Path('/home/user/models/bin/modeld'))] + Implementation(Reference("a"), script="/home/user/models/bin/modela"), + Implementation(Reference("b"), script="/home/user/models/bin/modelb"), + Implementation( + Reference("c"), + base_env=BaseEnv.LOGIN, + modules=["gcc-6.3.0", "openmpi-1.10"], + execution_model=ExecutionModel.OPENMPI, + executable=Path("/home/user/models/bin/modelc"), + ), + Implementation( + Reference("d"), + base_env=BaseEnv.CLEAN, + modules=["icc-18.0", "IntelMPI-2021-3"], + execution_model=ExecutionModel.INTELMPI, + executable=Path("/home/user/models/bin/modeld"), + ), + ] resources = [ - ThreadedResReq(Reference('singlethreaded'), 1), - ThreadedResReq(Reference('multithreaded'), 8), - MPICoresResReq(Reference('mpi_cores1'), 16), - MPICoresResReq(Reference('mpi_cores2'), 4, 4), - MPINodesResReq(Reference('mpi_nodes1'), 10, 16), - MPINodesResReq(Reference('mpi_nodes2'), 10, 4, 4)] + ThreadedResReq(Reference("singlethreaded"), 1), + ThreadedResReq(Reference("multithreaded"), 8), + MPICoresResReq(Reference("mpi_cores1"), 16), + MPICoresResReq(Reference("mpi_cores2"), 4, 4), + MPINodesResReq(Reference("mpi_nodes1"), 10, 16), + MPINodesResReq(Reference("mpi_nodes2"), 10, 4, 4), + ] return Configuration(model, None, implementations, resources) @@ -358,176 +378,206 @@ def test_config6() -> Configuration: @pytest.fixture def test_yaml7() -> str: text = ( - 'ymmsl_version: v0.1\n' - 'model:\n' - ' name: ports_test\n' - ' components:\n' - ' macro:\n' - ' ports:\n' - ' o_i:\n' - ' - state_out\n' - ' s:\n' - ' - x_in\n' - ' implementation: macro_python\n' - ' micro:\n' - ' ports:\n' - ' f_init:\n' - ' - init_in\n' - ' o_f:\n' - ' - final_output\n' - ' - extra_output\n' - ' implementation: micro_fortran\n' - ' conduits:\n' - ' macro.state_out: micro.init_in\n' - ' micro.final_output: macro.x_in\n') + "ymmsl_version: v0.1\n" + "model:\n" + " name: ports_test\n" + " components:\n" + " macro:\n" + " ports:\n" + " o_i:\n" + " - state_out\n" + " s:\n" + " - x_in\n" + " implementation: macro_python\n" + " micro:\n" + " ports:\n" + " f_init:\n" + " - init_in\n" + " o_f:\n" + " - final_output\n" + " - extra_output\n" + " implementation: micro_fortran\n" + " conduits:\n" + " macro.state_out: micro.init_in\n" + " micro.final_output: macro.x_in\n" + ) return text @pytest.fixture def test_config7() -> Configuration: model = Model( - 'ports_test', - [ - Component('macro', 'macro_python', ports=Ports( - o_i=['state_out'], s=['x_in'])), - Component('micro', 'micro_fortran', ports=Ports( - f_init=['init_in'], - o_f=['final_output', 'extra_output']))], - [ - Conduit('macro.state_out', 'micro.init_in'), - Conduit('micro.final_output', 'macro.x_in')]) + "ports_test", + [ + Component( + "macro", "macro_python", ports=Ports(o_i=["state_out"], s=["x_in"]) + ), + Component( + "micro", + "micro_fortran", + ports=Ports(f_init=["init_in"], o_f=["final_output", "extra_output"]), + ), + ], + [ + Conduit("macro.state_out", "micro.init_in"), + Conduit("micro.final_output", "macro.x_in"), + ], + ) return Configuration(model) @pytest.fixture def test_yaml8() -> str: text = ( - 'ymmsl_version: v0.1\n' - 'model:\n' - ' name: checkpoints\n' - ' components:\n' - ' macro:\n' - ' ports:\n' - ' o_i:\n' - ' - state_out1\n' - ' - state_out2\n' - ' s:\n' - ' - x_in1\n' - ' - x_in2\n' - ' implementation: macro_python\n' - ' micro1:\n' - ' ports:\n' - ' f_init:\n' - ' - init_in\n' - ' o_f:\n' - ' - final_output\n' - ' implementation: micro1_python\n' - ' micro2:\n' - ' ports:\n' - ' f_init:\n' - ' - init_in\n' - ' o_f:\n' - ' - final_output\n' - ' - extra_output\n' - ' implementation: micro2_fortran\n' - ' conduits:\n' - ' macro.state_out1: micro1.init_in\n' - ' macro.state_out2: micro2.init_in\n' - ' micro1.final_output: macro.x_in1\n' - ' micro2.final_output: macro.x_in2\n' - 'implementations:\n' - ' macro_python:\n' - ' executable: python\n' - ' args:\n' - ' - macro.py\n' - ' micro1_python:\n' - ' executable: python\n' - ' args:\n' - ' - micro1.py\n' - ' keeps_state_for_next_use: helpful\n' - ' micro2_fortran:\n' - ' executable: bin/micro2\n' - ' keeps_state_for_next_use: \'no\'\n' - 'resources:\n' - ' macro:\n' - ' threads: 1\n' - ' micro1:\n' - ' threads: 1\n' - ' micro2:\n' - ' threads: 4\n' - 'description: |-\n' - ' Snapshot for checkpoints taken on 2022-08-25 12:24:01\n' - ' Snapshot triggers:\n' - ' - wallclock_time >= 1800\n' - 'checkpoints:\n' - ' wallclock_time:\n' - ' - every: 600\n' - 'resume:\n' - ' macro: macro.pack\n' - ' micro1: micro1.pack\n') + "ymmsl_version: v0.1\n" + "model:\n" + " name: checkpoints\n" + " components:\n" + " macro:\n" + " ports:\n" + " o_i:\n" + " - state_out1\n" + " - state_out2\n" + " s:\n" + " - x_in1\n" + " - x_in2\n" + " implementation: macro_python\n" + " micro1:\n" + " ports:\n" + " f_init:\n" + " - init_in\n" + " o_f:\n" + " - final_output\n" + " implementation: micro1_python\n" + " micro2:\n" + " ports:\n" + " f_init:\n" + " - init_in\n" + " o_f:\n" + " - final_output\n" + " - extra_output\n" + " implementation: micro2_fortran\n" + " conduits:\n" + " macro.state_out1: micro1.init_in\n" + " macro.state_out2: micro2.init_in\n" + " micro1.final_output: macro.x_in1\n" + " micro2.final_output: macro.x_in2\n" + "implementations:\n" + " macro_python:\n" + " executable: python\n" + " args:\n" + " - macro.py\n" + " micro1_python:\n" + " executable: python\n" + " args:\n" + " - micro1.py\n" + " keeps_state_for_next_use: helpful\n" + " micro2_fortran:\n" + " executable: bin/micro2\n" + " keeps_state_for_next_use: 'no'\n" + "resources:\n" + " macro:\n" + " threads: 1\n" + " micro1:\n" + " threads: 1\n" + " micro2:\n" + " threads: 4\n" + "description: |-\n" + " Snapshot for checkpoints taken on 2022-08-25 12:24:01\n" + " Snapshot triggers:\n" + " - wallclock_time >= 1800\n" + "checkpoints:\n" + " wallclock_time:\n" + " - every: 600\n" + "resume:\n" + " macro: macro.pack\n" + " micro1: micro1.pack\n" + ) return text @pytest.fixture def test_config8() -> Configuration: model = Model( - 'checkpoints', - [ - Component('macro', 'macro_python', ports=Ports( - o_i=['state_out1', 'state_out2'], s=['x_in1', 'x_in2'])), - Component('micro1', 'micro1_python', ports=Ports( - f_init=['init_in'], o_f=['final_output'])), - Component('micro2', 'micro2_fortran', ports=Ports( - f_init=['init_in'], - o_f=['final_output', 'extra_output']))], - [ - Conduit('macro.state_out1', 'micro1.init_in'), - Conduit('macro.state_out2', 'micro2.init_in'), - Conduit('micro1.final_output', 'macro.x_in1'), - Conduit('micro2.final_output', 'macro.x_in2')]) + "checkpoints", + [ + Component( + "macro", + "macro_python", + ports=Ports(o_i=["state_out1", "state_out2"], s=["x_in1", "x_in2"]), + ), + Component( + "micro1", + "micro1_python", + ports=Ports(f_init=["init_in"], o_f=["final_output"]), + ), + Component( + "micro2", + "micro2_fortran", + ports=Ports(f_init=["init_in"], o_f=["final_output", "extra_output"]), + ), + ], + [ + Conduit("macro.state_out1", "micro1.init_in"), + Conduit("macro.state_out2", "micro2.init_in"), + Conduit("micro1.final_output", "macro.x_in1"), + Conduit("micro2.final_output", "macro.x_in2"), + ], + ) implementations = [ - Implementation( - Reference('macro_python'), base_env=BaseEnv.MANAGER, - executable=Path('python'), args='macro.py'), - Implementation( - Reference('micro1_python'), executable=Path('python'), - args='micro1.py', - keeps_state_for_next_use=KeepsStateForNextUse.HELPFUL), - Implementation( - Reference('micro2_fortran'), - executable=Path('bin/micro2'), - keeps_state_for_next_use=KeepsStateForNextUse.NO)] + Implementation( + Reference("macro_python"), + base_env=BaseEnv.MANAGER, + executable=Path("python"), + args="macro.py", + ), + Implementation( + Reference("micro1_python"), + executable=Path("python"), + args="micro1.py", + keeps_state_for_next_use=KeepsStateForNextUse.HELPFUL, + ), + Implementation( + Reference("micro2_fortran"), + executable=Path("bin/micro2"), + keeps_state_for_next_use=KeepsStateForNextUse.NO, + ), + ] resources = [ - ThreadedResReq(Reference('macro'), 1), - ThreadedResReq(Reference('micro1'), 1), - ThreadedResReq(Reference('micro2'), 4)] + ThreadedResReq(Reference("macro"), 1), + ThreadedResReq(Reference("micro1"), 1), + ThreadedResReq(Reference("micro2"), 4), + ] - description = ('Snapshot for checkpoints taken on 2022-08-25 12:24:01\n' - 'Snapshot triggers:\n' - '- wallclock_time >= 1800') + description = ( + "Snapshot for checkpoints taken on 2022-08-25 12:24:01\n" + "Snapshot triggers:\n" + "- wallclock_time >= 1800" + ) checkpoints = Checkpoints(wallclock_time=[CheckpointRangeRule(every=600)]) - resume = { - Ref('macro'): Path('macro.pack'), - Ref('micro1'): Path('micro1.pack')} + resume = {Ref("macro"): Path("macro.pack"), Ref("micro1"): Path("micro1.pack")} return Configuration( - model, None, implementations, resources, description, checkpoints, resume) + model, None, implementations, resources, description, checkpoints, resume + ) @pytest.fixture def test_yaml9() -> str: - text = ('ymmsl_version: v0.1\n' - 'implementations:\n' - ' isr2d.initial_conditions:\n' - ' execution_model: openmpi\n' - ' can_share_resources: true\n' - ' keeps_state_for_next_use: helpful\n' - ' script: |\n' - ' #!/bin/bash\n' - '\n' - ' mpirun my_model\n') + text = ( + "ymmsl_version: v0.1\n" + "implementations:\n" + " isr2d.initial_conditions:\n" + " execution_model: openmpi\n" + " can_share_resources: true\n" + " keeps_state_for_next_use: helpful\n" + " script: |\n" + " #!/bin/bash\n" + "\n" + " mpirun my_model\n" + ) return text diff --git a/ymmsl/tests/test_examples.py b/ymmsl/tests/test_examples.py index 854b126..923cacf 100644 --- a/ymmsl/tests/test_examples.py +++ b/ymmsl/tests/test_examples.py @@ -5,8 +5,8 @@ def test_load_examples() -> None: - doc_dir = Path(__file__).parents[2] / 'docs' + doc_dir = Path(__file__).parents[2] / "docs" - for example_file in doc_dir.glob('*.ymmsl'): + for example_file in doc_dir.glob("*.ymmsl"): # Just check that there are no errors load_as(Configuration, example_file) diff --git a/ymmsl/tests/test_io.py b/ymmsl/tests/test_io.py index 2402a19..b0bae19 100644 --- a/ymmsl/tests/test_io.py +++ b/ymmsl/tests/test_io.py @@ -8,7 +8,7 @@ def test_invalid_version() -> None: """This is a regression test, the error was really confusing""" with pytest.raises(yatiml.RecognitionError): - ymmsl.load('ymmsl_version: v0_1') + ymmsl.load("ymmsl_version: v0_1") def test_component_description_trailing_whitespace() -> None: @@ -16,78 +16,73 @@ def test_component_description_trailing_whitespace() -> None: PyYAML refuses to use block mode if there is trailing whitespace. """ - c = v0_2.Component('c1', v0_2.Ports(), 'Test \nmore test!') + c = v0_2.Component("c1", v0_2.Ports(), "Test \nmore test!") dump = yatiml.dumps_function(v0_2.Component, v0_2.Ports, v0_2.Reference) text = dump(c) - assert text == ( - 'name: c1\n' - 'ports: {}\n' - 'description: |\n' - ' Test\n' - ' more test!\n' - ) + assert text == ("name: c1\nports: {}\ndescription: |\n Test\n more test!\n") def test_configuration_description_trailing_whitespace() -> None: """Regression test, see above""" - c = v0_2.Configuration('Test\nmore test! ') + c = v0_2.Configuration("Test\nmore test! ") dump = yatiml.dumps_function(v0_2.Configuration, v0_2.Settings, v0_2.Checkpoints) text = dump(c) - assert text == ( - 'description: |\n' - ' Test\n' - ' more test!\n' - ) + assert text == ("description: |\n Test\n more test!\n") def test_model_description_trailing_whitespace() -> None: """Regression test, see above""" - m = v0_2.Model('test_model', v0_2.Ports(), 'Test\nmore test \nand more') + m = v0_2.Model("test_model", v0_2.Ports(), "Test\nmore test \nand more") dump = yatiml.dumps_function( - v0_2.Model, v0_2.Implementation, v0_2.Ports, v0_2.Reference) + v0_2.Model, v0_2.Implementation, v0_2.Ports, v0_2.Reference + ) text = dump(m) assert text == ( - 'name: test_model\n' - 'description: |\n' - ' Test\n' - ' more test\n' - ' and more\n' - 'components: {}\n' - ) + "name: test_model\n" + "description: |\n" + " Test\n" + " more test\n" + " and more\n" + "components: {}\n" + ) def test_program_description_trailing_whitespace() -> None: """Regression test, see above""" - p = v0_2.Program('test_program', v0_2.Ports(), 'Test ', script='') + p = v0_2.Program("test_program", v0_2.Ports(), "Test ", script="") dump = yatiml.dumps_function( - v0_2.Program, v0_2.BaseEnv, v0_2.ExecutionModel, v0_2.Implementation, - v0_2.KeepsStateForNextUse, v0_2.Ports, v0_2.Reference) + v0_2.Program, + v0_2.BaseEnv, + v0_2.ExecutionModel, + v0_2.Implementation, + v0_2.KeepsStateForNextUse, + v0_2.Ports, + v0_2.Reference, + ) text = dump(p) - assert text == ( - 'name: test_program\n' - 'description: |\n' - ' Test\n' - 'script: \'\'\n' - ) + assert text == ("name: test_program\ndescription: |\n Test\nscript: ''\n") def test_supported_settings_trailing_whitespace() -> None: """Regression test, see above""" - s = v0_2.SupportedSettings({ - 'alpha': 'float Collision angle ', - 'beta': 'float Dampening coefficient', - 'gamma': 'int Number\n of\nrays '}) + s = v0_2.SupportedSettings( + { + "alpha": "float Collision angle ", + "beta": "float Dampening coefficient", + "gamma": "int Number\n of\nrays ", + } + ) dump = yatiml.dumps_function( - v0_2.SupportedSettings, v0_2.Identifier, v0_2.SettingType, - v0_2.SupportedSetting) + v0_2.SupportedSettings, v0_2.Identifier, v0_2.SettingType, v0_2.SupportedSetting + ) text = dump(s) assert text == ( - 'alpha: float Collision angle\n' - 'beta: float Dampening coefficient\n' - 'gamma:\n' - ' type: int\n' - ' description: |\n' - ' Number\n' - ' of\n' - ' rays\n' - ) + "alpha: float Collision angle\n" + "beta: float Dampening coefficient\n" + "gamma:\n" + " type: int\n" + " description: |\n" + " Number\n" + " of\n" + " rays\n" + ) diff --git a/ymmsl/tests/test_io_v0_1.py b/ymmsl/tests/test_io_v0_1.py index 238a138..9f7b2c0 100644 --- a/ymmsl/tests/test_io_v0_1.py +++ b/ymmsl/tests/test_io_v0_1.py @@ -34,15 +34,15 @@ def test_load_string1(test_yaml1: str) -> None: settings = configuration.settings assert settings is not None assert len(settings) == 4 - assert isinstance(settings['test_list'], list) - assert settings['test_list'][1] == 1.3 + assert isinstance(settings["test_list"], list) + assert settings["test_list"][1] == 1.3 configuration = load(test_yaml1) assert isinstance(configuration, PartialConfiguration) assert configuration.settings is not None - assert configuration.settings['test_str'] == 'value' - assert configuration.settings['test_int'] == 13 - assert configuration.settings['test_list'] == [12.3, 1.3] + assert configuration.settings["test_str"] == "value" + assert configuration.settings["test_int"] == 13 + assert configuration.settings["test_list"] == [12.3, 1.3] def test_load_string2(test_yaml2: str) -> None: @@ -51,14 +51,14 @@ def test_load_string2(test_yaml2: str) -> None: assert not isinstance(configuration, Configuration) model = configuration.model assert isinstance(model, Model) - assert str(model.name) == 'test_model' + assert str(model.name) == "test_model" assert len(model.components) == 5 - assert str(model.components[4].name) == 'bf2smc' - assert str(model.components[2].implementation) == 'isr2d.blood_flow' + assert str(model.components[4].name) == "bf2smc" + assert str(model.components[2].implementation) == "isr2d.blood_flow" assert len(model.conduits) == 5 - assert str(model.conduits[0].sender) == 'ic.out' - assert str(model.conduits[1].sending_port()) == 'cell_positions' - assert str(model.conduits[3].receiving_component()) == 'bf2smc' + assert str(model.conduits[0].sender) == "ic.out" + assert str(model.conduits[1].sending_port()) == "cell_positions" + assert str(model.conduits[3].receiving_component()) == "bf2smc" def test_load_string3(test_yaml3: str) -> None: @@ -66,7 +66,7 @@ def test_load_string3(test_yaml3: str) -> None: assert isinstance(configuration, PartialConfiguration) assert not isinstance(configuration, Configuration) assert isinstance(configuration.model, ModelReference) - assert str(configuration.model.name) == 'test_model' + assert str(configuration.model.name) == "test_model" def test_load_string4(test_yaml4: str) -> None: @@ -75,17 +75,17 @@ def test_load_string4(test_yaml4: str) -> None: assert not isinstance(configuration, Configuration) assert len(configuration.implementations) == 5 impls = configuration.implementations - ic = Reference('isr2d.initial_conditions') - bf2smc = Reference('isr2d.bf2smc') - assert impls[ic].name == 'isr2d.initial_conditions' - assert impls[bf2smc].script == 'isr2d/bin/bf2smc.py' + ic = Reference("isr2d.initial_conditions") + bf2smc = Reference("isr2d.bf2smc") + assert impls[ic].name == "isr2d.initial_conditions" + assert impls[bf2smc].script == "isr2d/bin/bf2smc.py" assert len(configuration.resources) == 5 - ic_res = configuration.resources[Reference('ic')] + ic_res = configuration.resources[Reference("ic")] assert isinstance(ic_res, ThreadedResReq) assert ic_res.threads == 4 - bf2smc_res = configuration.resources[Reference('bf2smc')] + bf2smc_res = configuration.resources[Reference("bf2smc")] assert isinstance(bf2smc_res, ThreadedResReq) assert bf2smc_res.threads == 1 @@ -107,30 +107,30 @@ def test_load_string6(test_yaml6: str) -> None: res = configuration.resources assert len(res) == 6 - singlethreaded = res[Reference('singlethreaded')] + singlethreaded = res[Reference("singlethreaded")] assert isinstance(singlethreaded, ThreadedResReq) assert singlethreaded.threads == 1 - multithreaded = res[Reference('multithreaded')] + multithreaded = res[Reference("multithreaded")] assert isinstance(multithreaded, ThreadedResReq) assert multithreaded.threads == 8 - mpi_cores1 = res[Reference('mpi_cores1')] + mpi_cores1 = res[Reference("mpi_cores1")] assert isinstance(mpi_cores1, MPICoresResReq) assert mpi_cores1.mpi_processes == 16 - mpi_cores2 = res[Reference('mpi_cores2')] + mpi_cores2 = res[Reference("mpi_cores2")] assert isinstance(mpi_cores2, MPICoresResReq) assert mpi_cores2.mpi_processes == 4 assert mpi_cores2.threads_per_mpi_process == 4 - mpi_nodes1 = res[Reference('mpi_nodes1')] + mpi_nodes1 = res[Reference("mpi_nodes1")] assert isinstance(mpi_nodes1, MPINodesResReq) assert mpi_nodes1.nodes == 10 assert mpi_nodes1.mpi_processes_per_node == 16 assert mpi_nodes1.threads_per_mpi_process == 1 - mpi_nodes2 = res[Reference('mpi_nodes2')] + mpi_nodes2 = res[Reference("mpi_nodes2")] assert isinstance(mpi_nodes2, MPINodesResReq) assert mpi_nodes2.nodes == 10 assert mpi_nodes2.mpi_processes_per_node == 4 @@ -144,18 +144,18 @@ def test_load_string7(test_yaml7: str) -> None: assert isinstance(configuration.model, Model) components = configuration.model.components assert components is not None - assert components[0].name == 'macro' - assert components[0].implementation == 'macro_python' + assert components[0].name == "macro" + assert components[0].implementation == "macro_python" assert components[0].ports is not None assert components[0].ports.f_init == [] - assert components[0].ports.o_i == ['state_out'] - assert components[0].ports.s == ['x_in'] + assert components[0].ports.o_i == ["state_out"] + assert components[0].ports.s == ["x_in"] assert components[0].ports.o_f == [] assert components[1].ports is not None - assert components[1].ports.f_init == ['init_in'] + assert components[1].ports.f_init == ["init_in"] assert components[1].ports.o_i == [] assert components[1].ports.s == [] - assert components[1].ports.o_f == ['final_output', 'extra_output'] + assert components[1].ports.o_f == ["final_output", "extra_output"] def test_load_string8(test_yaml8: str) -> None: @@ -178,20 +178,20 @@ def test_load_string9(test_yaml9: str) -> None: configuration = load(test_yaml9) assert isinstance(configuration, PartialConfiguration) - implementation = configuration.implementations[Ref('isr2d.initial_conditions')] - assert implementation.name == 'isr2d.initial_conditions' + implementation = configuration.implementations[Ref("isr2d.initial_conditions")] + assert implementation.name == "isr2d.initial_conditions" assert implementation.execution_model == ExecutionModel.OPENMPI assert implementation.can_share_resources is True assert implementation.keeps_state_for_next_use == KeepsStateForNextUse.HELPFUL - assert implementation.script == '#!/bin/bash\n\nmpirun my_model\n' + assert implementation.script == "#!/bin/bash\n\nmpirun my_model\n" def test_load_file(test_yaml1: str, tmpdir_path: Path) -> None: - test_file = tmpdir_path / 'test_yaml1.ymmsl' - with test_file.open('w') as f: + test_file = tmpdir_path / "test_yaml1.ymmsl" + with test_file.open("w") as f: f.write(test_yaml1) - with test_file.open('r') as f: + with test_file.open("r") as f: configuration = load(f) assert isinstance(configuration, PartialConfiguration) @@ -199,8 +199,8 @@ def test_load_file(test_yaml1: str, tmpdir_path: Path) -> None: def test_load_path(test_yaml1: str, tmpdir_path: Path) -> None: - test_file = tmpdir_path / 'test_yaml1.ymmsl' - with test_file.open('w') as f: + test_file = tmpdir_path / "test_yaml1.ymmsl" + with test_file.open("w") as f: f.write(test_yaml1) configuration = load(test_file) @@ -248,48 +248,48 @@ def test_dump8(test_yaml8: str, test_config8: Configuration) -> None: assert text == test_yaml8 -def test_save_str(test_config1: PartialConfiguration, test_yaml1: str, - tmpdir_path: Path) -> None: - test_file = tmpdir_path / 'test_yaml1.ymmsl' +def test_save_str( + test_config1: PartialConfiguration, test_yaml1: str, tmpdir_path: Path +) -> None: + test_file = tmpdir_path / "test_yaml1.ymmsl" save(test_config1, str(test_file)) - with test_file.open('r') as f: + with test_file.open("r") as f: yaml_out = f.read() assert yaml_out == test_yaml1 -def test_save_path(test_config2: PartialConfiguration, test_yaml2: str, - tmpdir_path: Path) -> None: - test_file = tmpdir_path / 'test_yaml1.ymmsl' +def test_save_path( + test_config2: PartialConfiguration, test_yaml2: str, tmpdir_path: Path +) -> None: + test_file = tmpdir_path / "test_yaml1.ymmsl" save(test_config2, test_file) - with test_file.open('r') as f: + with test_file.open("r") as f: yaml_out = f.read() assert yaml_out == test_yaml2 -def test_save_file(test_config2: PartialConfiguration, test_yaml2: str, - tmpdir_path: Path) -> None: - test_file = tmpdir_path / 'test_yaml1.ymmsl' +def test_save_file( + test_config2: PartialConfiguration, test_yaml2: str, tmpdir_path: Path +) -> None: + test_file = tmpdir_path / "test_yaml1.ymmsl" - with test_file.open('w') as f: + with test_file.open("w") as f: save(test_config2, f) - with test_file.open('r') as f: + with test_file.open("r") as f: yaml_out = f.read() assert yaml_out == test_yaml2 def test_resource_requirements() -> None: - text = ( - 'ymmsl_version: v0.1\n' - 'resources:\n' - ' - name: submodel\n') + text = "ymmsl_version: v0.1\nresources:\n - name: submodel\n" with pytest.raises(RecognitionError): load(text) diff --git a/ymmsl/tests/test_load_as.py b/ymmsl/tests/test_load_as.py index 16ee4b1..d7ce027 100644 --- a/ymmsl/tests/test_load_as.py +++ b/ymmsl/tests/test_load_as.py @@ -10,12 +10,12 @@ def test_load_as_v0_1(test_yaml1: str) -> None: config = load_as(v0_1.PartialConfiguration, test_yaml1) assert isinstance(config, v0_1.PartialConfiguration) - assert cast(List[int], config.settings['test_list'])[0] == 12.3 + assert cast(List[int], config.settings["test_list"])[0] == 12.3 -@pytest.mark.filterwarnings('ignore:Comments.*') -@pytest.mark.filterwarnings('ignore:When specifying resources.*') +@pytest.mark.filterwarnings("ignore:Comments.*") +@pytest.mark.filterwarnings("ignore:When specifying resources.*") def test_load_as_v0_2(test_yaml1: str) -> None: config = load_as(v0_2.Configuration, test_yaml1) assert isinstance(config, v0_2.Configuration) - assert config.settings['test_str'] == 'value' + assert config.settings["test_str"] == "value" diff --git a/ymmsl/util.py b/ymmsl/util.py index 785aa67..d3bbdc7 100644 --- a/ymmsl/util.py +++ b/ymmsl/util.py @@ -4,4 +4,4 @@ def remove_trailing_whitespace(text: str) -> str: This ensures that each line in text ends with only a newline, with no whitespace in between the text and that newline. """ - return '\n'.join([line.rstrip() for line in text.split('\n')]) + '\n' + return "\n".join([line.rstrip() for line in text.split("\n")]) + "\n" diff --git a/ymmsl/v0_1/__init__.py b/ymmsl/v0_1/__init__.py index a0d3580..76aa771 100644 --- a/ymmsl/v0_1/__init__.py +++ b/ymmsl/v0_1/__init__.py @@ -5,31 +5,51 @@ """ from ymmsl.v0_1.checkpoint import ( - CheckpointAtRule, - CheckpointRangeRule, - CheckpointRule, - Checkpoints, + CheckpointAtRule, + CheckpointRangeRule, + CheckpointRule, + Checkpoints, ) from ymmsl.v0_1.component import Component, Operator, Port, Ports from ymmsl.v0_1.configuration import Configuration, PartialConfiguration from ymmsl.v0_1.execution import ( - BaseEnv, - ExecutionModel, - Implementation, - KeepsStateForNextUse, - MPICoresResReq, - MPINodesResReq, - ResourceRequirements, - ThreadedResReq, + BaseEnv, + ExecutionModel, + Implementation, + KeepsStateForNextUse, + MPICoresResReq, + MPINodesResReq, + ResourceRequirements, + ThreadedResReq, ) from ymmsl.v0_1.identity import Identifier, Reference from ymmsl.v0_1.model import Conduit, Model, ModelReference from ymmsl.v0_1.settings import Settings, SettingValue __all__ = [ - 'BaseEnv', 'CheckpointRule', 'CheckpointRangeRule', 'CheckpointAtRule', - 'Checkpoints', 'Component', 'Conduit', 'Configuration', 'ExecutionModel', - 'Identifier', 'Implementation', 'KeepsStateForNextUse', 'Model', - 'ModelReference', 'MPICoresResReq', 'MPINodesResReq', 'Operator', - 'PartialConfiguration', 'Port', 'Ports', 'Reference', 'ResourceRequirements', - 'Settings', 'SettingValue', 'ThreadedResReq'] + "BaseEnv", + "CheckpointRule", + "CheckpointRangeRule", + "CheckpointAtRule", + "Checkpoints", + "Component", + "Conduit", + "Configuration", + "ExecutionModel", + "Identifier", + "Implementation", + "KeepsStateForNextUse", + "Model", + "ModelReference", + "MPICoresResReq", + "MPINodesResReq", + "Operator", + "PartialConfiguration", + "Port", + "Ports", + "Reference", + "ResourceRequirements", + "Settings", + "SettingValue", + "ThreadedResReq", +] diff --git a/ymmsl/v0_1/checkpoint.py b/ymmsl/v0_1/checkpoint.py index 70030e6..ea019ed 100644 --- a/ymmsl/v0_1/checkpoint.py +++ b/ymmsl/v0_1/checkpoint.py @@ -34,10 +34,12 @@ class CheckpointRangeRule(CheckpointRule): every: Step size of the range, must be positive. """ - def __init__(self, - start: Optional[Union[float, int]] = None, - stop: Optional[Union[float, int]] = None, - every: Union[float, int] = 0) -> None: + def __init__( + self, + start: Optional[Union[float, int]] = None, + stop: Optional[Union[float, int]] = None, + every: Union[float, int] = 0, + ) -> None: """Create a checkpoint range. Args: @@ -46,19 +48,22 @@ def __init__(self, every: Step size, must be larger than 0. """ if every <= 0: - raise ValueError(f"Invalid every {every} in checkpoint range:" - " must be larger than 0.") + raise ValueError( + f"Invalid every {every} in checkpoint range: must be larger than 0." + ) if start is not None and stop is not None and start > stop: - raise ValueError(f"Invalid start {start} and stop {stop} in" - "checkpoint range: stop cannot be smaller than" - " start.") + raise ValueError( + f"Invalid start {start} and stop {stop} in" + "checkpoint range: stop cannot be smaller than" + " start." + ) self.start = start self.stop = stop self.every = every @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: - node.require_attribute('every') + node.require_attribute("every") @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: @@ -91,20 +96,21 @@ def __init__(self, at: Optional[List[Union[float, int]]]) -> None: @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: - node.require_attribute('at') + node.require_attribute("at") @classmethod def _yatiml_savorize(cls, node: yatiml.Node) -> None: - if node.has_attribute('at'): - if (node.has_attribute_type('at', int) or - node.has_attribute_type('at', float)): - attr = node.get_attribute('at') + if node.has_attribute("at"): + if node.has_attribute_type("at", int) or node.has_attribute_type( + "at", float + ): + attr = node.get_attribute("at") start_mark = attr.yaml_node.start_mark end_mark = attr.yaml_node.end_mark new_seq = yaml.nodes.SequenceNode( - 'tag:yaml.org,2002:seq', [attr.yaml_node], start_mark, - end_mark) - node.set_attribute('at', new_seq) + "tag:yaml.org,2002:seq", [attr.yaml_node], start_mark, end_mark + ) + node.set_attribute("at", new_seq) class Checkpoints: @@ -126,11 +132,13 @@ class Checkpoints: wallclock_time: Checkpoint rules for the wallclock_time trigger. simulation_time: Checkpoint rules for the simulation_time trigger. """ - def __init__(self, - at_end: bool = False, - wallclock_time: Optional[List[CheckpointRule]] = None, - simulation_time: Optional[List[CheckpointRule]] = None - ) -> None: + + def __init__( + self, + at_end: bool = False, + wallclock_time: Optional[List[CheckpointRule]] = None, + simulation_time: Optional[List[CheckpointRule]] = None, + ) -> None: """Create checkpoint definitions. Args: @@ -147,11 +155,9 @@ def __init__(self, def __bool__(self) -> bool: """Evaluate to true iff any rules are defined.""" - return (self.at_end or - bool(self.wallclock_time) or - bool(self.simulation_time)) + return self.at_end or bool(self.wallclock_time) or bool(self.simulation_time) - def update(self, overlay: 'Checkpoints') -> None: + def update(self, overlay: "Checkpoints") -> None: """Update this checkpoints with the given overlay. Sets `at_end` to True if it is set in the overlay, otherwise `at_end` @@ -169,10 +175,12 @@ def _yatiml_sweeten(cls, node: yatiml.Node) -> None: wallclock_time = node.get_attribute("wallclock_time") if wallclock_time.is_scalar(type(None)) or ( - wallclock_time.is_sequence() and wallclock_time.is_empty()): + wallclock_time.is_sequence() and wallclock_time.is_empty() + ): node.remove_attribute("wallclock_time") simulation_time = node.get_attribute("simulation_time") if simulation_time.is_scalar(type(None)) or ( - simulation_time.is_sequence() and simulation_time.is_empty()): + simulation_time.is_sequence() and simulation_time.is_empty() + ): node.remove_attribute("simulation_time") diff --git a/ymmsl/v0_1/component.py b/ymmsl/v0_1/component.py index b296e81..2cd926d 100644 --- a/ymmsl/v0_1/component.py +++ b/ymmsl/v0_1/component.py @@ -1,4 +1,5 @@ """Definitions for describing simulation components.""" + import logging from collections import OrderedDict from enum import Enum @@ -25,11 +26,11 @@ class Operator(Enum): and operators for other components such as mappers. """ - NONE = 0 #: No operator + NONE = 0 #: No operator F_INIT = 1 #: Initialisation phase, before start of the SEL - O_I = 2 #: State observation within the model's main loop - S = 3 #: State update in the model's main loop - O_F = 5 #: Observation of final state, after the SEL + O_I = 2 #: State observation within the model's main loop + S = 3 #: State update in the model's main loop + O_F = 5 #: Observation of final state, after the SEL def allows_sending(self) -> bool: """Whether ports on this operator can send.""" @@ -87,11 +88,14 @@ class Ports: s: The ports associated with the S operator. o_f: The ports associated with the O_F operator """ + def __init__( - self, f_init: Union[None, str, List[str]] = None, - o_i: Union[None, str, List[str]] = None, - s: Union[None, str, List[str]] = None, - o_f: Union[None, str, List[str]] = None) -> None: + self, + f_init: Union[None, str, List[str]] = None, + o_i: Union[None, str, List[str]] = None, + s: Union[None, str, List[str]] = None, + o_f: Union[None, str, List[str]] = None, + ) -> None: """Create a Ports declaration. Args: @@ -100,12 +104,13 @@ def __init__( s: The ports associated with the S operator. o_f: The ports associated with the O_F operator """ + def to_list(ports: Union[None, str, List[str]]) -> List[Identifier]: if ports is None: return list() if isinstance(ports, str): - ports = ports.split(' ') + ports = ports.split(" ") return list(map(Identifier, ports)) self.f_init = to_list(f_init) @@ -115,11 +120,12 @@ def to_list(ports: Union[None, str, List[str]]) -> List[Identifier]: all_names = sorted(self.port_names()) for i in range(len(all_names) - 1): - if all_names[i] == all_names[i+1]: + if all_names[i] == all_names[i + 1]: raise RuntimeError( - 'Invalid ports specification: port "{}" is specified' - ' more than once. Port names must be unique within' - ' the component.') + 'Invalid ports specification: port "{}" is specified' + " more than once. Port names must be unique within" + " the component." + ) def port_names(self) -> Iterable[Identifier]: """Returns an iterable containing the names of all ports.""" @@ -128,10 +134,11 @@ def port_names(self) -> Iterable[Identifier]: def all_ports(self) -> Iterable[Port]: """Returns an iterable containing all ports.""" return ( - [Port(n, Operator.F_INIT) for n in self.f_init] + - [Port(name, Operator.O_I) for name in self.o_i] + - [Port(name, Operator.S) for name in self.s] + - [Port(name, Operator.O_F) for name in self.o_f]) + [Port(n, Operator.F_INIT) for n in self.f_init] + + [Port(name, Operator.O_I) for name in self.o_i] + + [Port(name, Operator.S) for name in self.s] + + [Port(name, Operator.O_F) for name in self.o_f] + ) def operator(self, port_name: Identifier) -> Operator: """Looks up the operator for a given port. @@ -160,10 +167,11 @@ def operator(self, port_name: Identifier) -> Operator: raise KeyError(f'No port named "{port_name}" was found') _yatiml_defaults: dict[str, Optional[list[str]]] = { - 'f_init': [], - 'o_i': [], - 's': [], - 'o_f': []} + "f_init": [], + "o_i": [], + "s": [], + "o_f": [], + } @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: @@ -190,9 +198,13 @@ class Component: """ - def __init__(self, name: str, implementation: Optional[str] = None, - multiplicity: Union[None, int, List[int]] = None, - ports: Optional[Ports] = None) -> None: + def __init__( + self, + name: str, + implementation: Optional[str] = None, + multiplicity: Union[None, int, List[int]] = None, + ports: Optional[Ports] = None, + ) -> None: """Create a Component. Args: @@ -213,8 +225,10 @@ def __init__(self, name: str, implementation: Optional[str] = None, self.implementation = Reference(implementation) for part in self.implementation: if isinstance(part, int): - raise ValueError(f"Component implementation {self.name} contains a" - " subscript, which is not allowed.") + raise ValueError( + f"Component implementation {self.name} contains a" + " subscript, which is not allowed." + ) if multiplicity is None: self.multiplicity = list() @@ -243,6 +257,7 @@ def instances(self) -> List[Reference]: A list with one Reference for each instance of this component. """ + def increment(index: List[int], dims: List[int]) -> None: # assumes index and dims are the same length > 0 # modifies index argument @@ -279,37 +294,40 @@ def generate_indices(multiplicity: List[int]) -> List[List[int]]: def _yatiml_attributes(self) -> OrderedDict: """Reorder attributes for saving to YAML.""" - return OrderedDict([ - ('name', self.name), - ('ports', self.ports), - ('multiplicity', self.multiplicity), - ('implementation', self.implementation)]) + return OrderedDict( + [ + ("name", self.name), + ("ports", self.ports), + ("multiplicity", self.multiplicity), + ("implementation", self.implementation), + ] + ) @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: - node.require_attribute('name', str) + node.require_attribute("name", str) @classmethod def _yatiml_savorize(cls, node: yatiml.Node) -> None: - if node.has_attribute('multiplicity'): - if node.has_attribute_type('multiplicity', int): - attr = node.get_attribute('multiplicity') + if node.has_attribute("multiplicity"): + if node.has_attribute_type("multiplicity", int): + attr = node.get_attribute("multiplicity") start_mark = attr.yaml_node.start_mark end_mark = attr.yaml_node.end_mark new_seq = yaml.nodes.SequenceNode( - 'tag:yaml.org,2002:seq', [attr.yaml_node], start_mark, - end_mark) - node.set_attribute('multiplicity', new_seq) + "tag:yaml.org,2002:seq", [attr.yaml_node], start_mark, end_mark + ) + node.set_attribute("multiplicity", new_seq) @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - multiplicity = node.get_attribute('multiplicity') + multiplicity = node.get_attribute("multiplicity") items = multiplicity.seq_items() if len(items) == 0: - node.remove_attribute('multiplicity') + node.remove_attribute("multiplicity") elif len(items) == 1: - node.set_attribute('multiplicity', items[0].get_value()) + node.set_attribute("multiplicity", items[0].get_value()) - ports_node = node.get_attribute('ports').yaml_node - if ports_node.tag == 'tag:yaml.org,2002:null': - node.remove_attribute('ports') + ports_node = node.get_attribute("ports").yaml_node + if ports_node.tag == "tag:yaml.org,2002:null": + node.remove_attribute("ports") diff --git a/ymmsl/v0_1/configuration.py b/ymmsl/v0_1/configuration.py index 8390a71..c2a2ab1 100644 --- a/ymmsl/v0_1/configuration.py +++ b/ymmsl/v0_1/configuration.py @@ -1,4 +1,5 @@ """This module contains all the definitions for yMMSL.""" + import collections.abc as abc import logging from collections import OrderedDict @@ -46,19 +47,24 @@ class PartialConfiguration(Document): checkpoints: Defines when each model component should create a snapshot resume: Defines what snapshot each model component should resume from """ - def __init__(self, - model: Optional[ModelReference] = None, - settings: Optional[Settings] = None, - implementations: Optional[Union[ - List[Implementation], - Dict[Reference, Implementation]]] = None, - resources: Optional[Union[ - Sequence[ResourceRequirements], - MutableMapping[Reference, ResourceRequirements]]] = None, - description: Optional[str] = None, - checkpoints: Optional[Checkpoints] = None, - resume: Optional[Dict[Reference, Path]] = None - ) -> None: + + def __init__( + self, + model: Optional[ModelReference] = None, + settings: Optional[Settings] = None, + implementations: Optional[ + Union[List[Implementation], Dict[Reference, Implementation]] + ] = None, + resources: Optional[ + Union[ + Sequence[ResourceRequirements], + MutableMapping[Reference, ResourceRequirements], + ] + ] = None, + description: Optional[str] = None, + checkpoints: Optional[Checkpoints] = None, + resume: Optional[Dict[Reference, Path]] = None, + ) -> None: """Create a Configuration. Implementations and resources may be either a list of such @@ -85,21 +91,21 @@ def __init__(self, if implementations is None: self.implementations: _ImplType = dict() elif isinstance(implementations, list): - self.implementations = OrderedDict([ - (impl.name, impl) for impl in implementations]) + self.implementations = OrderedDict( + [(impl.name, impl) for impl in implementations] + ) else: self.implementations = implementations if resources is None: self.resources: _ResType = dict() elif isinstance(resources, abc.Sequence): - self.resources = OrderedDict([ - (res.name, res) for res in resources]) + self.resources = OrderedDict([(res.name, res) for res in resources]) else: self.resources = resources if description is None: - self.description = '' + self.description = "" else: self.description = description @@ -113,7 +119,7 @@ def __init__(self, else: self.resume = resume - def update(self, overlay: 'PartialConfiguration') -> None: + def update(self, overlay: "PartialConfiguration") -> None: """Update this configuration with the given overlay. This will update the model according to :meth:`Model.update`, copy @@ -145,12 +151,12 @@ def update(self, overlay: 'PartialConfiguration') -> None: if not self.description: self.description = overlay.description elif overlay.description: - self.description += '\n\n' + overlay.description + self.description += "\n\n" + overlay.description self.checkpoints.update(overlay.checkpoints) self.resume.update(overlay.resume) - def as_configuration(self) -> 'Configuration': + def as_configuration(self) -> "Configuration": """Converts to a full Configuration object. This checks that this PartialConfiguration has all the pieces @@ -168,15 +174,23 @@ def as_configuration(self) -> 'Configuration': ValueError: If this configuration isn't complete. """ if ( - self.model is not None and isinstance(self.model, Model) and - self.implementations and self.resources): + self.model is not None + and isinstance(self.model, Model) + and self.implementations + and self.resources + ): return Configuration( - self.model, self.settings, self.implementations, - self.resources, self.description, self.checkpoints, - self.resume) + self.model, + self.settings, + self.implementations, + self.resources, + self.description, + self.checkpoints, + self.resume, + ) raise ValueError( - 'Model, implementations or resources are missing from the' - ' configuration.') + "Model, implementations or resources are missing from the configuration." + ) @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: @@ -188,53 +202,45 @@ def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: @classmethod def _yatiml_savorize(cls, node: yatiml.Node) -> None: - if not node.has_attribute('settings'): - node.set_attribute('settings', None) - node.map_attribute_to_index('implementations', 'name', 'script') - node.map_attribute_to_index('resources', 'name') + if not node.has_attribute("settings"): + node.set_attribute("settings", None) + node.map_attribute_to_index("implementations", "name", "script") + node.map_attribute_to_index("resources", "name") @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - if node.get_attribute('model').is_scalar(type(None)): - node.remove_attribute('model') - - if node.get_attribute('settings').is_scalar(type(None)): - node.remove_attribute('settings') - if len(node.get_attribute('settings').yaml_node.value) == 0: - node.remove_attribute('settings') - - impl = node.get_attribute('implementations') - if ( - impl.is_scalar(type(None)) or - impl.is_mapping() and impl.is_empty()): - node.remove_attribute('implementations') - node.index_attribute_to_map('implementations', 'name', 'script') - - res = node.get_attribute('resources') - if ( - res.is_scalar(type(None)) or - res.is_mapping() and res.is_empty()): - node.remove_attribute('resources') - node.index_attribute_to_map('resources', 'name') - - descr = node.get_attribute('description') - if descr.is_scalar(type(None)) or descr.get_value() == '': - node.remove_attribute('description') - if descr.is_scalar(str) and '\n' in cast(str, descr.get_value()): + if node.get_attribute("model").is_scalar(type(None)): + node.remove_attribute("model") + + if node.get_attribute("settings").is_scalar(type(None)): + node.remove_attribute("settings") + if len(node.get_attribute("settings").yaml_node.value) == 0: + node.remove_attribute("settings") + + impl = node.get_attribute("implementations") + if impl.is_scalar(type(None)) or impl.is_mapping() and impl.is_empty(): + node.remove_attribute("implementations") + node.index_attribute_to_map("implementations", "name", "script") + + res = node.get_attribute("resources") + if res.is_scalar(type(None)) or res.is_mapping() and res.is_empty(): + node.remove_attribute("resources") + node.index_attribute_to_map("resources", "name") + + descr = node.get_attribute("description") + if descr.is_scalar(type(None)) or descr.get_value() == "": + node.remove_attribute("description") + if descr.is_scalar(str) and "\n" in cast(str, descr.get_value()): # output multi-line string in literal mode - cast(yaml.ScalarNode, descr.yaml_node).style = '|' + cast(yaml.ScalarNode, descr.yaml_node).style = "|" - cp = node.get_attribute('checkpoints') - if ( - cp.is_scalar(type(None)) or - cp.is_mapping() and cp.is_empty()): - node.remove_attribute('checkpoints') + cp = node.get_attribute("checkpoints") + if cp.is_scalar(type(None)) or cp.is_mapping() and cp.is_empty(): + node.remove_attribute("checkpoints") - resu = node.get_attribute('resume') - if ( - resu.is_scalar(type(None)) or - resu.is_mapping() and resu.is_empty()): - node.remove_attribute('resume') + resu = node.get_attribute("resume") + if resu.is_scalar(type(None)) or resu.is_mapping() and resu.is_empty(): + node.remove_attribute("resume") class Configuration(PartialConfiguration): @@ -261,19 +267,24 @@ class Configuration(PartialConfiguration): checkpoints: Defines when each model component should create a snapshot resume: Defines what snapshot each model component should resume from """ - def __init__(self, - model: Model, - settings: Optional[Settings] = None, - implementations: Optional[Union[ - List[Implementation], - Dict[Reference, Implementation]]] = None, - resources: Optional[Union[ - Sequence[ResourceRequirements], - MutableMapping[Reference, ResourceRequirements]]] = None, - description: Optional[str] = None, - checkpoints: Optional[Checkpoints] = None, - resume: Optional[Dict[Reference, Path]] = None - ) -> None: + + def __init__( + self, + model: Model, + settings: Optional[Settings] = None, + implementations: Optional[ + Union[List[Implementation], Dict[Reference, Implementation]] + ] = None, + resources: Optional[ + Union[ + Sequence[ResourceRequirements], + MutableMapping[Reference, ResourceRequirements], + ] + ] = None, + description: Optional[str] = None, + checkpoints: Optional[Checkpoints] = None, + resume: Optional[Dict[Reference, Path]] = None, + ) -> None: """Create a Configuration. Implementations and resources may be either a list of such @@ -294,8 +305,14 @@ def __init__(self, # Configuration.model always has type Model self.model: Model super().__init__( - model, settings, implementations, resources, description, - checkpoints, resume) + model, + settings, + implementations, + resources, + description, + checkpoints, + resume, + ) def check_consistent(self) -> None: """Checks that the configuration is internally consistent. @@ -328,21 +345,22 @@ def check_consistent(self) -> None: # script for specifying an implementation. raise RuntimeError( f"Model component {comp}'s implementation specifies MPI," - ' specify MPI, but mpi_processes are specified in its' + " specify MPI, but mpi_processes are specified in its" ' resources. Please either set "execution_model" to' - ' an MPI model, or specify a number of threads.' + " an MPI model, or specify a number of threads." ) else: if isinstance(res, ThreadedResReq): raise RuntimeError( f"Model component {comp}'s implementation specifies MPI," - ' but threads are specified in its resources. Please' + " but threads are specified in its resources. Please" ' either set "execution_model" to "direct", or' - ' specify a number of mpi processes.') + " specify a number of mpi processes." + ) @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: Document._yatiml_recognize(node) - node.require_attribute('model', Model) - node.require_attribute('implementations') - node.require_attribute('resources') + node.require_attribute("model", Model) + node.require_attribute("implementations") + node.require_attribute("resources") diff --git a/ymmsl/v0_1/document.py b/ymmsl/v0_1/document.py index 6387f06..9d0a740 100644 --- a/ymmsl/v0_1/document.py +++ b/ymmsl/v0_1/document.py @@ -1,4 +1,5 @@ """Defines the YAML document and version tag.""" + import yatiml from ymmsl.document import Document as DocumentBase @@ -10,18 +11,19 @@ class Document(DocumentBase): This gets specialised by a top-level content class, and takes care of a special ymmsl_version attribute in the root of the YAML file. """ + def __init__(self) -> None: pass @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: node.require_mapping() - node.require_attribute('ymmsl_version') - node.require_attribute_value('ymmsl_version', 'v0.1') + node.require_attribute("ymmsl_version") + node.require_attribute_value("ymmsl_version", "v0.1") @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - node.set_attribute('ymmsl_version', 'v0.1') + node.set_attribute("ymmsl_version", "v0.1") # The above adds the attribute to the end, but we want it at # the top; this moves it there. node.yaml_node.value.insert(0, node.yaml_node.value[-1]) @@ -29,4 +31,4 @@ def _yatiml_sweeten(cls, node: yatiml.Node) -> None: @classmethod def _yatiml_savorize(cls, node: yatiml.Node) -> None: - node.remove_attribute('ymmsl_version') + node.remove_attribute("ymmsl_version") diff --git a/ymmsl/v0_1/execution.py b/ymmsl/v0_1/execution.py index 0ae4afd..418b377 100644 --- a/ymmsl/v0_1/execution.py +++ b/ymmsl/v0_1/execution.py @@ -1,4 +1,5 @@ """Definitions for specifying how to start a component.""" + from enum import Enum from pathlib import Path from typing import Dict, List, Optional, Union, cast @@ -52,6 +53,7 @@ class BaseEnv(Enum): the active environment. """ + LOGIN = 1 """The environment you get after logging in when using bash.""" CLEAN = 2 @@ -103,6 +105,7 @@ def _yatiml_sweeten(cls, node: yatiml.Node) -> None: class ExecutionModel(Enum): """Describes how to start a model component.""" + DIRECT = 1 """Start directly on the allocated core(s), without MPI.""" OPENMPI = 2 @@ -180,20 +183,19 @@ class Implementation: """ def __init__( - self, - name: Reference, - base_env: Optional[BaseEnv] = None, - modules: Union[str, List[str], None] = None, - virtual_env: Optional[Path] = None, - env: Optional[Dict[str, str]] = None, - execution_model: ExecutionModel = ExecutionModel.DIRECT, - executable: Optional[Path] = None, - args: Union[str, List[str], None] = None, - script: Union[str, List[str], None] = None, - can_share_resources: bool = True, - keeps_state_for_next_use: KeepsStateForNextUse - = KeepsStateForNextUse.NECESSARY - ) -> None: + self, + name: Reference, + base_env: Optional[BaseEnv] = None, + modules: Union[str, List[str], None] = None, + virtual_env: Optional[Path] = None, + env: Optional[Dict[str, str]] = None, + execution_model: ExecutionModel = ExecutionModel.DIRECT, + executable: Optional[Path] = None, + args: Union[str, List[str], None] = None, + script: Union[str, List[str], None] = None, + can_share_resources: bool = True, + keeps_state_for_next_use: KeepsStateForNextUse = KeepsStateForNextUse.NECESSARY, + ) -> None: """Create an Implementation description. An Implementation normally has an ``executable`` and any other @@ -241,29 +243,30 @@ def __init__( err_arg.append('"args"') if err_arg: raise RuntimeError( - 'When creating an Implementation, script was specified' - f' together with the arguments {", ".join(err_arg)},' - ' which is not supported, as they are supposed to be' - ' inside the script if there is one. Please use either' - ' a script or the arguments listed above.') + "When creating an Implementation, script was specified" + f" together with the arguments {', '.join(err_arg)}," + " which is not supported, as they are supposed to be" + " inside the script if there is one. Please use either" + " a script or the arguments listed above." + ) if executable is None and script is None: raise RuntimeError( - f'In {name}, neither a script nor an executable was given.' - ' Please specify either a script, or the other parameters.' - ) + f"In {name}, neither a script nor an executable was given." + " Please specify either a script, or the other parameters." + ) self.name = name if isinstance(script, list): - self.script: Optional[str] = '\n'.join(script) + '\n' + self.script: Optional[str] = "\n".join(script) + "\n" else: self.script = script self.base_env = base_env if base_env else BaseEnv.MANAGER if isinstance(modules, str): - self.modules: Optional[list[str]] = modules.split(' ') + self.modules: Optional[list[str]] = modules.split(" ") else: self.modules = modules self.virtual_env = virtual_env @@ -289,38 +292,39 @@ def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: @classmethod def _yatiml_savorize(cls, node: yatiml.Node) -> None: - if node.has_attribute('env'): - env_node = node.get_attribute('env') + if node.has_attribute("env"): + env_node = node.get_attribute("env") if env_node.is_mapping(): for _, value_node in env_node.yaml_node.value: if isinstance(value_node, yaml.ScalarNode): - if value_node.tag == 'tag:yaml.org,2002:int': - value_node.tag = 'tag:yaml.org,2002:str' - if value_node.tag == 'tag:yaml.org,2002:float': - value_node.tag = 'tag:yaml.org,2002:str' - if value_node.tag == 'tag:yaml.org,2002:bool': - value_node.tag = 'tag:yaml.org,2002:str' + if value_node.tag == "tag:yaml.org,2002:int": + value_node.tag = "tag:yaml.org,2002:str" + if value_node.tag == "tag:yaml.org,2002:float": + value_node.tag = "tag:yaml.org,2002:str" + if value_node.tag == "tag:yaml.org,2002:bool": + value_node.tag = "tag:yaml.org,2002:str" _yatiml_defaults = { - 'base_env': 'manager', - 'execution_model': 'direct', - 'keeps_state_for_next_use': 'necessary'} + "base_env": "manager", + "execution_model": "direct", + "keeps_state_for_next_use": "necessary", + } @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - if node.has_attribute('script'): - script_node = node.get_attribute('script') + if node.has_attribute("script"): + script_node = node.get_attribute("script") if script_node.is_scalar(str): text = cast(str, script_node.get_value()) - if '\n' in text: - cast(yaml.ScalarNode, script_node.yaml_node).style = '|' + if "\n" in text: + cast(yaml.ScalarNode, script_node.yaml_node).style = "|" node.remove_attributes_with_default_values(cls) - if node.has_attribute('env'): - env_attr = node.get_attribute('env') + if node.has_attribute("env"): + env_attr = node.get_attribute("env") if env_attr.is_mapping(): if env_attr.is_empty(): - node.remove_attribute('env') + node.remove_attribute("env") class ResourceRequirements: @@ -333,6 +337,7 @@ class ResourceRequirements: Attributes: name: Name of the component to configure. """ + def __init__(self, name: Reference) -> None: """Create a ResourceRequirements description. @@ -344,7 +349,8 @@ def __init__(self, name: Reference) -> None: @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: raise yatiml.RecognitionError( - 'Please specify either "threads" or "mpi_processes".') + 'Please specify either "threads" or "mpi_processes".' + ) class ThreadedResReq(ResourceRequirements): @@ -386,8 +392,8 @@ class MPICoresResReq(ResourceRequirements): """ def __init__( - self, name: Reference, mpi_processes: int, - threads_per_mpi_process: int = 1) -> None: + self, name: Reference, mpi_processes: int, threads_per_mpi_process: int = 1 + ) -> None: """Create a ThreadedMPIResourceRequirements description. Args: @@ -420,9 +426,12 @@ class MPINodesResReq(ResourceRequirements): """ def __init__( - self, name: Reference, nodes: int, - mpi_processes_per_node: int, threads_per_mpi_process: int = 1 - ) -> None: + self, + name: Reference, + nodes: int, + mpi_processes_per_node: int, + threads_per_mpi_process: int = 1, + ) -> None: """Create a NodeBasedMPIResourceRequirements description. Args: diff --git a/ymmsl/v0_1/identity.py b/ymmsl/v0_1/identity.py index 41b5378..f50fe28 100644 --- a/ymmsl/v0_1/identity.py +++ b/ymmsl/v0_1/identity.py @@ -1,4 +1,5 @@ """This module contains definitions for identity.""" + import re from collections import UserString from copy import copy @@ -26,12 +27,14 @@ def __init__(self, seq: Any) -> None: """ super().__init__(seq) - if not re.fullmatch(r'[a-zA-Z_]\w*', self.data, flags=re.ASCII): - raise ValueError('Identifiers must consist only of' - ' lower- and uppercase letters, digits and' - ' underscores, must start with a letter or' - ' an underscore, and must not be empty.' - f' "{self.data}" is therefore invalid.') + if not re.fullmatch(r"[a-zA-Z_]\w*", self.data, flags=re.ASCII): + raise ValueError( + "Identifiers must consist only of" + " lower- and uppercase letters, digits and" + " underscores, must start with a letter or" + " an underscore, and must not be empty." + f' "{self.data}" is therefore invalid.' + ) ReferencePart = Union[Identifier, int] @@ -82,8 +85,7 @@ def __init__(self, parts: Union[str, List[ReferencePart]]) -> None: if isinstance(parts, str): self._parts = self._string_to_parts(parts) elif len(parts) > 0 and not isinstance(parts[0], Identifier): - raise ValueError( - 'The first part of a Reference must be an Identifier') + raise ValueError("The first part of a Reference must be an Identifier") else: self._parts = parts @@ -153,7 +155,7 @@ def __lt__(self, other: Any) -> bool: if o < s: return False elif isinstance(s, Identifier) and isinstance(o, Identifier): - if s < o: # repeated because mypy is dumb + if s < o: # repeated because mypy is dumb return True if o < s: return False @@ -182,11 +184,9 @@ def __iter__(self) -> Generator[ReferencePart, None, None]: def __getitem__(self, key: int) -> ReferencePart: ... @overload - def __getitem__(self, key: slice) -> 'Reference': ... + def __getitem__(self, key: slice) -> "Reference": ... - def __getitem__( - self, key: Union[int, slice] - ) -> Union['Reference', ReferencePart]: + def __getitem__(self, key: Union[int, slice]) -> Union["Reference", ReferencePart]: """Get a part or a slice. If passed an int, e.g. ref[2], will return that part as an int @@ -208,7 +208,7 @@ def __getitem__( return self._parts[key] if isinstance(key, slice): return Reference(self._parts[key]) - raise ValueError('Subscript must be either an int or a slice') + raise ValueError("Subscript must be either an int or a slice") def __setitem__(self, key: Union[int, slice], value: Any) -> None: """Does not set the value of a part. @@ -220,11 +220,13 @@ def __setitem__(self, key: Union[int, slice], value: Any) -> None: RuntimeError: Always. """ - raise RuntimeError('Reference objects are immutable, please don\'t try' - ' to change them.') + raise RuntimeError( + "Reference objects are immutable, please don't try to change them." + ) - def __add__(self, other: Union['Reference', Iterable[ReferencePart], - ReferencePart]) -> 'Reference': + def __add__( + self, other: Union["Reference", Iterable[ReferencePart], ReferencePart] + ) -> "Reference": """Concatenates something onto a Reference. The object to add on can be another Reference, a list of @@ -244,11 +246,11 @@ def __add__(self, other: Union['Reference', Iterable[ReferencePart], ret._parts.extend(other._parts) elif isinstance(other, (Identifier, int)): ret._parts.append(other) - elif hasattr(other, '__iter__'): + elif hasattr(other, "__iter__"): ret._parts.extend(other) return ret - def without_trailing_ints(self) -> 'Reference': + def without_trailing_ints(self) -> "Reference": """Returns a copy of this Reference with trailing ints removed. Examples: @@ -260,7 +262,7 @@ def without_trailing_ints(self) -> 'Reference': i = len(self._parts) - 1 while i > 0 and isinstance(self._parts[i], int): i -= 1 - return Reference(self._parts[0:i+1]) + return Reference(self._parts[0 : i + 1]) @classmethod def _string_to_parts(cls, text: str) -> List[ReferencePart]: @@ -274,11 +276,12 @@ def _string_to_parts(cls, text: str) -> List[ReferencePart]: Reference. """ + def find_next_op(text: str, start: int) -> int: - next_bracket = text.find('[', start) + next_bracket = text.find("[", start) if next_bracket == -1: next_bracket = len(text) - next_period = text.find('.', start) + next_period = text.find(".", start) if next_period == -1: next_period = len(text) return min(next_period, next_bracket) @@ -287,16 +290,16 @@ def find_next_op(text: str, start: int) -> int: cur_op = find_next_op(text, 0) parts: list[ReferencePart] = [Identifier(text[0:cur_op])] while cur_op < end: - if text[cur_op] == '.': + if text[cur_op] == ".": next_op = find_next_op(text, cur_op + 1) - parts.append(Identifier(text[cur_op + 1:next_op])) + parts.append(Identifier(text[cur_op + 1 : next_op])) cur_op = next_op - elif text[cur_op] == '[': - close_bracket = text.find(']', cur_op) + elif text[cur_op] == "[": + close_bracket = text.find("]", cur_op) if close_bracket == -1: raise ValueError(f"Missing closing bracket in Reference {text}") try: - index = int(text[cur_op + 1:close_bracket]) + index = int(text[cur_op + 1 : close_bracket]) except ValueError as exc: raise ValueError( f"Invalid index '{text[cur_op + 1 : close_bracket]}' in " @@ -305,8 +308,10 @@ def find_next_op(text: str, start: int) -> int: parts.append(index) cur_op = close_bracket + 1 else: - raise ValueError(f"Invalid character '{text[cur_op]}' encountered in" - f" Reference {text}") + raise ValueError( + f"Invalid character '{text[cur_op]}' encountered in" + f" Reference {text}" + ) return parts @classmethod @@ -321,7 +326,7 @@ def _parts_to_string(cls, parts: List[ReferencePart]) -> str: """ if len(parts) == 0: - return '[]' + return "[]" text = str(parts[0]) for part in parts[1:]: diff --git a/ymmsl/v0_1/model.py b/ymmsl/v0_1/model.py index 0afdf9e..c1d5b80 100644 --- a/ymmsl/v0_1/model.py +++ b/ymmsl/v0_1/model.py @@ -1,4 +1,5 @@ """This module contains all the definitions for yMMSL.""" + from collections import OrderedDict from typing import ( Any, @@ -66,20 +67,22 @@ def __check_reference(ref: Reference) -> None: # check that subscripts are at the end for i, part in enumerate(ref): if isinstance(part, int): - if (i+1) < len(ref) and isinstance(ref[i+1], Identifier): - raise ValueError(f"Reference {ref} contains a subscript that" - " is not at the end, which is not allowed" - " in conduits.") + if (i + 1) < len(ref) and isinstance(ref[i + 1], Identifier): + raise ValueError( + f"Reference {ref} contains a subscript that" + " is not at the end, which is not allowed" + " in conduits." + ) # check that the length is at least 2 if len(Conduit.__stem(ref)) < 2: raise ValueError( - "Senders and receivers in conduits must have a component" - " name, a period, and then a port name and optionally a" - f" slot. Reference {ref} is missing either the component or" - " the port. Did you perhaps type a comma or an underscore" - ' instead of a period? It should be "component.port"' - ) + "Senders and receivers in conduits must have a component" + " name, a period, and then a port name and optionally a" + f" slot. Reference {ref} is missing either the component or" + " the port. Did you perhaps type a comma or an underscore" + ' instead of a period? It should be "component.port"' + ) def sending_component(self) -> Reference: """Returns a reference to the sending component.""" @@ -142,7 +145,7 @@ def __stem(reference: Reference) -> Reference: If there is no slot, returns the whole reference. """ i = len(reference) - while isinstance(reference[i-1], int): + while isinstance(reference[i - 1], int): i -= 1 return reference[:i] @@ -197,6 +200,7 @@ class ModelReference: name: The name of the simulation model this refers to. """ + def __init__(self, name: str) -> None: """Create a ModelReference. @@ -226,9 +230,13 @@ class Model(ModelReference): conduits: A list of conduits connecting the components. """ - def __init__(self, name: str, - components: List[Component], - conduits: Optional[Sequence[AnyConduit]] = None) -> None: + + def __init__( + self, + name: str, + components: List[Component], + conduits: Optional[Sequence[AnyConduit]] = None, + ) -> None: """Create a Model. Args: @@ -247,7 +255,7 @@ def __init__(self, name: str, if isinstance(conduit, MulticastConduit): self.conduits.extend(conduit.as_conduits()) - def update(self, overlay: 'Model') -> None: + def update(self, overlay: "Model") -> None: """Overlay another model definition on top of this one. This updates the object with the name, components and conduits @@ -290,6 +298,7 @@ def check_consistent(self) -> None: components, and will raise a RuntimeError with an explanation if one is not. """ + def component_exists(name: Reference) -> bool: for comp in self.components: if comp.name == name: @@ -297,8 +306,9 @@ def component_exists(name: Reference) -> bool: return False def component_has_receiving_port( - component: Reference, port: Identifier) -> bool: - if port == 'muscle_settings_in': + component: Reference, port: Identifier + ) -> bool: + if port == "muscle_settings_in": return True for comp in self.components: if comp.name == component: @@ -312,8 +322,7 @@ def component_has_receiving_port( pass return False - def component_has_sending_port( - component: Reference, port: Identifier) -> bool: + def component_has_sending_port(component: Reference, port: Identifier) -> bool: for comp in self.components: if comp.name == component: if not comp.ports: @@ -330,30 +339,33 @@ def component_has_sending_port( for conduit in self.conduits: scomp = conduit.sending_component() if not component_exists(scomp): - raise RuntimeError( - f'Unknown sending component "{scomp}" of {conduit}') + raise RuntimeError(f'Unknown sending component "{scomp}" of {conduit}') rcomp = conduit.receiving_component() if not component_exists(rcomp): raise RuntimeError( - f'Unknown receiving component "{rcomp}" of {conduit}') + f'Unknown receiving component "{rcomp}" of {conduit}' + ) sport = conduit.sending_port() if not component_has_sending_port(scomp, sport): raise RuntimeError( - f'Invalid conduit "{conduit}": component "{scomp}" does not' - f' have a sending port "{sport}"') + f'Invalid conduit "{conduit}": component "{scomp}" does not' + f' have a sending port "{sport}"' + ) rport = conduit.receiving_port() if not component_has_receiving_port(rcomp, rport): raise RuntimeError( - f'Invalid conduit "{conduit}": component "{rcomp}" does not' - f' have a receiving port "{rport}"') + f'Invalid conduit "{conduit}": component "{rcomp}" does not' + f' have a receiving port "{rport}"' + ) if conduit.receiver in receivers_seen: raise RuntimeError( - f'Receiving port "{conduit.receiver}" is connected by multiple' - " conduits.") + f'Receiving port "{conduit.receiver}" is connected by multiple' + " conduits." + ) receivers_seen.add(conduit.receiver) def __conduits_for_export(self) -> List[AnyConduit]: @@ -370,31 +382,36 @@ def __conduits_for_export(self) -> List[AnyConduit]: if len(conduits) == 1: conduit_list.append(conduits[0]) else: - conduit_list.append(MulticastConduit( - str(sender), - [str(conduit.receiver) for conduit in conduits])) + conduit_list.append( + MulticastConduit( + str(sender), [str(conduit.receiver) for conduit in conduits] + ) + ) return conduit_list def _yatiml_attributes(self) -> OrderedDict: - return OrderedDict([ - ('name', self.name), - ('components', self.components), - ('conduits', self.__conduits_for_export())]) + return OrderedDict( + [ + ("name", self.name), + ("components", self.components), + ("conduits", self.__conduits_for_export()), + ] + ) @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: node.require_mapping() - node.require_attribute('name') - node.require_attribute('components') + node.require_attribute("name") + node.require_attribute("components") @classmethod def _yatiml_savorize(cls, node: yatiml.Node) -> None: - node.map_attribute_to_seq('components', 'name', 'implementation') - node.map_attribute_to_seq('conduits', 'sender', 'receiver') + node.map_attribute_to_seq("components", "name", "implementation") + node.map_attribute_to_seq("conduits", "sender", "receiver") @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - node.seq_attribute_to_map('components', 'name', 'implementation') - if len(node.get_attribute('conduits').seq_items()) == 0: - node.remove_attribute('conduits') - node.seq_attribute_to_map('conduits', 'sender', 'receiver') + node.seq_attribute_to_map("components", "name", "implementation") + if len(node.get_attribute("conduits").seq_items()) == 0: + node.remove_attribute("conduits") + node.seq_attribute_to_map("conduits", "sender", "receiver") diff --git a/ymmsl/v0_1/settings.py b/ymmsl/v0_1/settings.py index 5905d3a..cf94f39 100644 --- a/ymmsl/v0_1/settings.py +++ b/ymmsl/v0_1/settings.py @@ -1,4 +1,5 @@ """Definitions for specifying model settings.""" + from collections import OrderedDict from collections.abc import MutableMapping from copy import deepcopy @@ -9,11 +10,18 @@ from ymmsl.v0_1.identity import Reference SettingValue = Union[ - str, int, float, bool, List[int], List[float], List[List[float]], - yatiml.bool_union_fix] + str, + int, + float, + bool, + List[int], + List[float], + List[List[float]], + yatiml.bool_union_fix, +] -_T = TypeVar('_T') +_T = TypeVar("_T") class Settings(MutableMapping): @@ -23,10 +31,7 @@ class Settings(MutableMapping): for the submodel scales, model parameters and any other configuration. """ - def __init__( - self, - settings: Optional[Dict[str, SettingValue]] = None - ) -> None: + def __init__(self, settings: Optional[Dict[str, SettingValue]] = None) -> None: """Create a Settings object. This will make a deep copy of the settings argument, if @@ -69,8 +74,7 @@ def __getitem__(self, key: Union[str, Reference]) -> SettingValue: key = Reference(key) return self._store[key] - def __setitem__(self, key: Union[str, Reference], value: SettingValue - ) -> None: + def __setitem__(self, key: Union[str, Reference], value: SettingValue) -> None: """Sets a value, implements settings[name] = value.""" if isinstance(key, str): key = Reference(key) @@ -94,11 +98,9 @@ def __len__(self) -> int: def get(self, key: Any, /) -> Union[Any, None]: ... @overload - def get( - self, key: Any, /, default: _T) -> Union[Any, _T]: ... + def get(self, key: Any, /, default: _T) -> Union[Any, _T]: ... - def get( - self, key: Any, /, default: Union[_T, None] = None) -> Union[Any, _T]: + def get(self, key: Any, /, default: Union[_T, None] = None) -> Union[Any, _T]: """Return the given setting, or default if it is not set. If default is not given, returns None. @@ -116,7 +118,7 @@ def ordered_items(self) -> List[Tuple[Reference, SettingValue]]: result.append((key, value)) return result - def copy(self) -> 'Settings': + def copy(self) -> "Settings": """Makes a shallow copy of these settings and returns it.""" new_settings = Settings() new_settings._store = self._store.copy() @@ -147,16 +149,16 @@ def _yatiml_savorize(cls, node: yatiml.Node) -> None: # wrap the existing mapping into a new mapping with attribute settings setting_values = node.yaml_node node.make_mapping() - node.set_attribute('settings', setting_values) + node.set_attribute("settings", setting_values) @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: # format lists and arrays nicely for _, value_node in node.yaml_node.value: - if value_node.tag == 'tag:yaml.org,2002:seq': + if value_node.tag == "tag:yaml.org,2002:seq": # this attribute is a list or list-of-list if len(value_node.value) > 0: - if value_node.value[0].tag != 'tag:yaml.org,2002:seq': + if value_node.value[0].tag != "tag:yaml.org,2002:seq": value_node.flow_style = True else: value_node.flow_style = False diff --git a/ymmsl/v0_1/tests/conftest.py b/ymmsl/v0_1/tests/conftest.py index 9a10278..66f09f3 100644 --- a/ymmsl/v0_1/tests/conftest.py +++ b/ymmsl/v0_1/tests/conftest.py @@ -26,95 +26,104 @@ @pytest.fixture def test_config2() -> PartialConfiguration: model = Model( - 'test_model', - [ - Component('ic', 'isr2d.initial_conditions'), - Component('smc', 'isr2d.smc'), - Component('bf', 'isr2d.blood_flow'), - Component('smc2bf', 'isr2d.smc2bf'), - Component('bf2smc', 'isr2d.bf2smc')], - [ - Conduit('ic.out', 'smc.initial_state'), - Conduit('smc.cell_positions', 'smc2bf.in'), - Conduit('smc2bf.out', 'bf.initial_domain'), - Conduit('bf.wss_out', 'bf2smc.in'), - Conduit('bf2smc.out', 'smc.wss_in')]) + "test_model", + [ + Component("ic", "isr2d.initial_conditions"), + Component("smc", "isr2d.smc"), + Component("bf", "isr2d.blood_flow"), + Component("smc2bf", "isr2d.smc2bf"), + Component("bf2smc", "isr2d.bf2smc"), + ], + [ + Conduit("ic.out", "smc.initial_state"), + Conduit("smc.cell_positions", "smc2bf.in"), + Conduit("smc2bf.out", "bf.initial_domain"), + Conduit("bf.wss_out", "bf2smc.in"), + Conduit("bf2smc.out", "smc.wss_in"), + ], + ) return PartialConfiguration(model) @pytest.fixture def test_config4() -> PartialConfiguration: implementations = [ - Implementation( - Reference('isr2d.initial_conditions'), script='isr2d/bin/ic'), - Implementation( - Reference('isr2d.smc'), script='isr2d/bin/smc'), - Implementation( - Reference('isr2d.blood_flow'), script='isr2d/bin/bf'), - Implementation( - Reference('isr2d.smc2bf'), script='isr2d/bin/smc2bf.py'), - Implementation( - Reference('isr2d.bf2smc'), script='isr2d/bin/bf2smc.py')] + Implementation(Reference("isr2d.initial_conditions"), script="isr2d/bin/ic"), + Implementation(Reference("isr2d.smc"), script="isr2d/bin/smc"), + Implementation(Reference("isr2d.blood_flow"), script="isr2d/bin/bf"), + Implementation(Reference("isr2d.smc2bf"), script="isr2d/bin/smc2bf.py"), + Implementation(Reference("isr2d.bf2smc"), script="isr2d/bin/bf2smc.py"), + ] resources = [ - ThreadedResReq(Reference('ic'), 4), - ThreadedResReq(Reference('smc'), 4), - MPICoresResReq(Reference('bf'), 4), - ThreadedResReq(Reference('smc2bf'), 1), - ThreadedResReq(Reference('bf2smc'), 1)] + ThreadedResReq(Reference("ic"), 4), + ThreadedResReq(Reference("smc"), 4), + MPICoresResReq(Reference("bf"), 4), + ThreadedResReq(Reference("smc2bf"), 1), + ThreadedResReq(Reference("bf2smc"), 1), + ] description = "Multiline description for\nthis workflow" checkpoints = Checkpoints( - True, - [CheckpointRangeRule(every=100), - CheckpointAtRule([10, 20, 50])], - [CheckpointRangeRule(start=0, stop=10, every=2), - CheckpointRangeRule(start=10, every=5)]) - resume = {Ref('ic'): Path('/path/to/snapshots/ic.pack'), - Ref('smc'): Path('/path/to/snapshots/smc.pack'), - Ref('bf'): Path('/path/to/snapshots/bf.pack'), - Ref('smc2bf'): Path('/path/to/snapshots/smc2bf.pack'), - Ref('bf2smc'): Path('/path/to/snapshots/bf2smc.pack')} + True, + [CheckpointRangeRule(every=100), CheckpointAtRule([10, 20, 50])], + [ + CheckpointRangeRule(start=0, stop=10, every=2), + CheckpointRangeRule(start=10, every=5), + ], + ) + resume = { + Ref("ic"): Path("/path/to/snapshots/ic.pack"), + Ref("smc"): Path("/path/to/snapshots/smc.pack"), + Ref("bf"): Path("/path/to/snapshots/bf.pack"), + Ref("smc2bf"): Path("/path/to/snapshots/smc2bf.pack"), + Ref("bf2smc"): Path("/path/to/snapshots/bf2smc.pack"), + } - return PartialConfiguration(None, None, implementations, resources, - description, checkpoints, resume) + return PartialConfiguration( + None, None, implementations, resources, description, checkpoints, resume + ) @pytest.fixture def test_config6() -> Configuration: model = Model( - 'resources_test', - [ - Component('singlethreaded', 'a'), - Component('multithreaded', 'b'), - Component('mpi_cores1', 'c'), - Component('mpi_cores2', 'd'), - Component('mpi_nodes1', 'c'), - Component('mpi_nodes2', 'd')], - []) + "resources_test", + [ + Component("singlethreaded", "a"), + Component("multithreaded", "b"), + Component("mpi_cores1", "c"), + Component("mpi_cores2", "d"), + Component("mpi_nodes1", "c"), + Component("mpi_nodes2", "d"), + ], + [], + ) implementations = [ - Implementation( - Reference('a'), script='/home/user/models/bin/modela'), - Implementation( - Reference('b'), script='/home/user/models/bin/modelb'), - Implementation( - Reference('c'), - base_env=BaseEnv.LOGIN, - modules=['gcc-6.3.0', 'openmpi-1.10'], - execution_model=ExecutionModel.OPENMPI, - executable=Path('/home/user/models/bin/modelc')), - Implementation( - Reference('d'), - base_env=BaseEnv.CLEAN, - modules=['icc-18.0', 'IntelMPI-2021-3'], - execution_model=ExecutionModel.INTELMPI, - executable=Path('/home/user/models/bin/modeld'))] + Implementation(Reference("a"), script="/home/user/models/bin/modela"), + Implementation(Reference("b"), script="/home/user/models/bin/modelb"), + Implementation( + Reference("c"), + base_env=BaseEnv.LOGIN, + modules=["gcc-6.3.0", "openmpi-1.10"], + execution_model=ExecutionModel.OPENMPI, + executable=Path("/home/user/models/bin/modelc"), + ), + Implementation( + Reference("d"), + base_env=BaseEnv.CLEAN, + modules=["icc-18.0", "IntelMPI-2021-3"], + execution_model=ExecutionModel.INTELMPI, + executable=Path("/home/user/models/bin/modeld"), + ), + ] resources = [ - ThreadedResReq(Reference('singlethreaded'), 1), - ThreadedResReq(Reference('multithreaded'), 8), - MPICoresResReq(Reference('mpi_cores1'), 16), - MPICoresResReq(Reference('mpi_cores2'), 4, 4), - MPINodesResReq(Reference('mpi_nodes1'), 10, 16), - MPINodesResReq(Reference('mpi_nodes2'), 10, 4, 4)] + ThreadedResReq(Reference("singlethreaded"), 1), + ThreadedResReq(Reference("multithreaded"), 8), + MPICoresResReq(Reference("mpi_cores1"), 16), + MPICoresResReq(Reference("mpi_cores2"), 4, 4), + MPINodesResReq(Reference("mpi_nodes1"), 10, 16), + MPINodesResReq(Reference("mpi_nodes2"), 10, 4, 4), + ] return Configuration(model, None, implementations, resources) diff --git a/ymmsl/v0_1/tests/test_checkpoint.py b/ymmsl/v0_1/tests/test_checkpoint.py index e189eb0..8577e9f 100644 --- a/ymmsl/v0_1/tests/test_checkpoint.py +++ b/ymmsl/v0_1/tests/test_checkpoint.py @@ -54,7 +54,8 @@ def test_checkpoints() -> None: def test_checkpointrules_update() -> None: load = yatiml.load_function( - Checkpoints, CheckpointRangeRule, CheckpointAtRule, CheckpointRule) + Checkpoints, CheckpointRangeRule, CheckpointAtRule, CheckpointRule + ) rule1 = load("simulation_time: [{every: 300}]") rule2 = load("wallclock_time: [{at: [10, 20]}]") rule3 = load("wallclock_time: [{at: [15, 5]}]") diff --git a/ymmsl/v0_1/tests/test_component.py b/ymmsl/v0_1/tests/test_component.py index c402db1..eb0c368 100644 --- a/ymmsl/v0_1/tests/test_component.py +++ b/ymmsl/v0_1/tests/test_component.py @@ -20,84 +20,89 @@ def test_operator() -> None: def test_port() -> None: - ep1 = Port(Identifier('test_in'), Operator.F_INIT) + ep1 = Port(Identifier("test_in"), Operator.F_INIT) - assert str(ep1.name) == 'test_in' + assert str(ep1.name) == "test_in" assert ep1.operator == Operator.F_INIT def test_ports() -> None: - p1 = Ports(['initial_state'], ['obs_i'], ['bc_i'], ['final_output']) - p2 = Ports(f_init=['input'], o_f=['output']) - p3 = Ports(['init1', 'init2'], ['obs1', 'obs2', 'obs3']) + p1 = Ports(["initial_state"], ["obs_i"], ["bc_i"], ["final_output"]) + p2 = Ports(f_init=["input"], o_f=["output"]) + p3 = Ports(["init1", "init2"], ["obs1", "obs2", "obs3"]) - assert set(p1.port_names()) == { - 'initial_state', 'obs_i', 'bc_i', 'final_output'} - assert set(p2.port_names()) == {'input', 'output'} - assert set(p3.port_names()) == {'init1', 'init2', 'obs1', 'obs2', 'obs3'} + assert set(p1.port_names()) == {"initial_state", "obs_i", "bc_i", "final_output"} + assert set(p2.port_names()) == {"input", "output"} + assert set(p3.port_names()) == {"init1", "init2", "obs1", "obs2", "obs3"} def in_ports(ports: Iterable[Port], name: str, op: Operator) -> bool: return any(map(lambda p: p.name == name and p.operator == op, ports)) assert len(list(p1.all_ports())) == 4 - assert in_ports(p1.all_ports(), 'initial_state', Operator.F_INIT) - assert in_ports(p1.all_ports(), 'obs_i', Operator.O_I) - assert in_ports(p1.all_ports(), 'bc_i', Operator.S) - assert in_ports(p1.all_ports(), 'final_output', Operator.O_F) + assert in_ports(p1.all_ports(), "initial_state", Operator.F_INIT) + assert in_ports(p1.all_ports(), "obs_i", Operator.O_I) + assert in_ports(p1.all_ports(), "bc_i", Operator.S) + assert in_ports(p1.all_ports(), "final_output", Operator.O_F) assert len(list(p2.all_ports())) == 2 - assert in_ports(p2.all_ports(), 'input', Operator.F_INIT) - assert in_ports(p2.all_ports(), 'output', Operator.O_F) + assert in_ports(p2.all_ports(), "input", Operator.F_INIT) + assert in_ports(p2.all_ports(), "output", Operator.O_F) assert len(list(p3.all_ports())) == 5 - assert in_ports(p3.all_ports(), 'init1', Operator.F_INIT) - assert in_ports(p3.all_ports(), 'init2', Operator.F_INIT) - assert in_ports(p3.all_ports(), 'obs1', Operator.O_I) - assert in_ports(p3.all_ports(), 'obs2', Operator.O_I) - assert in_ports(p3.all_ports(), 'obs3', Operator.O_I) - - assert p1.operator(Identifier('initial_state')) == Operator.F_INIT - assert p1.operator(Identifier('bc_i')) == Operator.S - assert p2.operator(Identifier('output')) == Operator.O_F - assert p3.operator(Identifier('init2')) == Operator.F_INIT - assert p3.operator(Identifier('obs3')) == Operator.O_I + assert in_ports(p3.all_ports(), "init1", Operator.F_INIT) + assert in_ports(p3.all_ports(), "init2", Operator.F_INIT) + assert in_ports(p3.all_ports(), "obs1", Operator.O_I) + assert in_ports(p3.all_ports(), "obs2", Operator.O_I) + assert in_ports(p3.all_ports(), "obs3", Operator.O_I) + + assert p1.operator(Identifier("initial_state")) == Operator.F_INIT + assert p1.operator(Identifier("bc_i")) == Operator.S + assert p2.operator(Identifier("output")) == Operator.O_F + assert p3.operator(Identifier("init2")) == Operator.F_INIT + assert p3.operator(Identifier("obs3")) == Operator.O_I with pytest.raises(KeyError): - p3.operator(Identifier('x')) + p3.operator(Identifier("x")) def test_component_declaration() -> None: - test_decl = Component('test', 'ns.model') - assert str(test_decl.name) == 'test' - assert str(test_decl.implementation) == 'ns.model' + test_decl = Component("test", "ns.model") + assert str(test_decl.name) == "test" + assert str(test_decl.implementation) == "ns.model" assert test_decl.multiplicity == [] - assert str(test_decl) == 'test' + assert str(test_decl) == "test" - test_decl = Component('test', 'ns.model', 10) + test_decl = Component("test", "ns.model", 10) assert isinstance(test_decl.name, Reference) - assert str(test_decl.name) == 'test' + assert str(test_decl.name) == "test" assert test_decl.multiplicity == [10] - assert str(test_decl) == 'test[0:10]' + assert str(test_decl) == "test[0:10]" - test_decl = Component('test', 'ns2.model2', [1, 2]) + test_decl = Component("test", "ns2.model2", [1, 2]) assert isinstance(test_decl.name, Reference) - assert str(test_decl.name) == 'test' - assert str(test_decl.implementation) == 'ns2.model2' + assert str(test_decl.name) == "test" + assert str(test_decl.implementation) == "ns2.model2" assert test_decl.multiplicity == [1, 2] - assert str(test_decl) == 'test[0:1][0:2]' + assert str(test_decl) == "test[0:1][0:2]" with pytest.raises(ValueError): - test_decl = Component('test', 'ns2.model2[1]') + test_decl = Component("test", "ns2.model2[1]") def test_component_instances() -> None: - c1 = Component('test', 'model') - assert c1.instances() == [Reference('test')] + c1 = Component("test", "model") + assert c1.instances() == [Reference("test")] - c2 = Component('test', 'model', 3) + c2 = Component("test", "model", 3) assert c2.instances() == [ - Reference('test[0]'), Reference('test[1]'), Reference('test[2]')] + Reference("test[0]"), + Reference("test[1]"), + Reference("test[2]"), + ] - c3 = Component('test', 'model', [2, 2]) + c3 = Component("test", "model", [2, 2]) assert c3.instances() == [ - Reference('test[0][0]'), Reference('test[0][1]'), - Reference('test[1][0]'), Reference('test[1][1]')] + Reference("test[0][0]"), + Reference("test[0][1]"), + Reference("test[1][0]"), + Reference("test[1][1]"), + ] diff --git a/ymmsl/v0_1/tests/test_configuration.py b/ymmsl/v0_1/tests/test_configuration.py index 48dde2b..8df2051 100644 --- a/ymmsl/v0_1/tests/test_configuration.py +++ b/ymmsl/v0_1/tests/test_configuration.py @@ -32,9 +32,9 @@ def test_configuration() -> None: def test_configuration_update_model1() -> None: - model_ref1 = ModelReference('model1') + model_ref1 = ModelReference("model1") base = PartialConfiguration(model_ref1) - model_ref2 = ModelReference('model2') + model_ref2 = ModelReference("model2") overlay = PartialConfiguration(model_ref2) base.update(overlay) @@ -45,11 +45,11 @@ def test_configuration_update_model1() -> None: def test_configuration_update_model2() -> None: - model_ref1 = ModelReference('model1') + model_ref1 = ModelReference("model1") base = PartialConfiguration(model_ref1) - component1 = Component('macro', 'my.macro') - model2 = Model('model2', [component1]) + component1 = Component("macro", "my.macro") + model2 = Model("model2", [component1]) overlay = PartialConfiguration(model2) base.update(overlay) @@ -58,106 +58,110 @@ def test_configuration_update_model2() -> None: def test_configuration_update_model3() -> None: - component1 = Component('macro', 'my.macro') - model1 = Model('model1', [component1]) + component1 = Component("macro", "my.macro") + model1 = Model("model1", [component1]) base = Configuration(model1) - model_ref2 = ModelReference('model2') + model_ref2 = ModelReference("model2") overlay = PartialConfiguration(model_ref2) base.update(overlay) assert base.model == model1 - assert model1.name == 'model2' + assert model1.name == "model2" assert model1.components == [component1] def test_configuration_update_model4() -> None: - component1 = Component('macro', 'my.macro') - model1 = Model('model1', [component1]) + component1 = Component("macro", "my.macro") + model1 = Model("model1", [component1]) base = Configuration(model1) - component2 = Component('micro', 'my.micro') - model2 = Model('model2', [component2]) + component2 = Component("micro", "my.micro") + model2 = Model("model2", [component2]) overlay = Configuration(model2) base.update(overlay) assert isinstance(base.model, Model) assert base.model == model1 - assert base.model.name == 'model2' + assert base.model.name == "model2" assert component1 in base.model.components assert component2 in base.model.components def test_configuration_update_implementations_add() -> None: implementation1 = Implementation( - Ref('my.macro'), executable=Path('/home/test/macro.py')) + Ref("my.macro"), executable=Path("/home/test/macro.py") + ) base = PartialConfiguration(implementations=[implementation1]) implementation2 = Implementation( - Ref('my.micro'), executable=Path('/home/test/micro.py')) + Ref("my.micro"), executable=Path("/home/test/micro.py") + ) overlay = PartialConfiguration(implementations=[implementation2]) base.update(overlay) assert len(base.implementations) == 2 - assert base.implementations[Ref('my.macro')] == implementation1 - assert base.implementations[Ref('my.micro')] == implementation2 + assert base.implementations[Ref("my.macro")] == implementation1 + assert base.implementations[Ref("my.micro")] == implementation2 def test_configuration_update_implementations_override() -> None: implementation1 = Implementation( - Ref('my.macro'), executable=Path('/home/test/macro.py')) + Ref("my.macro"), executable=Path("/home/test/macro.py") + ) implementation2 = Implementation( - Ref('my.micro'), executable=Path('/home/test/micro.py')) - base = PartialConfiguration( - implementations=[implementation1, implementation2]) + Ref("my.micro"), executable=Path("/home/test/micro.py") + ) + base = PartialConfiguration(implementations=[implementation1, implementation2]) implementation3 = Implementation( - Ref('my.micro'), executable=Path('/home/test/surrogate.py')) + Ref("my.micro"), executable=Path("/home/test/surrogate.py") + ) overlay = PartialConfiguration(implementations=[implementation3]) base.update(overlay) assert len(base.implementations) == 2 - assert base.implementations[Ref('my.macro')] == implementation1 - assert base.implementations[Ref('my.micro')] == implementation3 + assert base.implementations[Ref("my.macro")] == implementation1 + assert base.implementations[Ref("my.micro")] == implementation3 def test_configuration_update_resources_add() -> None: - resources1 = ThreadedResReq(Ref('my.macro'), 10) + resources1 = ThreadedResReq(Ref("my.macro"), 10) base = PartialConfiguration(resources=[resources1]) - resources2 = ThreadedResReq(Ref('my.micro'), 2) + resources2 = ThreadedResReq(Ref("my.micro"), 2) overlay = PartialConfiguration(resources=[resources2]) base.update(overlay) assert len(base.resources) == 2 - assert base.resources[Ref('my.macro')] == resources1 - assert base.resources[Ref('my.micro')] == resources2 + assert base.resources[Ref("my.macro")] == resources1 + assert base.resources[Ref("my.micro")] == resources2 def test_configuration_update_resources_override() -> None: - resources1 = ThreadedResReq(Ref('my.macro'), 10) - resources2 = ThreadedResReq(Ref('my.micro'), 100) + resources1 = ThreadedResReq(Ref("my.macro"), 10) + resources2 = ThreadedResReq(Ref("my.micro"), 100) base = PartialConfiguration(resources=[resources1, resources2]) - resources3 = ThreadedResReq(Ref('my.micro'), 2) + resources3 = ThreadedResReq(Ref("my.micro"), 2) overlay = PartialConfiguration(resources=[resources3]) base.update(overlay) assert len(base.resources) == 2 - assert base.resources[Ref('my.macro')] == resources1 - assert base.resources[Ref('my.micro')] == resources3 + assert base.resources[Ref("my.macro")] == resources1 + assert base.resources[Ref("my.micro")] == resources3 def test_configuration_update_description() -> None: - description1 = '' - description2 = 'single line description' - description3 = 'multiline\ndescription' + description1 = "" + description2 = "single line description" + description3 = "multiline\ndescription" overlay1 = PartialConfiguration(description=description1) overlay2 = PartialConfiguration(description=description2) @@ -166,57 +170,59 @@ def test_configuration_update_description() -> None: base = PartialConfiguration(description=description1) base.update(overlay1) - assert base.description == '' + assert base.description == "" base.update(overlay2) assert base.description == description2 base.update(overlay3) - assert base.description == description2 + '\n\n' + description3 + assert base.description == description2 + "\n\n" + description3 base.update(overlay1) - assert base.description == description2 + '\n\n' + description3 + assert base.description == description2 + "\n\n" + description3 -def test_configuration_update_checkpoint( - test_config4: PartialConfiguration) -> None: +def test_configuration_update_checkpoint(test_config4: PartialConfiguration) -> None: # Note: test_checkpoint.py tests merging of checkpoint definitions - base = PartialConfiguration(checkpoints=Checkpoints( - wallclock_time=test_config4.checkpoints.wallclock_time)) - overlay = PartialConfiguration(checkpoints=Checkpoints( - simulation_time=test_config4.checkpoints.simulation_time)) + base = PartialConfiguration( + checkpoints=Checkpoints(wallclock_time=test_config4.checkpoints.wallclock_time) + ) + overlay = PartialConfiguration( + checkpoints=Checkpoints( + simulation_time=test_config4.checkpoints.simulation_time + ) + ) assert base.checkpoints.simulation_time == [] assert overlay.checkpoints.wallclock_time == [] base.update(overlay) - assert (base.checkpoints.simulation_time - == overlay.checkpoints.simulation_time) + assert base.checkpoints.simulation_time == overlay.checkpoints.simulation_time def test_configuration_update_resume() -> None: - base = PartialConfiguration(resume={Ref('a'): Path('a')}) - overlay = PartialConfiguration(resume={Ref('b'): Path('b')}) + base = PartialConfiguration(resume={Ref("a"): Path("a")}) + overlay = PartialConfiguration(resume={Ref("b"): Path("b")}) base.update(overlay) assert len(base.resume) == 2 - assert base.resume[Ref('a')] == Path('a') - assert base.resume[Ref('b')] == Path('b') + assert base.resume[Ref("a")] == Path("a") + assert base.resume[Ref("b")] == Path("b") def test_configuration_update_resume_override() -> None: - base = PartialConfiguration(resume={Ref('a'): Path('a'), Ref('b'): Path('b')}) - overlay = PartialConfiguration(resume={Ref('b'): Path('b_update')}) + base = PartialConfiguration(resume={Ref("a"): Path("a"), Ref("b"): Path("b")}) + overlay = PartialConfiguration(resume={Ref("b"): Path("b_update")}) base.update(overlay) assert len(base.resume) == 2 - assert base.resume[Ref('a')] == Path('a') - assert base.resume[Ref('b')] == Path('b_update') + assert base.resume[Ref("a")] == Path("a") + assert base.resume[Ref("b")] == Path("b_update") def test_as_configuration( - test_config2: PartialConfiguration, test_config4: PartialConfiguration - ) -> None: + test_config2: PartialConfiguration, test_config4: PartialConfiguration +) -> None: with pytest.raises(ValueError): test_config2.as_configuration() @@ -238,24 +244,20 @@ def test_as_configuration( def test_check_consistent(test_config6: Configuration) -> None: test_config6.check_consistent() - test_config6.implementations[Ref('c')].execution_model = ( - ExecutionModel.DIRECT) + test_config6.implementations[Ref("c")].execution_model = ExecutionModel.DIRECT with pytest.raises(RuntimeError): test_config6.check_consistent() - test_config6.implementations[Ref('c')].execution_model = ( - ExecutionModel.OPENMPI) - test_config6.resources[Ref('singlethreaded')] = MPICoresResReq( - Ref('singlethreaded'), 16, 8) + test_config6.implementations[Ref("c")].execution_model = ExecutionModel.OPENMPI + test_config6.resources[Ref("singlethreaded")] = MPICoresResReq( + Ref("singlethreaded"), 16, 8 + ) # singlethreaded is started with a script, for which we allow either # MPICoresResReq or ThreadedResReq test_config6.check_consistent() def test_load_nil_settings() -> None: - text = ( - 'ymmsl_version: v0.1\n' - 'settings:\n' - ) + text = "ymmsl_version: v0.1\nsettings:\n" configuration = load(text) @@ -267,9 +269,7 @@ def test_load_nil_settings() -> None: def test_load_no_settings() -> None: - text = ( - 'ymmsl_version: v0.1\n' - ) + text = "ymmsl_version: v0.1\n" configuration = load(text) @@ -284,181 +284,181 @@ def test_dump_empty_settings() -> None: configuration = PartialConfiguration(None, Settings()) text = dump(configuration) - assert text == 'ymmsl_version: v0.1\n' + assert text == "ymmsl_version: v0.1\n" def test_load_implementations() -> None: text = ( - 'ymmsl_version: v0.1\n' - 'implementations:\n' - ' macro:\n' - ' script: |\n' - ' #!/bin/bash\n' - '\n' - ' /usr/bin/python3 /home/test/macro.py\n' - '\n' - ' meso: |\n' - ' #!/bin/bash\n' - '\n' - ' /home/test/meso.py\n' - '\n' - ' micro: /home/test/micro\n' - ' micro2:\n' - ' modules: python/3.6.0\n' - ' virtual_env: /home/test/venv\n' - ' env:\n' - ' VAR1: 13\n' - ' VAR2: Testing\n' - ' VAR3: 12.34\n' - ' VAR4: true\n' - ' execution_model: direct\n' - ' executable: /home/test/micro2\n' - ' args:\n' - ' - -s\n' - ' - -t\n' - ) + "ymmsl_version: v0.1\n" + "implementations:\n" + " macro:\n" + " script: |\n" + " #!/bin/bash\n" + "\n" + " /usr/bin/python3 /home/test/macro.py\n" + "\n" + " meso: |\n" + " #!/bin/bash\n" + "\n" + " /home/test/meso.py\n" + "\n" + " micro: /home/test/micro\n" + " micro2:\n" + " modules: python/3.6.0\n" + " virtual_env: /home/test/venv\n" + " env:\n" + " VAR1: 13\n" + " VAR2: Testing\n" + " VAR3: 12.34\n" + " VAR4: true\n" + " execution_model: direct\n" + " executable: /home/test/micro2\n" + " args:\n" + " - -s\n" + " - -t\n" + ) configuration = load(text) assert isinstance(configuration, PartialConfiguration) - assert configuration.implementations[Ref('macro')].name == 'macro' - assert configuration.implementations[Ref('macro')].script == ( - '#!/bin/bash\n\n/usr/bin/python3 /home/test/macro.py\n') - assert configuration.implementations[Ref('meso')].name == 'meso' - assert configuration.implementations[Ref('meso')].script == ( - '#!/bin/bash\n\n/home/test/meso.py\n') - assert configuration.implementations[Ref('micro')].name == 'micro' - assert configuration.implementations[Ref('micro')].script == ( - '/home/test/micro') - - m2 = configuration.implementations[Ref('micro2')] - assert m2.name == 'micro2' + assert configuration.implementations[Ref("macro")].name == "macro" + assert configuration.implementations[Ref("macro")].script == ( + "#!/bin/bash\n\n/usr/bin/python3 /home/test/macro.py\n" + ) + assert configuration.implementations[Ref("meso")].name == "meso" + assert configuration.implementations[Ref("meso")].script == ( + "#!/bin/bash\n\n/home/test/meso.py\n" + ) + assert configuration.implementations[Ref("micro")].name == "micro" + assert configuration.implementations[Ref("micro")].script == ("/home/test/micro") + + m2 = configuration.implementations[Ref("micro2")] + assert m2.name == "micro2" assert m2.script is None - assert m2.modules == ['python/3.6.0'] - assert m2.virtual_env == Path('/home/test/venv') + assert m2.modules == ["python/3.6.0"] + assert m2.virtual_env == Path("/home/test/venv") assert m2.env is not None - assert m2.env['VAR1'] == '13' - assert m2.env['VAR2'] == 'Testing' - assert m2.env['VAR3'] == '12.34' - assert m2.env['VAR4'] == 'true' - assert m2.executable == Path('/home/test/micro2') - assert m2.args == ['-s', '-t'] + assert m2.env["VAR1"] == "13" + assert m2.env["VAR2"] == "Testing" + assert m2.env["VAR3"] == "12.34" + assert m2.env["VAR4"] == "true" + assert m2.executable == Path("/home/test/micro2") + assert m2.args == ["-s", "-t"] assert m2.execution_model == ExecutionModel.DIRECT def test_load_implementations_script_list() -> None: text = ( - 'ymmsl_version: v0.1\n' - 'implementations:\n' - ' macro:\n' - ' script:\n' - ' - "#!/bin/bash"\n' - ' - ""\n' - ' - /usr/bin/python3 /home/test/macro.py\n' - ' - ""\n' - ' meso:\n' - ' - "#!/bin/bash"\n' - ' - ""\n' - ' - /home/test/meso.py\n' - ' micro: /home/test/micro\n' - ) + "ymmsl_version: v0.1\n" + "implementations:\n" + " macro:\n" + " script:\n" + ' - "#!/bin/bash"\n' + ' - ""\n' + " - /usr/bin/python3 /home/test/macro.py\n" + ' - ""\n' + " meso:\n" + ' - "#!/bin/bash"\n' + ' - ""\n' + " - /home/test/meso.py\n" + " micro: /home/test/micro\n" + ) configuration = load(text) assert isinstance(configuration, PartialConfiguration) - assert configuration.implementations[Ref('macro')].name == 'macro' - assert configuration.implementations[Ref('macro')].script == ( - '#!/bin/bash\n\n/usr/bin/python3 /home/test/macro.py\n\n') - assert configuration.implementations[Ref('meso')].name == 'meso' - assert configuration.implementations[Ref('meso')].script == ( - '#!/bin/bash\n\n/home/test/meso.py\n') - assert configuration.implementations[Ref('micro')].name == 'micro' - assert configuration.implementations[Ref('micro')].script == ( - '/home/test/micro') + assert configuration.implementations[Ref("macro")].name == "macro" + assert configuration.implementations[Ref("macro")].script == ( + "#!/bin/bash\n\n/usr/bin/python3 /home/test/macro.py\n\n" + ) + assert configuration.implementations[Ref("meso")].name == "meso" + assert configuration.implementations[Ref("meso")].script == ( + "#!/bin/bash\n\n/home/test/meso.py\n" + ) + assert configuration.implementations[Ref("micro")].name == "micro" + assert configuration.implementations[Ref("micro")].script == ("/home/test/micro") def test_dump_implementations() -> None: implementations = [ - Implementation( - name=Ref('macro'), - script='#!/bin/bash\n\n/usr/bin/python3 /home/test/macro.py\n' - ), - Implementation( - name=Ref('meso'), - script='#!/bin/bash\n\n/home/test/meso.py'), - Implementation(name=Ref('micro'), script='/home/test/micro'), - Implementation( - name=Ref('micro2'), - modules=['python/3.6.0', 'gcc/9.3.0'], - virtual_env=Path('/home/test/env'), - env={'VAR1': '10', 'VAR2': 'Test'}, - execution_model=ExecutionModel.OPENMPI, - executable=Path('/home/test/micro2'), - args=['-v', '-s'])] + Implementation( + name=Ref("macro"), + script="#!/bin/bash\n\n/usr/bin/python3 /home/test/macro.py\n", + ), + Implementation(name=Ref("meso"), script="#!/bin/bash\n\n/home/test/meso.py"), + Implementation(name=Ref("micro"), script="/home/test/micro"), + Implementation( + name=Ref("micro2"), + modules=["python/3.6.0", "gcc/9.3.0"], + virtual_env=Path("/home/test/env"), + env={"VAR1": "10", "VAR2": "Test"}, + execution_model=ExecutionModel.OPENMPI, + executable=Path("/home/test/micro2"), + args=["-v", "-s"], + ), + ] configuration = PartialConfiguration(None, None, implementations) text = dump(configuration) assert text == ( - 'ymmsl_version: v0.1\n' - 'implementations:\n' - ' macro: |\n' - ' #!/bin/bash\n' - '\n' - ' /usr/bin/python3 /home/test/macro.py\n' - ' meso: |-\n' - ' #!/bin/bash\n' - '\n' - ' /home/test/meso.py\n' - ' micro: /home/test/micro\n' - ' micro2:\n' - ' modules:\n' - ' - python/3.6.0\n' - ' - gcc/9.3.0\n' - ' virtual_env: /home/test/env\n' - ' env:\n' - ' VAR1: \'10\'\n' - ' VAR2: Test\n' - ' execution_model: openmpi\n' - ' executable: /home/test/micro2\n' - ' args:\n' - ' - -v\n' - ' - -s\n' - ) + "ymmsl_version: v0.1\n" + "implementations:\n" + " macro: |\n" + " #!/bin/bash\n" + "\n" + " /usr/bin/python3 /home/test/macro.py\n" + " meso: |-\n" + " #!/bin/bash\n" + "\n" + " /home/test/meso.py\n" + " micro: /home/test/micro\n" + " micro2:\n" + " modules:\n" + " - python/3.6.0\n" + " - gcc/9.3.0\n" + " virtual_env: /home/test/env\n" + " env:\n" + " VAR1: '10'\n" + " VAR2: Test\n" + " execution_model: openmpi\n" + " executable: /home/test/micro2\n" + " args:\n" + " - -v\n" + " - -s\n" + ) def test_load_resources() -> None: text = ( - 'ymmsl_version: v0.1\n' - 'resources:\n' - ' macro:\n' - ' threads: 10\n' - ' micro:\n' - ' threads: 1\n' - ) + "ymmsl_version: v0.1\n" + "resources:\n" + " macro:\n" + " threads: 10\n" + " micro:\n" + " threads: 1\n" + ) configuration = load(text) assert isinstance(configuration, PartialConfiguration) - assert configuration.resources[Ref('macro')].name == 'macro' - assert configuration.resources[Ref('macro')].threads == 10 # type: ignore - assert configuration.resources[Ref('micro')].name == 'micro' - assert configuration.resources[Ref('micro')].threads == 1 # type: ignore + assert configuration.resources[Ref("macro")].name == "macro" + assert configuration.resources[Ref("macro")].threads == 10 # type: ignore + assert configuration.resources[Ref("micro")].name == "micro" + assert configuration.resources[Ref("micro")].threads == 1 # type: ignore def test_dump_resources() -> None: - resources = [ - ThreadedResReq(Ref('macro'), 10), - ThreadedResReq(Ref('micro'), 1)] + resources = [ThreadedResReq(Ref("macro"), 10), ThreadedResReq(Ref("micro"), 1)] configuration = PartialConfiguration(None, None, None, resources) text = dump(configuration) assert text == ( - 'ymmsl_version: v0.1\n' - 'resources:\n' - ' macro:\n' - ' threads: 10\n' - ' micro:\n' - ' threads: 1\n' - ) + "ymmsl_version: v0.1\n" + "resources:\n" + " macro:\n" + " threads: 10\n" + " micro:\n" + " threads: 1\n" + ) diff --git a/ymmsl/v0_1/tests/test_document.py b/ymmsl/v0_1/tests/test_document.py index d734f19..0775b76 100644 --- a/ymmsl/v0_1/tests/test_document.py +++ b/ymmsl/v0_1/tests/test_document.py @@ -17,16 +17,16 @@ def dump_document() -> Callable: def test_valid_document(load_document: Callable) -> None: - text = 'ymmsl_version: v0.1' + text = "ymmsl_version: v0.1" load_document(text) def test_load_unknown_version(load_document: Callable) -> None: - text = 'ymmsl_version: v0.2' + text = "ymmsl_version: v0.2" with pytest.raises(yatiml.RecognitionError): load_document(text) def test_dump_document(dump_document: Callable) -> None: text = dump_document(Document()) - assert text == 'ymmsl_version: v0.1\n' + assert text == "ymmsl_version: v0.1\n" diff --git a/ymmsl/v0_1/tests/test_execution.py b/ymmsl/v0_1/tests/test_execution.py index 53c5c32..3348430 100644 --- a/ymmsl/v0_1/tests/test_execution.py +++ b/ymmsl/v0_1/tests/test_execution.py @@ -12,69 +12,63 @@ def test_implementation() -> None: - impl = Implementation(Reference('test_impl'), script='run_test_impl') - assert impl.name == 'test_impl' - assert impl.script == 'run_test_impl' + impl = Implementation(Reference("test_impl"), script="run_test_impl") + assert impl.name == "test_impl" + assert impl.script == "run_test_impl" def test_implementation_script_list() -> None: impl = Implementation( - name=Reference('test_impl'), - script=[ - '#!/bin/bash', - '', - 'test_impl']) + name=Reference("test_impl"), script=["#!/bin/bash", "", "test_impl"] + ) - assert impl.name == 'test_impl' - assert impl.script == '#!/bin/bash\n\ntest_impl\n' + assert impl.name == "test_impl" + assert impl.script == "#!/bin/bash\n\ntest_impl\n" def test_implementations_executable() -> None: impl = Implementation( - name=Reference('test_impl'), - base_env=BaseEnv.MANAGER, - modules=['python/3.6.0', 'gcc/9.3.0'], - virtual_env=Path('/home/user/envs/venv'), - env={ - 'VAR1': '1', - 'VAR2': 'Testing'}, - executable=Path('/home/user/software/my_submodel/bin/model'), - args='-v -a', - execution_model=ExecutionModel.OPENMPI, - can_share_resources=False) - - assert impl.name == 'test_impl' + name=Reference("test_impl"), + base_env=BaseEnv.MANAGER, + modules=["python/3.6.0", "gcc/9.3.0"], + virtual_env=Path("/home/user/envs/venv"), + env={"VAR1": "1", "VAR2": "Testing"}, + executable=Path("/home/user/software/my_submodel/bin/model"), + args="-v -a", + execution_model=ExecutionModel.OPENMPI, + can_share_resources=False, + ) + + assert impl.name == "test_impl" assert impl.base_env == BaseEnv.MANAGER - assert impl.modules == ['python/3.6.0', 'gcc/9.3.0'] - assert impl.virtual_env == Path('/home/user/envs/venv') + assert impl.modules == ["python/3.6.0", "gcc/9.3.0"] + assert impl.virtual_env == Path("/home/user/envs/venv") assert impl.env is not None - assert impl.env['VAR1'] == '1' - assert impl.env['VAR2'] == 'Testing' - assert impl.executable == Path('/home/user/software/my_submodel/bin/model') - assert impl.args == ['-v -a'] + assert impl.env["VAR1"] == "1" + assert impl.env["VAR2"] == "Testing" + assert impl.executable == Path("/home/user/software/my_submodel/bin/model") + assert impl.args == ["-v -a"] assert impl.execution_model == ExecutionModel.OPENMPI assert impl.can_share_resources is False def test_implementations_exclusive() -> None: with pytest.raises(RuntimeError): - Implementation(name=Reference('test'), script='', executable=Path()) + Implementation(name=Reference("test"), script="", executable=Path()) def test_implementations_script() -> None: - script = ( - '#!/bin/bash\n' - '\n' - 'mpirun my_model\n') + script = "#!/bin/bash\n\nmpirun my_model\n" impl = Implementation( - name=Reference('test_impl'), - execution_model=ExecutionModel.OPENMPI, - can_share_resources=False, - keeps_state_for_next_use=KeepsStateForNextUse.HELPFUL, - script=script) - - assert impl.name == 'test_impl' + name=Reference("test_impl"), + execution_model=ExecutionModel.OPENMPI, + can_share_resources=False, + keeps_state_for_next_use=KeepsStateForNextUse.HELPFUL, + script=script, + ) + + assert impl.name == "test_impl" assert impl.execution_model == ExecutionModel.OPENMPI assert impl.can_share_resources is False assert impl.keeps_state_for_next_use == KeepsStateForNextUse.HELPFUL @@ -84,7 +78,8 @@ def test_implementations_script() -> None: def test_implementations_script_invalid_args() -> None: with pytest.raises(RuntimeError): Implementation( - name=Reference('test_impl'), - execution_model=ExecutionModel.DIRECT, - env={'TEST': 'NOT_ALLOWED'}, - script='test') + name=Reference("test_impl"), + execution_model=ExecutionModel.DIRECT, + env={"TEST": "NOT_ALLOWED"}, + script="test", + ) diff --git a/ymmsl/v0_1/tests/test_identity.py b/ymmsl/v0_1/tests/test_identity.py index 6f8126c..b4ab919 100644 --- a/ymmsl/v0_1/tests/test_identity.py +++ b/ymmsl/v0_1/tests/test_identity.py @@ -5,199 +5,196 @@ def test_create_identifier() -> None: - part = Identifier('testing') - assert str(part) == 'testing' + part = Identifier("testing") + assert str(part) == "testing" - part = Identifier('CapiTaLs') - assert str(part) == 'CapiTaLs' + part = Identifier("CapiTaLs") + assert str(part) == "CapiTaLs" - part = Identifier('under_score') - assert str(part) == 'under_score' + part = Identifier("under_score") + assert str(part) == "under_score" - part = Identifier('_underscore') - assert str(part) == '_underscore' + part = Identifier("_underscore") + assert str(part) == "_underscore" - part = Identifier('digits123') - assert str(part) == 'digits123' + part = Identifier("digits123") + assert str(part) == "digits123" with pytest.raises(ValueError): - Identifier('1initialdigit') + Identifier("1initialdigit") with pytest.raises(ValueError): - Identifier('test.period') + Identifier("test.period") with pytest.raises(ValueError): - Identifier('test-hyphen') + Identifier("test-hyphen") with pytest.raises(ValueError): - Identifier('test space') + Identifier("test space") with pytest.raises(ValueError): - Identifier('test/slash') + Identifier("test/slash") def test_compare_identifier() -> None: - assert Identifier('test') == Identifier('test') - assert Identifier('test1') != Identifier('test2') + assert Identifier("test") == Identifier("test") + assert Identifier("test1") != Identifier("test2") - assert Identifier('test') == 'test' - assert 'test' == Identifier('test') # pylint: disable=C0122 - assert Identifier('test') != 'test2' - assert 'test2' != Identifier('test') # pylint: disable=C0122 + assert Identifier("test") == "test" + assert "test" == Identifier("test") # pylint: disable=C0122 + assert Identifier("test") != "test2" + assert "test2" != Identifier("test") # pylint: disable=C0122 def test_identifier_dict_key() -> None: - test_dict = {Identifier('test'): 1} - assert test_dict[Identifier('test')] == 1 + test_dict = {Identifier("test"): 1} + assert test_dict[Identifier("test")] == 1 def test_create_reference() -> None: - test_ref = Reference('_testing') - assert str(test_ref) == '_testing' + test_ref = Reference("_testing") + assert str(test_ref) == "_testing" assert len(test_ref) == 1 assert isinstance(test_ref[0], Identifier) - assert str(test_ref[0]) == '_testing' + assert str(test_ref[0]) == "_testing" with pytest.raises(ValueError): - Reference('1test') + Reference("1test") - test_ref = Reference('test.testing') + test_ref = Reference("test.testing") assert len(test_ref) == 2 assert isinstance(test_ref[0], Identifier) - assert str(test_ref[0]) == 'test' + assert str(test_ref[0]) == "test" assert isinstance(test_ref[1], Identifier) - assert str(test_ref[1]) == 'testing' - assert str(test_ref) == 'test.testing' + assert str(test_ref[1]) == "testing" + assert str(test_ref) == "test.testing" - test_ref = Reference('test[12]') + test_ref = Reference("test[12]") assert len(test_ref) == 2 assert isinstance(test_ref[0], Identifier) - assert str(test_ref[0]) == 'test' + assert str(test_ref[0]) == "test" assert isinstance(test_ref[1], int) assert test_ref[1] == 12 - assert str(test_ref) == 'test[12]' + assert str(test_ref) == "test[12]" - test_ref = Reference('test[12].testing.ok.index[3][5]') + test_ref = Reference("test[12].testing.ok.index[3][5]") assert len(test_ref) == 7 assert isinstance(test_ref[0], Identifier) - assert str(test_ref[0]) == 'test' + assert str(test_ref[0]) == "test" assert isinstance(test_ref[1], int) assert test_ref[1] == 12 assert isinstance(test_ref[2], Identifier) - assert str(test_ref[2]) == 'testing' + assert str(test_ref[2]) == "testing" assert isinstance(test_ref[3], Identifier) - assert str(test_ref[3]) == 'ok' + assert str(test_ref[3]) == "ok" assert isinstance(test_ref[4], Identifier) - assert str(test_ref[4]) == 'index' + assert str(test_ref[4]) == "index" assert isinstance(test_ref[5], int) assert test_ref[5] == 3 assert isinstance(test_ref[6], int) assert test_ref[6] == 5 - assert str(test_ref) == 'test[12].testing.ok.index[3][5]' + assert str(test_ref) == "test[12].testing.ok.index[3][5]" with pytest.raises(ValueError): Reference([4]) with pytest.raises(ValueError): - Reference([3, Identifier('test')]) + Reference([3, Identifier("test")]) with pytest.raises(ValueError): Reference('ua",.u8[') with pytest.raises(ValueError): - Reference('test[4') + Reference("test[4") with pytest.raises(ValueError): - Reference('test4]') + Reference("test4]") with pytest.raises(ValueError): - Reference('test[_t]') + Reference("test[_t]") with pytest.raises(ValueError): - Reference('testing_{3}') + Reference("testing_{3}") with pytest.raises(ValueError): - Reference('test.(x)') + Reference("test.(x)") with pytest.raises(ValueError): - Reference('[3]test') + Reference("[3]test") with pytest.raises(ValueError): - Reference('[4].test') + Reference("[4].test") def test_reference_slicing() -> None: - test_ref = Reference('test[12].testing.ok.index[3][5]') + test_ref = Reference("test[12].testing.ok.index[3][5]") - assert test_ref[0] == 'test' + assert test_ref[0] == "test" assert test_ref[1] == 12 - assert test_ref[3] == 'ok' - assert test_ref[:3] == 'test[12].testing' - assert test_ref[2:] == 'testing.ok.index[3][5]' + assert test_ref[3] == "ok" + assert test_ref[:3] == "test[12].testing" + assert test_ref[2:] == "testing.ok.index[3][5]" with pytest.raises(RuntimeError): - test_ref[0] = 'test2' + test_ref[0] = "test2" with pytest.raises(ValueError): - test_ref[1:] # pylint: disable=pointless-statement + test_ref[1:] # pylint: disable=pointless-statement def test_reference_dict_key() -> None: - test_dict = {Reference('test[4]'): 1} - assert test_dict[Reference('test[4]')] == 1 + test_dict = {Reference("test[4]"): 1} + assert test_dict[Reference("test[4]")] == 1 def test_reference_equivalence() -> None: - assert Reference('test.test[3]') == Reference('test.test[3]') - assert Reference('test.test[3]') != Reference('test1.test[3]') + assert Reference("test.test[3]") == Reference("test.test[3]") + assert Reference("test.test[3]") != Reference("test1.test[3]") - assert Reference('test.test[3]') == 'test.test[3]' - assert Reference('test.test[3]') != 'test1.test[3]' - assert 'test.test[3]' == Reference('test.test[3]') # pylint: disable=C0122 - assert 'test1.test[3]' != Reference( - 'test.test[3]') # pylint: disable=C0122 + assert Reference("test.test[3]") == "test.test[3]" + assert Reference("test.test[3]") != "test1.test[3]" + assert "test.test[3]" == Reference("test.test[3]") # pylint: disable=C0122 + assert "test1.test[3]" != Reference("test.test[3]") # pylint: disable=C0122 def test_reference_sort() -> None: - assert Reference('a') < Reference('b') - assert Reference('b') > Reference('a') - assert Reference('test[5]') < Reference('test[15]') - assert Reference('test[3]') < Reference('test.a') - assert sorted([Reference('c'), Reference('a')]) == [ - Reference('a'), Reference('c')] + assert Reference("a") < Reference("b") + assert Reference("b") > Reference("a") + assert Reference("test[5]") < Reference("test[15]") + assert Reference("test[3]") < Reference("test.a") + assert sorted([Reference("c"), Reference("a")]) == [Reference("a"), Reference("c")] def test_reference_concatenation() -> None: - assert Reference('test') + Reference('test2') == 'test.test2' - assert Reference('test') + Identifier('test2') == 'test.test2' - assert Reference('test') + 5 == 'test[5]' - assert Reference('test') + [Identifier('test2'), 5] == 'test.test2[5]' + assert Reference("test") + Reference("test2") == "test.test2" + assert Reference("test") + Identifier("test2") == "test.test2" + assert Reference("test") + 5 == "test[5]" + assert Reference("test") + [Identifier("test2"), 5] == "test.test2[5]" - assert Reference('test[5]') + Reference('test2[3]') == 'test[5].test2[3]' - assert Reference('test[5]') + Identifier('test2') == 'test[5].test2' - assert Reference('test[5]') + 3 == 'test[5][3]' - assert (Reference('test[5]') + [3, Identifier('test2')] == - 'test[5][3].test2') + assert Reference("test[5]") + Reference("test2[3]") == "test[5].test2[3]" + assert Reference("test[5]") + Identifier("test2") == "test[5].test2" + assert Reference("test[5]") + 3 == "test[5][3]" + assert Reference("test[5]") + [3, Identifier("test2")] == "test[5][3].test2" def test_reference_without_trailing_ints() -> None: Ref = Reference - assert Ref('a.b.c[1][2]').without_trailing_ints() == Ref('a.b.c') - assert Ref('a[1].b.c').without_trailing_ints() == Ref('a[1].b.c') - assert Ref('a.b.c').without_trailing_ints() == Ref('a.b.c') - assert Ref('a[1].b.c[2]').without_trailing_ints() == Ref('a[1].b.c') + assert Ref("a.b.c[1][2]").without_trailing_ints() == Ref("a.b.c") + assert Ref("a[1].b.c").without_trailing_ints() == Ref("a[1].b.c") + assert Ref("a.b.c").without_trailing_ints() == Ref("a.b.c") + assert Ref("a[1].b.c[2]").without_trailing_ints() == Ref("a[1].b.c") def test_reference_io() -> None: load_reference = yatiml.load_function(Reference, Identifier) - text = 'test[12]' + text = "test[12]" doc = load_reference(text) - assert str(doc[0]) == 'test' + assert str(doc[0]) == "test" assert doc[1] == 12 dump_reference = yatiml.dumps_function(Identifier, Reference) - doc = Reference('test[12].testing.ok.index[3][5]') + doc = Reference("test[12].testing.ok.index[3][5]") text = dump_reference(doc) - assert text == 'test[12].testing.ok.index[3][5]\n...\n' + assert text == "test[12].testing.ok.index[3][5]\n...\n" diff --git a/ymmsl/v0_1/tests/test_model.py b/ymmsl/v0_1/tests/test_model.py index 2e43229..6aee9e9 100644 --- a/ymmsl/v0_1/tests/test_model.py +++ b/ymmsl/v0_1/tests/test_model.py @@ -20,135 +20,140 @@ @pytest.fixture def load_model() -> Callable: return yatiml.load_function( - Model, Component, Conduit, Identifier, Ports, Reference, - MulticastConduit) + Model, Component, Conduit, Identifier, Ports, Reference, MulticastConduit + ) @pytest.fixture def dump_model() -> Callable: return yatiml.dumps_function( - Component, Conduit, Identifier, Model, Ports, Reference, - MulticastConduit) + Component, Conduit, Identifier, Model, Ports, Reference, MulticastConduit + ) @pytest.fixture def macro_micro() -> Model: - macro = Component('macro', 'my.macro', ports=Ports( - o_i=['intermediate_state'], s=['state_update'])) - micro = Component('micro', 'my.micro', ports=Ports( - f_init=['initial_state'], o_f=['final_state'])) + macro = Component( + "macro", "my.macro", ports=Ports(o_i=["intermediate_state"], s=["state_update"]) + ) + micro = Component( + "micro", "my.micro", ports=Ports(f_init=["initial_state"], o_f=["final_state"]) + ) components = [macro, micro] - conduit1 = Conduit('macro.intermediate_state', 'micro.initial_state') - conduit2 = Conduit('micro.final_state', 'macro.state_update') + conduit1 = Conduit("macro.intermediate_state", "micro.initial_state") + conduit2 = Conduit("micro.final_state", "macro.state_update") conduits = [conduit1, conduit2] - return Model('macro_micro', components, conduits) + return Model("macro_micro", components, conduits) def test_conduit() -> None: - test_conduit = Conduit('submodel1.port1', 'submodel2.port2') - assert str(test_conduit.sender[0]) == 'submodel1' - assert str(test_conduit.sender[1]) == 'port1' - assert str(test_conduit.receiver[0]) == 'submodel2' - assert str(test_conduit.receiver[1]) == 'port2' - - assert str(test_conduit.sending_component()) == 'submodel1' - assert str(test_conduit.sending_port()) == 'port1' + test_conduit = Conduit("submodel1.port1", "submodel2.port2") + assert str(test_conduit.sender[0]) == "submodel1" + assert str(test_conduit.sender[1]) == "port1" + assert str(test_conduit.receiver[0]) == "submodel2" + assert str(test_conduit.receiver[1]) == "port2" + + assert str(test_conduit.sending_component()) == "submodel1" + assert str(test_conduit.sending_port()) == "port1" assert test_conduit.sending_slot() == [] - assert str(test_conduit.receiving_component()) == 'submodel2' - assert str(test_conduit.receiving_port()) == 'port2' + assert str(test_conduit.receiving_component()) == "submodel2" + assert str(test_conduit.receiving_port()) == "port2" assert test_conduit.receiving_slot() == [] with pytest.raises(ValueError): - Conduit('x', 'submodel1.port1') + Conduit("x", "submodel1.port1") with pytest.raises(ValueError): - Conduit('x[3].y.z', 'submodel1.port1') + Conduit("x[3].y.z", "submodel1.port1") with pytest.raises(ValueError): - Conduit('x[3]', 'submodel1.port1') + Conduit("x[3]", "submodel1.port1") - test_conduit2 = Conduit('submodel1.port1', 'submodel2.port2') + test_conduit2 = Conduit("submodel1.port1", "submodel2.port2") assert test_conduit == test_conduit2 - assert 'Conduit' in str(test_conduit) - assert 'submodel1.port1' in str(test_conduit) - assert 'submodel2.port2' in str(test_conduit) + assert "Conduit" in str(test_conduit) + assert "submodel1.port1" in str(test_conduit) + assert "submodel2.port2" in str(test_conduit) - test_conduit4 = Conduit('x.y[1][2]', 'a.b[3]') + test_conduit4 = Conduit("x.y[1][2]", "a.b[3]") assert test_conduit4.sender[2] == 1 assert test_conduit4.sender[3] == 2 - assert str(test_conduit4.sending_component()) == 'x' - assert str(test_conduit4.sending_port()) == 'y' + assert str(test_conduit4.sending_component()) == "x" + assert str(test_conduit4.sending_port()) == "y" assert test_conduit4.sending_slot() == [1, 2] assert test_conduit4.receiver[2] == 3 - assert str(test_conduit4.receiving_component()) == 'a' - assert str(test_conduit4.receiving_port()) == 'b' + assert str(test_conduit4.receiving_component()) == "a" + assert str(test_conduit4.receiving_port()) == "b" assert test_conduit4.receiving_slot() == [3] def test_load_model_reference() -> None: load = yatiml.load_function( - ModelReference, Component, Conduit, Identifier, Model, - MulticastConduit, Reference) - - text = 'name: test_model\n' + ModelReference, + Component, + Conduit, + Identifier, + Model, + MulticastConduit, + Reference, + ) + + text = "name: test_model\n" model = load(text) assert isinstance(model, ModelReference) - assert str(model.name) == 'test_model' - - text = ('name: test_model\n' - 'components:\n' - ' ic: isr2d.initial_conditions\n' - ' smc: isr2d.smc\n' - ' bf: isr2d.blood_flow\n' - ' smc2bf: isr2d.smc2bf\n' - ' bf2smc: isr2d.bf2smc\n' - 'conduits:\n' - ' ic.out: smc.initial_state\n' - ' smc.cell_positions: smc2bf.in\n' - ' smc2bf.out: bf.initial_domain\n' - ' bf.wss_out: bf2smc.in\n' - ' bf2smc.out: smc.wss_in\n' - ) + assert str(model.name) == "test_model" + + text = ( + "name: test_model\n" + "components:\n" + " ic: isr2d.initial_conditions\n" + " smc: isr2d.smc\n" + " bf: isr2d.blood_flow\n" + " smc2bf: isr2d.smc2bf\n" + " bf2smc: isr2d.bf2smc\n" + "conduits:\n" + " ic.out: smc.initial_state\n" + " smc.cell_positions: smc2bf.in\n" + " smc2bf.out: bf.initial_domain\n" + " bf.wss_out: bf2smc.in\n" + " bf2smc.out: smc.wss_in\n" + ) model = load(text) assert isinstance(model, Model) - assert str(model.name) == 'test_model' + assert str(model.name) == "test_model" def test_model(macro_micro: Model) -> None: - assert str(macro_micro.name) == 'macro_micro' + assert str(macro_micro.name) == "macro_micro" assert len(macro_micro.components) == 2 assert len(macro_micro.conduits) == 2 def test_model_no_impl(load_model: Callable) -> None: # Test making a component with no implementation - Component('macro') + Component("macro") # Test loading from YAML - text = ( - 'name: macro_micro\n' - 'components:\n' - ' macro:\n' - ' micro:\n') + text = "name: macro_micro\ncomponents:\n macro:\n micro:\n" model = load_model(text) - assert model.name == 'macro_micro' + assert model.name == "macro_micro" assert len(model.components) == 2 - assert model.components[0].name in ('macro', 'micro') + assert model.components[0].name in ("macro", "micro") assert model.components[0].implementation is None - assert model.components[1].name in ('macro', 'micro') + assert model.components[1].name in ("macro", "micro") assert model.components[0].implementation is None def test_model_update_add_component() -> None: - macro = Component('macro', 'my.macro') - base = Model('test_update', [macro]) + macro = Component("macro", "my.macro") + base = Model("test_update", [macro]) - micro = Component('micro', 'my.micro') - conduit1 = Conduit('macro.intermediate_state', 'micro.initial_state') - conduit2 = Conduit('micro.final_state', 'macro.state_update') + micro = Component("micro", "my.micro") + conduit1 = Conduit("macro.intermediate_state", "micro.initial_state") + conduit2 = Conduit("micro.final_state", "macro.state_update") conduits = [conduit1, conduit2] - overlay = Model('test_update_add', [micro], conduits) + overlay = Model("test_update_add", [micro], conduits) base.update(overlay) @@ -160,18 +165,18 @@ def test_model_update_add_component() -> None: def test_model_update_insert_component_on_conduit() -> None: - macro = Component('macro', 'my.macro') - micro = Component('micro', 'my.micro') + macro = Component("macro", "my.macro") + micro = Component("micro", "my.micro") components = [macro, micro] - conduit1 = Conduit('macro.intermediate_state', 'micro.initial_state') - conduit2 = Conduit('micro.final_state', 'macro.state_update') + conduit1 = Conduit("macro.intermediate_state", "micro.initial_state") + conduit2 = Conduit("micro.final_state", "macro.state_update") conduits = [conduit1, conduit2] - base = Model('test_update', components, conduits) + base = Model("test_update", components, conduits) - tee = Component('tee', 'muscle3.tee') - conduit3 = Conduit('macro.intermediate_state', 'tee.input') - conduit4 = Conduit('tee.output', 'micro.initial_state') - overlay = Model('test_update_tee', [tee], [conduit3, conduit4]) + tee = Component("tee", "muscle3.tee") + conduit3 = Conduit("macro.intermediate_state", "tee.input") + conduit4 = Conduit("tee.output", "micro.initial_state") + overlay = Model("test_update_tee", [tee], [conduit3, conduit4]) base.update(overlay) @@ -187,16 +192,16 @@ def test_model_update_insert_component_on_conduit() -> None: def test_model_update_replace_component() -> None: - macro = Component('macro', 'my.macro') - micro = Component('micro', 'my.micro') + macro = Component("macro", "my.macro") + micro = Component("micro", "my.micro") components = [macro, micro] - conduit1 = Conduit('macro.intermediate_state', 'micro.initial_state') - conduit2 = Conduit('micro.final_state', 'macro.state_update') + conduit1 = Conduit("macro.intermediate_state", "micro.initial_state") + conduit2 = Conduit("micro.final_state", "macro.state_update") conduits = [conduit1, conduit2] - base = Model('test_update', components, conduits) + base = Model("test_update", components, conduits) - surrogate = Component('micro', 'my.surrogate') - overlay = Model('test_update_surrogate', [surrogate]) + surrogate = Component("micro", "my.surrogate") + overlay = Model("test_update_surrogate", [surrogate]) base.update(overlay) @@ -210,19 +215,19 @@ def test_model_update_replace_component() -> None: def test_model_update_set_implementation() -> None: - abstract_reaction = Component('reaction') - base = Model('test_set_impl', [abstract_reaction]) - assert base.components[0].name == 'reaction' + abstract_reaction = Component("reaction") + base = Model("test_set_impl", [abstract_reaction]) + assert base.components[0].name == "reaction" assert base.components[0].implementation is None - reaction_python = Component('reaction', 'reaction_python') - overlay = Model('test_set_impl', [reaction_python]) + reaction_python = Component("reaction", "reaction_python") + overlay = Model("test_set_impl", [reaction_python]) base.update(overlay) assert len(base.components) == 1 - assert base.components[0].name == 'reaction' - assert base.components[0].implementation == 'reaction_python' + assert base.components[0].name == "reaction" + assert base.components[0].implementation == "reaction_python" def test_model_multicast() -> None: @@ -251,13 +256,14 @@ def test_model_multicast() -> None: assert config.model.conduits[2].sender == "sender.multicast" assert config.model.conduits[2].receiver == "receiver2.port" - model_overlay = Model( - 'multicast', [], [Conduit("sender.port", "receiver2.port")]) + model_overlay = Model("multicast", [], [Conduit("sender.port", "receiver2.port")]) config.model.update(model_overlay) assert len(config.model.conduits) == 3 updated_config_txt = dump(config) - assert updated_config_txt == """ymmsl_version: v0.1 + assert ( + updated_config_txt + == """ymmsl_version: v0.1 model: name: multicast components: @@ -271,6 +277,7 @@ def test_model_multicast() -> None: - receiver2.port sender.multicast: receiver1.port """ + ) def test_model_check_consistent1(macro_micro: Model) -> None: @@ -278,109 +285,105 @@ def test_model_check_consistent1(macro_micro: Model) -> None: def test_model_check_consistent2(macro_micro: Model) -> None: - macro_micro.conduits[0].sender = Reference('marco.intermediate_state') + macro_micro.conduits[0].sender = Reference("marco.intermediate_state") with pytest.raises(RuntimeError): macro_micro.check_consistent() def test_model_check_consistent3(macro_micro: Model) -> None: - macro_micro.conduits[1].receiver = Reference('Macro.state_update') + macro_micro.conduits[1].receiver = Reference("Macro.state_update") with pytest.raises(RuntimeError): macro_micro.check_consistent() def test_model_check_consistent4(macro_micro: Model) -> None: - macro_micro.conduits[1].receiver = Reference('macro.does_not_exist') + macro_micro.conduits[1].receiver = Reference("macro.does_not_exist") with pytest.raises(RuntimeError): macro_micro.check_consistent() def test_model_check_consistent5(macro_micro: Model) -> None: - macro_micro.conduits[0].sender = Reference('macro.state_update') + macro_micro.conduits[0].sender = Reference("macro.state_update") with pytest.raises(RuntimeError): macro_micro.check_consistent() def test_model_check_consistent6(macro_micro: Model) -> None: - macro_micro.conduits[0].receiver = Reference('micro.final_state') + macro_micro.conduits[0].receiver = Reference("micro.final_state") with pytest.raises(RuntimeError): macro_micro.check_consistent() def test_model_check_consistent7(macro_micro: Model) -> None: - conduit = Conduit('macro.intermediate_state', 'micro.initial_state') + conduit = Conduit("macro.intermediate_state", "micro.initial_state") macro_micro.conduits.append(conduit) with pytest.raises(RuntimeError): macro_micro.check_consistent() def test_load_model(load_model: Callable) -> None: - text = ('name: test_model\n' - 'components:\n' - ' ic: isr2d.initial_conditions\n' - ' smc: isr2d.smc\n' - ' bf: isr2d.blood_flow\n' - ' smc2bf: isr2d.smc2bf\n' - ' bf2smc: isr2d.bf2smc\n' - 'conduits:\n' - ' ic.out: smc.initial_state\n' - ' smc.cell_positions: smc2bf.in\n' - ' smc2bf.out: bf.initial_domain\n' - ' bf.wss_out: bf2smc.in\n' - ' bf2smc.out: smc.wss_in\n' - ) + text = ( + "name: test_model\n" + "components:\n" + " ic: isr2d.initial_conditions\n" + " smc: isr2d.smc\n" + " bf: isr2d.blood_flow\n" + " smc2bf: isr2d.smc2bf\n" + " bf2smc: isr2d.bf2smc\n" + "conduits:\n" + " ic.out: smc.initial_state\n" + " smc.cell_positions: smc2bf.in\n" + " smc2bf.out: bf.initial_domain\n" + " bf.wss_out: bf2smc.in\n" + " bf2smc.out: smc.wss_in\n" + ) model = load_model(text) - assert str(model.name) == 'test_model' + assert str(model.name) == "test_model" assert len(model.components) == 5 - assert str(model.components[2].implementation) == 'isr2d.blood_flow' - assert str(model.components[4].name) == 'bf2smc' + assert str(model.components[2].implementation) == "isr2d.blood_flow" + assert str(model.components[4].name) == "bf2smc" assert len(model.conduits) == 5 - assert str(model.conduits[0].sending_component()) == 'ic' - assert str(model.conduits[0].sending_port()) == 'out' - assert str(model.conduits[3].receiving_component()) == 'bf2smc' - assert str(model.conduits[3].receiving_port()) == 'in' + assert str(model.conduits[0].sending_component()) == "ic" + assert str(model.conduits[0].sending_port()) == "out" + assert str(model.conduits[3].receiving_component()) == "bf2smc" + assert str(model.conduits[3].receiving_port()) == "in" def test_load_no_conduits(load_model: Callable) -> None: - text = ('name: test_model\n' - 'components:\n' - ' smc: isr2d.smc\n' - ) + text = "name: test_model\ncomponents:\n smc: isr2d.smc\n" model = load_model(text) - assert str(model.name) == 'test_model' + assert str(model.name) == "test_model" assert len(model.components) == 1 - assert str(model.components[0].name) == 'smc' - assert str(model.components[0].implementation) == 'isr2d.smc' + assert str(model.components[0].name) == "smc" + assert str(model.components[0].implementation) == "isr2d.smc" assert isinstance(model.conduits, list) assert len(model.conduits) == 0 def test_dump_model(dump_model: Callable) -> None: - ce1 = Component('ce1', 'test.impl1') - ce2 = Component('ce2', 'test.impl2') - cd1 = Conduit('ce1.state_out', 'ce2.init_in') - cd2 = Conduit('ce2.fini_out', 'ce1.boundary_in') - model = Model('test_model', [ce1, ce2], [cd1, cd2]) + ce1 = Component("ce1", "test.impl1") + ce2 = Component("ce2", "test.impl2") + cd1 = Conduit("ce1.state_out", "ce2.init_in") + cd2 = Conduit("ce2.fini_out", "ce1.boundary_in") + model = Model("test_model", [ce1, ce2], [cd1, cd2]) text = dump_model(model) - assert text == ('name: test_model\n' - 'components:\n' - ' ce1: test.impl1\n' - ' ce2: test.impl2\n' - 'conduits:\n' - ' ce1.state_out: ce2.init_in\n' - ' ce2.fini_out: ce1.boundary_in\n' - ) + assert text == ( + "name: test_model\n" + "components:\n" + " ce1: test.impl1\n" + " ce2: test.impl2\n" + "conduits:\n" + " ce1.state_out: ce2.init_in\n" + " ce2.fini_out: ce1.boundary_in\n" + ) def test_dump_no_conduits(dump_model: Callable) -> None: - ce1 = Component('ce1', 'test.impl1') - model = Model('test_model', [ce1]) + ce1 = Component("ce1", "test.impl1") + model = Model("test_model", [ce1]) text = dump_model(model) - assert text == ('name: test_model\n' - 'components:\n' - ' ce1: test.impl1\n' - ) + assert text == ("name: test_model\ncomponents:\n ce1: test.impl1\n") diff --git a/ymmsl/v0_1/tests/test_settings.py b/ymmsl/v0_1/tests/test_settings.py index 56c4dd3..a3ac19e 100644 --- a/ymmsl/v0_1/tests/test_settings.py +++ b/ymmsl/v0_1/tests/test_settings.py @@ -22,28 +22,27 @@ def test_create_settings(settings: Settings) -> None: def test_create_settings2() -> None: - setting_values: OrderedDict[str, SettingValue] = OrderedDict([ - ('submodel._muscle_grain', [0.01, 0.01]), - ('submodel._muscle_extent', [10.0, 3.0]), - ('submodel._muscle_timestep', 0.001), - ('submodel._muscle_total_time', 0.1), - ('bf.velocity', 0.48), - ('init.max_depth', 0.11) - ]) + setting_values: OrderedDict[str, SettingValue] = OrderedDict( + [ + ("submodel._muscle_grain", [0.01, 0.01]), + ("submodel._muscle_extent", [10.0, 3.0]), + ("submodel._muscle_timestep", 0.001), + ("submodel._muscle_total_time", 0.1), + ("bf.velocity", 0.48), + ("init.max_depth", 0.11), + ] + ) settings = Settings(setting_values) - assert list(settings.ordered_items()[0][0]) == [ - 'submodel', '_muscle_grain' - ] + assert list(settings.ordered_items()[0][0]) == ["submodel", "_muscle_grain"] assert cast(List[float], settings.ordered_items()[1][1])[1] == 3.0 - assert settings['submodel._muscle_timestep'] == 0.001 - assert list(settings.ordered_items()[4][0]) == ['bf', 'velocity'] - assert settings['init.max_depth'] == 0.11 + assert settings["submodel._muscle_timestep"] == 0.001 + assert list(settings.ordered_items()[4][0]) == ["bf", "velocity"] + assert settings["init.max_depth"] == 0.11 def test_from_broken_dict() -> None: with pytest.raises(ValueError): - Settings(OrderedDict([ - ('$invalid&', 12)])) + Settings(OrderedDict([("$invalid&", 12)])) def test_equality(settings: Settings) -> None: @@ -54,87 +53,91 @@ def test_equality(settings: Settings) -> None: assert not (settings2 != settings) settings1 = Settings() - settings1._store[Reference('x')] = 12 - settings1._store[Reference('y')] = 'test' - settings1._store[Reference('z')] = [1.4, 5.3] + settings1._store[Reference("x")] = 12 + settings1._store[Reference("y")] = "test" + settings1._store[Reference("z")] = [1.4, 5.3] - settings2._store[Reference('x')] = 12 - settings2._store[Reference('y')] = 'test' - settings2._store[Reference('z')] = [1.4, 5.3] + settings2._store[Reference("x")] = 12 + settings2._store[Reference("y")] = "test" + settings2._store[Reference("z")] = [1.4, 5.3] assert settings1 == settings2 assert settings2 == settings1 settings3 = Settings() - settings3._store[Reference('x')] = 12 - settings3._store[Reference('z')] = [1.4, 5.3] + settings3._store[Reference("x")] = 12 + settings3._store[Reference("z")] = [1.4, 5.3] assert settings3 != settings1 assert settings1 != settings3 settings4 = Settings() - settings4._store[Reference('x')] = 12 - settings4._store[Reference('y')] = 'test' - settings4._store[Reference('z')] = [1.41, 5.3] + settings4._store[Reference("x")] = 12 + settings4._store[Reference("y")] = "test" + settings4._store[Reference("z")] = [1.41, 5.3] assert settings1 != settings4 assert settings4 != settings1 assert settings3 != settings4 assert settings4 != settings3 - assert settings1 != 'test' + assert settings1 != "test" assert not (settings4 == 13) def test_to_string(settings: Settings) -> None: - assert str(settings) == 'OrderedDict()' - settings['test'] = 42 - assert 'test' in str(settings) - assert '42' in str(settings) + assert str(settings) == "OrderedDict()" + settings["test"] = 42 + assert "test" in str(settings) + assert "42" in str(settings) def test_contains(settings: Settings) -> None: - assert 'test' not in settings - assert Reference('test') not in settings + assert "test" not in settings + assert Reference("test") not in settings - settings['test'] = 1 - assert 'test' in settings - assert Reference('test') in settings + settings["test"] = 1 + assert "test" in settings + assert Reference("test") in settings def test_get_item(settings: Settings) -> None: - settings._store[Reference('test')] = 13 - assert settings[Reference('test')] == 13 - assert settings['test'] == 13 + settings._store[Reference("test")] = 13 + assert settings[Reference("test")] == 13 + assert settings["test"] == 13 def test_set_item(settings: Settings) -> None: - settings['param1'] = 3 - assert settings._store[Reference('param1')] == 3 + settings["param1"] = 3 + assert settings._store[Reference("param1")] == 3 - settings[Reference('param2')] = 4 - assert settings._store[Reference('param2')] == 4 + settings[Reference("param2")] = 4 + assert settings._store[Reference("param2")] == 4 def test_del_item(settings: Settings) -> None: - settings._store = OrderedDict([(Reference('param1'), 'test'), - (Reference('param2'), 0)]) - del settings['param1'] + settings._store = OrderedDict( + [(Reference("param1"), "test"), (Reference("param2"), 0)] + ) + del settings["param1"] assert len(settings._store) == 1 - assert Reference('param1') not in settings._store + assert Reference("param1") not in settings._store with pytest.raises(KeyError): - settings['param1'] # pylint: disable=pointless-statement - assert settings._store[Reference('param2')] == 0 + settings["param1"] # pylint: disable=pointless-statement + assert settings._store[Reference("param2")] == 0 def test_iter(settings: Settings) -> None: assert len(settings) == 0 assert list(settings.items()) == [] - settings._store = OrderedDict([ - (Reference('test1'), 13), - (Reference('test2'), 'testing'), - (Reference('test3'), [3.4, 5.6])]) + settings._store = OrderedDict( + [ + (Reference("test1"), 13), + (Reference("test2"), "testing"), + (Reference("test3"), [3.4, 5.6]), + ] + ) assert len(settings) == 3 for setting in settings: @@ -142,68 +145,75 @@ def test_iter(settings: Settings) -> None: for setting, value in settings.items(): assert ( - setting == 'test1' and value == 13 or - setting == 'test2' and value == 'testing' or - setting == 'test3' and value == [3.4, 5.6]) + setting == "test1" + and value == 13 + or setting == "test2" + and value == "testing" + or setting == "test3" + and value == [3.4, 5.6] + ) def test_update(settings: Settings) -> None: settings1 = Settings() - settings1['param1'] = 13 - settings1['param2'] = 'testing' + settings1["param1"] = 13 + settings1["param2"] = "testing" settings.update(settings1) assert len(settings) == 2 - assert settings['param1'] == 13 - assert settings['param2'] == 'testing' + assert settings["param1"] == 13 + assert settings["param2"] == "testing" settings2 = Settings() - settings2[Reference('param2')] = [[1.0, 2.0], [2.0, 3.0]] - settings2['param3'] = 3.1415 + settings2[Reference("param2")] = [[1.0, 2.0], [2.0, 3.0]] + settings2["param3"] = 3.1415 settings.update(settings2) assert len(settings) == 3 - assert settings['param1'] == 13 - assert settings['param2'] == [[1, 2], [2, 3]] - assert settings['param3'] == 3.1415 + assert settings["param1"] == 13 + assert settings["param2"] == [[1, 2], [2, 3]] + assert settings["param3"] == 3.1415 settings3 = Settings() - settings3[Reference('param1')] = True + settings3[Reference("param1")] = True settings.update(settings3) assert len(settings) == 3 - assert settings['param1'] is True - assert settings['param2'] == [[1, 2], [2, 3]] - assert settings['param3'] == 3.1415 + assert settings["param1"] is True + assert settings["param2"] == [[1, 2], [2, 3]] + assert settings["param3"] == 3.1415 def test_copy() -> None: settings1 = Settings() - settings1['test'] = 12 - settings1['test2'] = [23.0, 12.0] - settings1['test3'] = 'test3' + settings1["test"] = 12 + settings1["test2"] = [23.0, 12.0] + settings1["test3"] = "test3" settings2 = settings1.copy() assert settings1 == settings2 - settings2['test'] = 13 - assert settings2['test'] == 13 - assert settings1['test'] == 12 + settings2["test"] = 13 + assert settings2["test"] == 13 + assert settings1["test"] == 12 - cast(List[float], settings2['test2'])[0] = 24.0 - assert settings2['test2'] == [24.0, 12.0] - assert settings1['test2'] == [24.0, 12.0] + cast(List[float], settings2["test2"])[0] = 24.0 + assert settings2["test2"] == [24.0, 12.0] + assert settings1["test2"] == [24.0, 12.0] def test_as_ordered_dict(settings: Settings) -> None: - settings._store = OrderedDict([ - (Reference('test1'), 12), - (Reference('test2'), '12'), - (Reference('test3'), 'testing'), - (Reference('test4'), [12.3, 45.6])]) + settings._store = OrderedDict( + [ + (Reference("test1"), 12), + (Reference("test2"), "12"), + (Reference("test3"), "testing"), + (Reference("test4"), [12.3, 45.6]), + ] + ) settings_dict = settings.as_ordered_dict() - assert settings_dict['test1'] == 12 - assert settings_dict['test2'] == '12' - assert settings_dict['test3'] == 'testing' - assert settings_dict['test4'] == [12.3, 45.6] + assert settings_dict["test1"] == 12 + assert settings_dict["test2"] == "12" + assert settings_dict["test3"] == "testing" + assert settings_dict["test4"] == [12.3, 45.6] for i, (key, _) in enumerate(settings_dict.items()): assert key == f"test{i + 1}" @@ -212,55 +222,70 @@ def test_as_ordered_dict(settings: Settings) -> None: def test_load_settings() -> None: load_settings = yatiml.load_function(Settings, Identifier, Reference) - text = ('domain1._muscle_grain: [0.01]\n' - 'domain1._muscle_extent: [1.5]\n' - 'submodel1._muscle_timestep: 0.001\n' - 'submodel1._muscle_total_time: 100.0\n' - 'test_str: value\n' - 'test_int: 13\n' - 'test_bool: true\n' - 'test_list_int: [5, 7]\n' - 'test_list_float: [12.3, 1.3]\n') + text = ( + "domain1._muscle_grain: [0.01]\n" + "domain1._muscle_extent: [1.5]\n" + "submodel1._muscle_timestep: 0.001\n" + "submodel1._muscle_total_time: 100.0\n" + "test_str: value\n" + "test_int: 13\n" + "test_bool: true\n" + "test_list_int: [5, 7]\n" + "test_list_float: [12.3, 1.3]\n" + ) settings = load_settings(text) assert len(settings) == 9 - assert str(settings.ordered_items()[0][0]) == 'domain1._muscle_grain' - assert cast(List[float], settings['domain1._muscle_grain'])[0] == 0.01 - assert settings['submodel1._muscle_total_time'] == 100.0 - - assert str(settings.ordered_items()[4][0]) == 'test_str' - assert ([str(s[0]) for s in settings.ordered_items()] - == ['domain1._muscle_grain', 'domain1._muscle_extent', - 'submodel1._muscle_timestep', 'submodel1._muscle_total_time', - 'test_str', 'test_int', 'test_bool', 'test_list_int', - 'test_list_float']) + assert str(settings.ordered_items()[0][0]) == "domain1._muscle_grain" + assert cast(List[float], settings["domain1._muscle_grain"])[0] == 0.01 + assert settings["submodel1._muscle_total_time"] == 100.0 + + assert str(settings.ordered_items()[4][0]) == "test_str" + assert [str(s[0]) for s in settings.ordered_items()] == [ + "domain1._muscle_grain", + "domain1._muscle_extent", + "submodel1._muscle_timestep", + "submodel1._muscle_total_time", + "test_str", + "test_int", + "test_bool", + "test_list_int", + "test_list_float", + ] - assert settings['test_bool'] is True - assert settings['test_list_int'] == [5, 7] - assert settings['test_list_float'] == [12.3, 1.3] + assert settings["test_bool"] is True + assert settings["test_list_int"] == [5, 7] + assert settings["test_list_float"] == [12.3, 1.3] def test_dump_settings() -> None: dump_settings = yatiml.dumps_function(Identifier, Reference, Settings) - settings = Settings(OrderedDict([ - ('domain1._muscle_grain', [0.01]), - ('domain1._muscle_extent', [1.5]), - ('submodel1._muscle_timestep', 0.001), - ('submodel1._muscle_total_time', 100.0), - ('test_str', 'value'), - ('test_int', 12), - ('test_bool', True), - ('test_list_int', [19, 13]), - ('test_list_float', [12.3, 1.3])])) + settings = Settings( + OrderedDict( + [ + ("domain1._muscle_grain", [0.01]), + ("domain1._muscle_extent", [1.5]), + ("submodel1._muscle_timestep", 0.001), + ("submodel1._muscle_total_time", 100.0), + ("test_str", "value"), + ("test_int", 12), + ("test_bool", True), + ("test_list_int", [19, 13]), + ("test_list_float", [12.3, 1.3]), + ] + ) + ) text = dump_settings(settings) - assert text == ('domain1._muscle_grain: [0.01]\n' - 'domain1._muscle_extent: [1.5]\n' - 'submodel1._muscle_timestep: 0.001\n' - 'submodel1._muscle_total_time: 100.0\n' - 'test_str: value\n' - 'test_int: 12\n' - 'test_bool: true\n' - 'test_list_int: [19, 13]\n' - 'test_list_float: [12.3, 1.3]\n') + assert text == ( + "domain1._muscle_grain: [0.01]\n" + "domain1._muscle_extent: [1.5]\n" + "submodel1._muscle_timestep: 0.001\n" + "submodel1._muscle_total_time: 100.0\n" + "test_str: value\n" + "test_int: 12\n" + "test_bool: true\n" + "test_list_int: [19, 13]\n" + "test_list_float: [12.3, 1.3]\n" + ) diff --git a/ymmsl/v0_2/__init__.py b/ymmsl/v0_2/__init__.py index ead2de9..0f25f54 100644 --- a/ymmsl/v0_2/__init__.py +++ b/ymmsl/v0_2/__init__.py @@ -1,8 +1,8 @@ from ymmsl.v0_2.checkpoint import ( - CheckpointAtRule, - CheckpointRangeRule, - CheckpointRule, - Checkpoints, + CheckpointAtRule, + CheckpointRangeRule, + CheckpointRule, + Checkpoints, ) from ymmsl.v0_2.component import Component from ymmsl.v0_2.configuration import Configuration @@ -16,24 +16,52 @@ from ymmsl.v0_2.program import Program from ymmsl.v0_2.resolver import resolve from ymmsl.v0_2.resources import ( - MPICoresResReq, - MPINodesResReq, - ResourceRequirements, - ThreadedResReq, + MPICoresResReq, + MPINodesResReq, + ResourceRequirements, + ThreadedResReq, ) from ymmsl.v0_2.settings import Settings, SettingValue from ymmsl.v0_2.supported_settings import ( - SettingType, - SupportedSetting, - SupportedSettings, + SettingType, + SupportedSetting, + SupportedSettings, ) __all__ = [ - 'BaseEnv', 'CheckpointRule', 'CheckpointRangeRule', 'CheckpointAtRule', - 'Checkpoints', 'Component', 'Ports', 'Conduit', 'ConduitFilter', - 'Configuration', 'Document', 'ExecutionModel', 'Identifier', 'Implementation', - 'ImportKind', 'ImportStatement', 'KeepsStateForNextUse', 'Model', - 'MPICoresResReq', 'MPINodesResReq', 'Operator', 'Port', 'Ports', 'Program', - 'Reference', 'ReferencePart', 'resolve', 'ResourceRequirements', 'Settings', - 'SettingType', 'SettingValue', 'SupportedSetting', 'SupportedSettings', - 'ThreadedResReq', 'Timeline'] + "BaseEnv", + "CheckpointRule", + "CheckpointRangeRule", + "CheckpointAtRule", + "Checkpoints", + "Component", + "Ports", + "Conduit", + "ConduitFilter", + "Configuration", + "Document", + "ExecutionModel", + "Identifier", + "Implementation", + "ImportKind", + "ImportStatement", + "KeepsStateForNextUse", + "Model", + "MPICoresResReq", + "MPINodesResReq", + "Operator", + "Port", + "Ports", + "Program", + "Reference", + "ReferencePart", + "resolve", + "ResourceRequirements", + "Settings", + "SettingType", + "SettingValue", + "SupportedSetting", + "SupportedSettings", + "ThreadedResReq", + "Timeline", +] diff --git a/ymmsl/v0_2/checkpoint.py b/ymmsl/v0_2/checkpoint.py index cf221df..810ac9a 100644 --- a/ymmsl/v0_2/checkpoint.py +++ b/ymmsl/v0_2/checkpoint.py @@ -1,6 +1,6 @@ from ymmsl.v0_1.checkpoint import ( # noqa: F401 - CheckpointAtRule, - CheckpointRangeRule, - CheckpointRule, - Checkpoints, + CheckpointAtRule, + CheckpointRangeRule, + CheckpointRule, + Checkpoints, ) diff --git a/ymmsl/v0_2/component.py b/ymmsl/v0_2/component.py index a4dc941..e04b340 100644 --- a/ymmsl/v0_2/component.py +++ b/ymmsl/v0_2/component.py @@ -31,11 +31,16 @@ class Component: optional: Whether this component is optional multiplicity: The shape of the set of instances """ + def __init__( - self, name: str, ports: Ports, - description: str, implementation: Optional[str] = None, - optional: bool = False, - multiplicity: Union[None, int, List[int]] = None) -> None: + self, + name: str, + ports: Ports, + description: str, + implementation: Optional[str] = None, + optional: bool = False, + multiplicity: Union[None, int, List[int]] = None, + ) -> None: """Create a Component Args: @@ -56,8 +61,10 @@ def __init__( self.implementation: Optional[Reference] = Reference(implementation) for part in self.implementation: if isinstance(part, int): - raise ValueError(f"Component implementation {self.name} contains a" - " subscript, which is not allowed.") + raise ValueError( + f"Component implementation {self.name} contains a" + " subscript, which is not allowed." + ) else: self.implementation = None @@ -86,6 +93,7 @@ def instances(self) -> List[Reference]: A list with one Reference for each instance of this component. """ + def increment(index: List[int], dims: List[int]) -> None: # assumes index and dims are the same length > 0 # modifies index argument @@ -122,27 +130,27 @@ def generate_indices(multiplicity: List[int]) -> List[List[int]]: @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: - node.require_attribute('name', str) + node.require_attribute("name", str) @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - ports_node = node.get_attribute('ports').yaml_node - if ports_node.tag == 'tag:yaml.org,2002:null': - node.remove_attribute('ports') + ports_node = node.get_attribute("ports").yaml_node + if ports_node.tag == "tag:yaml.org,2002:null": + node.remove_attribute("ports") - descr = node.get_attribute('description') + descr = node.get_attribute("description") if descr.is_scalar(str): # output in block style ynode = cast(yaml.ScalarNode, descr.yaml_node) - ynode.style = '|' + ynode.style = "|" # ensure PyYAML actually uses block style ynode.value = remove_trailing_whitespace(ynode.value) - multiplicity = node.get_attribute('multiplicity') + multiplicity = node.get_attribute("multiplicity") items = multiplicity.seq_items() if len(items) == 0: - node.remove_attribute('multiplicity') + node.remove_attribute("multiplicity") elif len(items) == 1: - node.set_attribute('multiplicity', items[0].get_value()) + node.set_attribute("multiplicity", items[0].get_value()) node.remove_attributes_with_default_values(cls) diff --git a/ymmsl/v0_2/configuration.py b/ymmsl/v0_2/configuration.py index 59409d7..5542df7 100644 --- a/ymmsl/v0_2/configuration.py +++ b/ymmsl/v0_2/configuration.py @@ -47,22 +47,30 @@ class Configuration(Document): checkpoints: Defines when each model component should create a snapshot resume: Defines what snapshot each model component should resume from """ + def __init__( - self, description: str = '', - imports: Optional[Sequence[ImportStatement]] = None, - models: Optional[Union[ - Sequence[Model], MutableMapping[Reference, Model]]] = None, - custom_implementations: Optional[ - MutableMapping[Reference, Optional[Reference]]] = None, - settings: Optional[Settings] = None, - programs: Optional[Union[ - Sequence[Program], MutableMapping[Reference, Program]]] = None, - resources: Optional[Union[ + self, + description: str = "", + imports: Optional[Sequence[ImportStatement]] = None, + models: Optional[ + Union[Sequence[Model], MutableMapping[Reference, Model]] + ] = None, + custom_implementations: Optional[ + MutableMapping[Reference, Optional[Reference]] + ] = None, + settings: Optional[Settings] = None, + programs: Optional[ + Union[Sequence[Program], MutableMapping[Reference, Program]] + ] = None, + resources: Optional[ + Union[ Sequence[ResourceRequirements], - MutableMapping[Reference, ResourceRequirements]]] = None, - checkpoints: Optional[Checkpoints] = None, - resume: Optional[Dict[Reference, Path]] = None - ) -> None: + MutableMapping[Reference, ResourceRequirements], + ] + ] = None, + checkpoints: Optional[Checkpoints] = None, + resume: Optional[Dict[Reference, Path]] = None, + ) -> None: """Create a Configuration. Implementations and resources may be either a list of such @@ -80,13 +88,16 @@ def __init__( checkpoints: When each component should create a snapshot resume: What snapshot each component should resume from """ + def check_duplicate_impl_names( - typs: str, impls: abc.Sequence[Implementation]) -> None: + typs: str, impls: abc.Sequence[Implementation] + ) -> None: for i, impl in enumerate(impls): for j in range(i): if impls[j].name == impl.name: raise ValueError( - f'Found two {typs} that are both named "{impl.name}"') + f'Found two {typs} that are both named "{impl.name}"' + ) self.description = description @@ -98,7 +109,7 @@ def check_duplicate_impl_names( if models is None: self.models: MutableMapping[Reference, Model] = dict() elif isinstance(models, abc.Sequence): - check_duplicate_impl_names('models', models) + check_duplicate_impl_names("models", models) self.models = {copy(model.name): model for model in models} elif isinstance(models, Model): self.models = {models.name: models} @@ -120,7 +131,7 @@ def check_duplicate_impl_names( if programs is None: self.programs: MutableMapping[Reference, Program] = dict() elif isinstance(programs, abc.Sequence): - check_duplicate_impl_names('programs', programs) + check_duplicate_impl_names("programs", programs) self.programs = {prog.name: prog for prog in programs} else: self.programs = programs @@ -142,7 +153,7 @@ def check_duplicate_impl_names( else: self.resume = resume - def update(self, overlay: 'Configuration') -> None: + def update(self, overlay: "Configuration") -> None: """Update this configuration with the given overlay. This will copy settings from overlay on top of the current settings, and merge @@ -154,9 +165,9 @@ def update(self, overlay: 'Configuration') -> None: if not self.description.strip(): self.description = overlay.description elif overlay.description.strip(): - if not self.description.endswith('\n'): - self.description += '\n' - self.description += '\n' + overlay.description + if not self.description.endswith("\n"): + self.description += "\n" + self.description += "\n" + overlay.description if not self.imports: self.imports = overlay.imports @@ -166,8 +177,9 @@ def update(self, overlay: 'Configuration') -> None: if self.models and overlay.models: raise RuntimeError( - 'Multiple ymmsl files containing models specified. Please' - ' use the import functionality instead.') + "Multiple ymmsl files containing models specified. Please" + " use the import functionality instead." + ) self.models.update(overlay.models) @@ -178,9 +190,10 @@ def update(self, overlay: 'Configuration') -> None: overlap = [p for p in self.programs if p in overlay.programs] if any(overlap): raise RuntimeError( - 'Multiple programs with the same name were found. Please ensure' - ' that all programs have a unique name. The duplicate names were:' - f' {", ".join(map(str, overlap))}.') + "Multiple programs with the same name were found. Please ensure" + " that all programs have a unique name. The duplicate names were:" + f" {', '.join(map(str, overlap))}." + ) self.programs.update(overlay.programs) self.resources.update(overlay.resources) @@ -188,8 +201,8 @@ def update(self, overlay: 'Configuration') -> None: self.resume.update(overlay.resume) def check_consistent( - self, check_runnable: bool = True, selected_model: Optional[str] = None - ) -> None: + self, check_runnable: bool = True, selected_model: Optional[str] = None + ) -> None: """Checks that the configuration is internally consistent. This checks: @@ -217,7 +230,7 @@ def check_consistent( for model in self.models.values(): model_errors = model.check_consistent() - errors.extend([f'In model {model.name}: {e}' for e in model_errors]) + errors.extend([f"In model {model.name}: {e}" for e in model_errors]) errors.extend(self._check_duplicate_implementations()) @@ -233,9 +246,9 @@ def check_consistent( if errors: raise RuntimeError( - 'The configuration is internally inconsistent. The following' - ' problems were found:\n- ' - + '\n- '.join(errors)) + "The configuration is internally inconsistent. The following" + " problems were found:\n- " + "\n- ".join(errors) + ) def get_resources(self, name: Reference) -> ResourceRequirements: """Get the resource requirements for a component. @@ -253,7 +266,8 @@ def get_resources(self, name: Reference) -> ResourceRequirements: res_req = self.resources.get(name) if res_req is None: _logger.debug( - f'No resources defined for {name}, using default of 1 thread.') + f"No resources defined for {name}, using default of 1 thread." + ) res_req = ThreadedResReq(name, 1) return res_req @@ -281,25 +295,27 @@ def root_model(self, selected_model: Optional[Reference] = None) -> Model: """ root_models = self._root_models() if not root_models: - raise RuntimeError('No model was found in this configuration.') + raise RuntimeError("No model was found in this configuration.") if selected_model: match = [m for m in root_models if m.name == selected_model] if match: return match[0] - models = '\n- '.join([str(m.name) for m in root_models]) + models = "\n- ".join([str(m.name) for m in root_models]) raise RuntimeError( - f'The selected model "{selected_model}" could not be found in this' - f' configuration. The following models are present:\n- {models}') + f'The selected model "{selected_model}" could not be found in this' + f" configuration. The following models are present:\n- {models}" + ) if len(root_models) == 1: return root_models[0] - models = '\n- '.join([str(m.name) for m in root_models]) + models = "\n- ".join([str(m.name) for m in root_models]) raise RuntimeError( - 'Multiple models were found in this configuration that are not used' - f' as implementations:\n\n- {models}') + "Multiple models were found in this configuration that are not used" + f" as implementations:\n\n- {models}" + ) def _root_models(self) -> List[Model]: """Models in this configuration that are not used as implementations.""" @@ -332,28 +348,26 @@ def _component_paths(self) -> Dict[Reference, Component]: RuntimeError: if a loop is detected """ result = dict() - queue: List[Tuple[Model, Reference, List[Tuple[Reference, Reference]]]] = \ - [(m, m.name, []) for m in self._root_models()] - _logger.debug(f'cmp_paths: initial queue: {[t[0].name for t in queue]}') + queue: List[Tuple[Model, Reference, List[Tuple[Reference, Reference]]]] = [ + (m, m.name, []) for m in self._root_models() + ] + _logger.debug(f"cmp_paths: initial queue: {[t[0].name for t in queue]}") while queue: model, prefix, seen = queue.pop(0) - _logger.debug(f'cmp_paths: {model.name} {prefix} {seen}') + _logger.debug(f"cmp_paths: {model.name} {prefix} {seen}") for component in model.components.values(): path = prefix + component.name impl = self.custom_implementations.get(path, component.implementation) if impl is not None: if impl in self.models: if [m for _, m in seen if m == impl]: - msg = 'A loop of components and models was detected:\n' + msg = "A loop of components and models was detected:\n" for p, m in seen: - msg += f'- component {p} is implemented using {m}\n' - msg += ( - f'- component {path} is implemented using' - f' {impl}\n') + msg += f"- component {p} is implemented using {m}\n" + msg += f"- component {path} is implemented using {impl}\n" raise RuntimeError(msg) - queue.append(( - self.models[impl], path, seen + [(path, impl)])) + queue.append((self.models[impl], path, seen + [(path, impl)])) result[path] = component return result @@ -371,12 +385,14 @@ def _check_duplicate_implementations(self) -> List[str]: for model_name in self.models: if program_name == model_name: errors.append( - f'There is a program named "{program_name}" and also a' - f' model named "{model_name}", which is not allowed') + f'There is a program named "{program_name}" and also a' + f' model named "{model_name}", which is not allowed' + ) return errors def _check_consistent_ports( - self, component_paths: Dict[Reference, Component]) -> List[str]: + self, component_paths: Dict[Reference, Component] + ) -> List[str]: """Check that components have implementations with compatible ports. This checks that each component declares ports that its implementation also @@ -403,21 +419,24 @@ def _check_consistent_ports( for port_name in component.ports: if port_name not in impl.ports: errors.append( - f'Component "{component.name}" declares port' - f' "{port_name}" that its implementation' - f' "{impl.name}" does not have') + f'Component "{component.name}" declares port' + f' "{port_name}" that its implementation' + f' "{impl.name}" does not have' + ) else: if component.ports[port_name] != impl.ports[port_name]: errors.append( - f'Component "{component.name}" declares port' - f' "{port_name}" that its implementation' - f' "{impl.name}" has, but with a different' - ' operator or timeline.') + f'Component "{component.name}" declares port' + f' "{port_name}" that its implementation' + f' "{impl.name}" has, but with a different' + " operator or timeline." + ) return errors def _check_implementations_exist( - self, component_paths: Dict[Reference, Component]) -> List[str]: + self, component_paths: Dict[Reference, Component] + ) -> List[str]: """Check that all components with an implementation have a valid one. Components with implementation None are not considered broken if marked as @@ -434,18 +453,21 @@ def _check_implementations_exist( if impl is None: if not component.optional: errors.append( - f'Component "{path}" is not marked optional, and does not' - ' have an implementation.') + f'Component "{path}" is not marked optional, and does not' + " have an implementation." + ) else: if impl not in self.models and impl not in self.programs: errors.append( - f'Component "{path}" has implementation "{impl}", but no' - ' program or model with that name was given. Did you forget' - ' to import it?') + f'Component "{path}" has implementation "{impl}", but no' + " program or model with that name was given. Did you forget" + " to import it?" + ) return errors def _check_custom_implementations( - self, component_paths: Dict[Reference, Component]) -> List[str]: + self, component_paths: Dict[Reference, Component] + ) -> List[str]: """Check that all custom implementations refer to a correct path. Args: @@ -457,12 +479,14 @@ def _check_custom_implementations( for path in self.custom_implementations: if path not in component_paths: errors.append( - f'A custom implementation is specified for component "{path}",' - ' but no such component is present in the model.') + f'A custom implementation is specified for component "{path}",' + " but no such component is present in the model." + ) return errors def _check_consistent_settings( - self, component_paths: Dict[Reference, Component]) -> List[str]: + self, component_paths: Dict[Reference, Component] + ) -> List[str]: """Check that setting names and types match supported settings. This checks that every implementation with a supported_settings declaration will @@ -491,24 +515,32 @@ def _check_consistent_settings( if impl.supported_settings: for name, sup_set in impl.supported_settings: errs.extend( - self._check_supported_setting( - component, path, name, sup_set.typ, impl)) + self._check_supported_setting( + component, path, name, sup_set.typ, impl + ) + ) if len(errs) > 7: n = len(errs) - 6 errs = errs[:6] errs.append( - f'Another {n} inconsistent settings were found. Is' - f' "{impl.name}" the correct implementation for component' - f' "{component.name}"?') + f"Another {n} inconsistent settings were found. Is" + f' "{impl.name}" the correct implementation for component' + f' "{component.name}"?' + ) errors.extend(errs) return errors def _check_supported_setting( - self, component: Component, component_path: Reference, name: Identifier, - typ: SettingType, impl: Implementation) -> List[str]: + self, + component: Component, + component_path: Reference, + name: Identifier, + typ: SettingType, + impl: Implementation, + ) -> List[str]: """Check that the value of the given setting matches the given type. This implements the standard setting lookup, then checks any found setting value @@ -531,12 +563,13 @@ def _check_supported_setting( if isinstance(val, str): val_str = f'"{val}"' errors.append( - f'Instance "{instance_path}" of component' - f' "{component_path[1:]}" with implementation' - f' "{impl.name}"' - f' has a supported setting "{name}" with type' - f' {typ.value}, but setting "{found_setting}" has value' - f' {val_str}, which does not match that type') + f'Instance "{instance_path}" of component' + f' "{component_path[1:]}" with implementation' + f' "{impl.name}"' + f' has a supported setting "{name}" with type' + f' {typ.value}, but setting "{found_setting}" has value' + f" {val_str}, which does not match that type" + ) break return errors @@ -565,8 +598,8 @@ def _setting_type_matches(self, value: SettingValue, typ: SettingType) -> bool: return False def _check_resources( - self, component_paths: Dict[Reference, Component], - selected_model: Optional[str]) -> List[str]: + self, component_paths: Dict[Reference, Component], selected_model: Optional[str] + ) -> List[str]: """Check that each component path has a corresponding resource request. For non-MPI components, resources are optional: if not specified, @@ -576,19 +609,21 @@ def _check_resources( """ errors = list() for path, component in component_paths.items(): - _logger.debug(f'Checking resources for {path} {component.name}') + _logger.debug(f"Checking resources for {path} {component.name}") if selected_model is not None and path[0] != selected_model: continue impl_ref = self.custom_implementations.get(path, component.implementation) - _logger.debug(f'Implementation: {impl_ref}') + _logger.debug(f"Implementation: {impl_ref}") if impl_ref is None or impl_ref not in self.programs: continue impl = self.programs[impl_ref] em_mpi = impl.execution_model in ( - ExecutionModel.OPENMPI, ExecutionModel.INTELMPI, - ExecutionModel.SRUNMPI) + ExecutionModel.OPENMPI, + ExecutionModel.INTELMPI, + ExecutionModel.SRUNMPI, + ) em_nompi = impl.execution_model is ExecutionModel.DIRECT @@ -597,18 +632,21 @@ def _check_resources( errors.append(f'Component "{path}" is missing a resource request') else: res_mpi = isinstance( - self.resources[path], (MPICoresResReq, MPINodesResReq)) + self.resources[path], (MPICoresResReq, MPINodesResReq) + ) if em_mpi and not res_mpi: errors.append( - f'Component "{path}" has implementation "{impl_ref}",' - ' which has an MPI execution model, but the resources' - ' requested for it do not ask for MPI processes.') + f'Component "{path}" has implementation "{impl_ref}",' + " which has an MPI execution model, but the resources" + " requested for it do not ask for MPI processes." + ) elif res_mpi and em_nompi: errors.append( - f'Component "{path}" has implementation "{impl_ref}",' - ' which has a non-MPI execution model, but the resources' - ' requested for it ask for MPI processes.') + f'Component "{path}" has implementation "{impl_ref}",' + " which has a non-MPI execution model, but the resources" + " requested for it ask for MPI processes." + ) return errors @@ -622,64 +660,58 @@ def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: @classmethod def _yatiml_savorize(cls, node: yatiml.Node) -> None: - node.map_attribute_to_index('models', 'name') - if not node.has_attribute('settings'): - node.set_attribute('settings', None) - node.map_attribute_to_index('programs', 'name') - node.map_attribute_to_index('resources', 'name') + node.map_attribute_to_index("models", "name") + if not node.has_attribute("settings"): + node.set_attribute("settings", None) + node.map_attribute_to_index("programs", "name") + node.map_attribute_to_index("resources", "name") @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - descr = node.get_attribute('description') + descr = node.get_attribute("description") if descr.is_scalar(str): - if descr.get_value() == '': - node.remove_attribute('description') + if descr.get_value() == "": + node.remove_attribute("description") else: # output in block style ynode = cast(yaml.ScalarNode, descr.yaml_node) - ynode.style = '|' + ynode.style = "|" # ensure PyYAML actually uses block style ynode.value = remove_trailing_whitespace(ynode.value) - imports = node.get_attribute('imports') + imports = node.get_attribute("imports") if imports.is_sequence() and imports.is_empty(): - node.remove_attribute('imports') - - models = node.get_attribute('models') - if (models.is_scalar(type(None)) or models.is_mapping() and models.is_empty()): - node.remove_attribute('models') - node.index_attribute_to_map('models', 'name') - - if node.get_attribute('custom_implementations').is_scalar(type(None)): - node.remove_attribute('custom_implementations') - if len(node.get_attribute('custom_implementations').yaml_node.value) == 0: - node.remove_attribute('custom_implementations') - - if node.get_attribute('settings').is_scalar(type(None)): - node.remove_attribute('settings') - if len(node.get_attribute('settings').yaml_node.value) == 0: - node.remove_attribute('settings') - - progs = node.get_attribute('programs') - if (progs.is_scalar(type(None)) or progs.is_mapping() and progs.is_empty()): - node.remove_attribute('programs') - node.index_attribute_to_map('programs', 'name') - - res = node.get_attribute('resources') - if ( - res.is_scalar(type(None)) or - res.is_mapping() and res.is_empty()): - node.remove_attribute('resources') - node.index_attribute_to_map('resources', 'name') - - cp = node.get_attribute('checkpoints') - if ( - cp.is_scalar(type(None)) or - cp.is_mapping() and cp.is_empty()): - node.remove_attribute('checkpoints') - - resu = node.get_attribute('resume') - if ( - resu.is_scalar(type(None)) or - resu.is_mapping() and resu.is_empty()): - node.remove_attribute('resume') + node.remove_attribute("imports") + + models = node.get_attribute("models") + if models.is_scalar(type(None)) or models.is_mapping() and models.is_empty(): + node.remove_attribute("models") + node.index_attribute_to_map("models", "name") + + if node.get_attribute("custom_implementations").is_scalar(type(None)): + node.remove_attribute("custom_implementations") + if len(node.get_attribute("custom_implementations").yaml_node.value) == 0: + node.remove_attribute("custom_implementations") + + if node.get_attribute("settings").is_scalar(type(None)): + node.remove_attribute("settings") + if len(node.get_attribute("settings").yaml_node.value) == 0: + node.remove_attribute("settings") + + progs = node.get_attribute("programs") + if progs.is_scalar(type(None)) or progs.is_mapping() and progs.is_empty(): + node.remove_attribute("programs") + node.index_attribute_to_map("programs", "name") + + res = node.get_attribute("resources") + if res.is_scalar(type(None)) or res.is_mapping() and res.is_empty(): + node.remove_attribute("resources") + node.index_attribute_to_map("resources", "name") + + cp = node.get_attribute("checkpoints") + if cp.is_scalar(type(None)) or cp.is_mapping() and cp.is_empty(): + node.remove_attribute("checkpoints") + + resu = node.get_attribute("resume") + if resu.is_scalar(type(None)) or resu.is_mapping() and resu.is_empty(): + node.remove_attribute("resume") diff --git a/ymmsl/v0_2/document.py b/ymmsl/v0_2/document.py index 160e594..4ff2db7 100644 --- a/ymmsl/v0_2/document.py +++ b/ymmsl/v0_2/document.py @@ -1,4 +1,5 @@ """Defines the YAML document and version tag.""" + import yatiml from ymmsl.document import Document as DocumentBase @@ -14,12 +15,12 @@ class Document(DocumentBase): @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: node.require_mapping() - node.require_attribute('ymmsl_version') - node.require_attribute_value('ymmsl_version', 'v0.2') + node.require_attribute("ymmsl_version") + node.require_attribute_value("ymmsl_version", "v0.2") @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - node.set_attribute('ymmsl_version', 'v0.2') + node.set_attribute("ymmsl_version", "v0.2") # The above adds the attribute to the end, but we want it at # the top; this moves it there. node.yaml_node.value.insert(0, node.yaml_node.value[-1]) @@ -27,4 +28,4 @@ def _yatiml_sweeten(cls, node: yatiml.Node) -> None: @classmethod def _yatiml_savorize(cls, node: yatiml.Node) -> None: - node.remove_attribute('ymmsl_version') + node.remove_attribute("ymmsl_version") diff --git a/ymmsl/v0_2/execution.py b/ymmsl/v0_2/execution.py index 34783d0..645a6bc 100644 --- a/ymmsl/v0_2/execution.py +++ b/ymmsl/v0_2/execution.py @@ -8,6 +8,7 @@ class ExecutionModel(Enum): """Describes how to start a model component.""" + DIRECT = 1 """Start directly on the allocated core(s), without MPI.""" OPENMPI = 2 diff --git a/ymmsl/v0_2/implementation.py b/ymmsl/v0_2/implementation.py index 0f37729..f41eddc 100644 --- a/ymmsl/v0_2/implementation.py +++ b/ymmsl/v0_2/implementation.py @@ -21,10 +21,14 @@ class Implementation: name: Name of this implementation, must be a valid Reference ports: The ports this implementation has on which it sends and receives messages """ + def __init__( - self, name: str, ports: Optional[Ports] = None, - description: str = 'Please add a description!', - supported_settings: Optional[SupportedSettings] = None) -> None: + self, + name: str, + ports: Optional[Ports] = None, + description: str = "Please add a description!", + supported_settings: Optional[SupportedSettings] = None, + ) -> None: """Create an Implementation Args: @@ -49,18 +53,18 @@ def __init__( @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - if len(node.get_attribute('ports').yaml_node.value) == 0: - node.remove_attribute('ports') + if len(node.get_attribute("ports").yaml_node.value) == 0: + node.remove_attribute("ports") - descr = node.get_attribute('description') + descr = node.get_attribute("description") if descr.is_scalar(str): # output in block style ynode = cast(yaml.ScalarNode, descr.yaml_node) - ynode.style = '|' + ynode.style = "|" # ensure PyYAML actually uses block style ynode.value = remove_trailing_whitespace(ynode.value) - if len(node.get_attribute('supported_settings').yaml_node.value) == 0: - node.remove_attribute('supported_settings') + if len(node.get_attribute("supported_settings").yaml_node.value) == 0: + node.remove_attribute("supported_settings") node.remove_attributes_with_default_values(cls) diff --git a/ymmsl/v0_2/imports.py b/ymmsl/v0_2/imports.py index 9dfc6f8..6e75880 100644 --- a/ymmsl/v0_2/imports.py +++ b/ymmsl/v0_2/imports.py @@ -13,6 +13,7 @@ class ImportKind(Enum): Currently only IMPLEMENTATION is supported, which is used to import either a program or a model. In the future other things will be importable as well. """ + IMPLEMENTATION = 1 @classmethod @@ -45,6 +46,7 @@ class ImportStatement: kind: Kind of object to import, currently always 'implementation' name: Name of the object to import from it """ + def __init__(self, module: str, kind: str, name: str) -> None: """Create an ImportStatement @@ -53,7 +55,7 @@ def __init__(self, module: str, kind: str, name: str) -> None: kind: Kind of object to import name: Name of the object to import """ - path_parts = module.split('.') + path_parts = module.split(".") for path_part in path_parts: Identifier(path_part) @@ -63,7 +65,7 @@ def __init__(self, module: str, kind: str, name: str) -> None: self.kind = ImportKind[kind.upper()] except KeyError: raise ValueError( - f'{kind} is not a valid kind of object to import. Try' + f"{kind} is not a valid kind of object to import. Try" ' "implementation" instead to import a program or a model.' ) from None @@ -74,7 +76,7 @@ def module_path(self) -> Path: This returns a path a/b/c.ymmsl for module a.b.c. """ - return Path('/'.join(map(str, self.module)) + '.ymmsl') + return Path("/".join(map(str, self.module)) + ".ymmsl") def full_name(self) -> Reference: """Return the full name of the imported object. @@ -93,16 +95,16 @@ def _yatiml_savorize(cls, node: yatiml.Node) -> None: module, kind, name = cls._parse_string_representation(text) node.make_mapping() - node.set_attribute('module', module) - node.set_attribute('kind', kind) - node.set_attribute('name', name) + node.set_attribute("module", module) + node.set_attribute("kind", kind) + node.set_attribute("name", name) @classmethod def _yatiml_sweeten(self, node: yatiml.Node) -> None: - module = node.get_attribute('module').get_value() - kind = node.get_attribute('kind').get_value() - name = node.get_attribute('name').get_value() - node.set_value(f'from {module} import {kind} {name}') + module = node.get_attribute("module").get_value() + kind = node.get_attribute("kind").get_value() + name = node.get_attribute("name").get_value() + node.set_value(f"from {module} import {kind} {name}") @classmethod def _parse_string_representation(cls, text: str) -> Tuple[str, str, str]: @@ -110,36 +112,38 @@ def _parse_string_representation(cls, text: str) -> Tuple[str, str, str]: parts = text.split() - if len(parts) > 0 and parts[0] != 'from': + if len(parts) > 0 and parts[0] != "from": raise RuntimeError( - f'Import statement "{text}" does not start with "from".\n' + - help_text) + f'Import statement "{text}" does not start with "from".\n' + help_text + ) if len(parts) > 1: module = parts[1] - if len(parts) > 2 and parts[2] != 'import': + if len(parts) > 2 and parts[2] != "import": raise RuntimeError( - f'Import statement "{text}" does not have "import" part.\n' + - help_text) + f'Import statement "{text}" does not have "import" part.\n' + help_text + ) if len(parts) > 3: kind = parts[3] else: raise RuntimeError( - f'Import statement "{text}" does not specify which kind of thing' - ' to import.\n' + help_text) + f'Import statement "{text}" does not specify which kind of thing' + " to import.\n" + help_text + ) if len(parts) > 4: name = parts[4] else: raise RuntimeError( - f'Import statement "{text}" does not specify the name of the thing' - ' to import.\n' + help_text) + f'Import statement "{text}" does not specify the name of the thing' + " to import.\n" + help_text + ) if len(parts) > 5: raise RuntimeError( - f'Extra text found at end of import statement "{text}".\n' + - help_text) + f'Extra text found at end of import statement "{text}".\n' + help_text + ) return module, kind, name diff --git a/ymmsl/v0_2/model.py b/ymmsl/v0_2/model.py index 9e0ccf1..cc6edcd 100644 --- a/ymmsl/v0_2/model.py +++ b/ymmsl/v0_2/model.py @@ -28,13 +28,14 @@ class ConduitFilter(Enum): Objects of this class represent the different possible filters. """ - LAST = 'last' + + LAST = "last" """Pass only the last message and drop any preceding ones.""" - REPEAT = 'repeat' + REPEAT = "repeat" """Repeat a single message as often as needed.""" - PAD = 'pad' + PAD = "pad" """Pass a single message and then generate nil-messages as needed.""" def is_reducer(self) -> bool: @@ -82,9 +83,13 @@ class Conduit: receiver: The receiving port that this conduit is connected to. filters: A list of filters, or a string containing space-separated filter names """ + def __init__( - self, sender: str, receiver: str, - filters: Optional[Union[str, List[ConduitFilter]]] = None) -> None: + self, + sender: str, + receiver: str, + filters: Optional[Union[str, List[ConduitFilter]]] = None, + ) -> None: """Create a Conduit. Args: @@ -101,7 +106,8 @@ def __init__( else: if filters: raise ValueError( - 'Cannot specify filters both in receiver and in filters') + "Cannot specify filters both in receiver and in filters" + ) filters = recv_pieces[0] self.receiver = Reference(recv_pieces[1]) @@ -123,10 +129,10 @@ def __init__( def __str__(self) -> str: """Return a string representation of the object.""" if not self.filters: - filter_clause = '' + filter_clause = "" else: - filter_clause = ' -> ' + ' '.join([f.value for f in self.filters]) - return f'Conduit({self.sender}{filter_clause} -> {self.receiver})' + filter_clause = " -> " + " ".join([f.value for f in self.filters]) + return f"Conduit({self.sender}{filter_clause} -> {self.receiver})" def __eq__(self, other: Any) -> bool: """Returns whether the conduits are equal. @@ -136,8 +142,10 @@ def __eq__(self, other: Any) -> bool: if not isinstance(other, Conduit): return NotImplemented return ( - self.sender == other.sender and self.receiver == other.receiver and - self.filters == other.filters) + self.sender == other.sender + and self.receiver == other.receiver + and self.filters == other.filters + ) @staticmethod def __check_reference(ref: Reference) -> None: @@ -146,14 +154,16 @@ def __check_reference(ref: Reference) -> None: for part in ref: if isinstance(part, int): raise ValueError( - f'Reference {ref} contains a subscript, which is not allowed in' - ' conduits.') + f"Reference {ref} contains a subscript, which is not allowed in" + " conduits." + ) # check that the length is at least 1 if len(ref) < 1: raise ValueError( - 'Senders and receivers in conduits must have either a port name,' - ' or a component name, a period, and then a port name.') + "Senders and receivers in conduits must have either a port name," + " or a component name, a period, and then a port name." + ) def sending_component(self) -> Reference: """Returns a reference to the sending component.""" @@ -185,20 +195,19 @@ def _filters_receiver(self) -> str: filter_strs = [f.value for f in self.filters] if filter_strs: - result = ' '.join(filter_strs) + ' ' + result + result = " ".join(filter_strs) + " " + result return result @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - filters = node.get_attribute('filters') + filters = node.get_attribute("filters") filter_strs = [v.value for v in filters.yaml_node.value] if filter_strs: - recv_str = cast(str, node.get_attribute('receiver').get_value()) - node.set_attribute( - 'receiver', ' '.join(filter_strs) + ' ' + recv_str) + recv_str = cast(str, node.get_attribute("receiver").get_value()) + node.set_attribute("receiver", " ".join(filter_strs) + " " + recv_str) - node.remove_attribute('filters') + node.remove_attribute("filters") class MulticastConduit: @@ -216,6 +225,7 @@ class MulticastConduit: Once parsed and populated in :class:`Model`, a multicast is identified by two or more conduits with the same :attr:`Conduit.sender`. """ + def __init__(self, sender: str, receiver: List[str]) -> None: """Create a Multicast Conduit. @@ -260,12 +270,16 @@ class Model(Implementation): components: A list of components making up the model. conduits: A list of conduits connecting the components. """ + def __init__( - self, name: str, ports: Optional[Ports] = None, - description: str = '', - supported_settings: Optional[SupportedSettings] = None, - components: Optional[Sequence[Component]] = None, - conduits: Optional[Sequence[AnyConduit]] = None) -> None: + self, + name: str, + ports: Optional[Ports] = None, + description: str = "", + supported_settings: Optional[SupportedSettings] = None, + components: Optional[Sequence[Component]] = None, + conduits: Optional[Sequence[AnyConduit]] = None, + ) -> None: """Create a Model. Args: @@ -283,13 +297,13 @@ def __init__( else: for i1, c1 in enumerate(components): num_conflicts = 0 - for i2 in range(i1-1): + for i2 in range(i1 - 1): if c1.name == components[i2].name: num_conflicts += 1 if num_conflicts > 0: raise ValueError( - f'There are {num_conflicts + 1} components named' - f' {c1.name}.') + f"There are {num_conflicts + 1} components named {c1.name}." + ) self.components = {copy(c.name): c for c in components} @@ -322,64 +336,70 @@ def check_consistent(self) -> List[str]: return errors def _check_sending_side( - self, conduit: Conduit, model_receiving_ports: List[Identifier] - ) -> List[str]: + self, conduit: Conduit, model_receiving_ports: List[Identifier] + ) -> List[str]: """Check the sending side of a conduit.""" errors = list() if not conduit.sending_component(): # from model input port if conduit.sending_port() not in model_receiving_ports: errors.append( - f'Conduit {conduit} refers to an incoming model port' - f' {conduit.sending_port()} that does not exist.') + f"Conduit {conduit} refers to an incoming model port" + f" {conduit.sending_port()} that does not exist." + ) else: # from component if conduit.sending_component() not in self.components: errors.append( - f'Conduit {conduit} refers to a component named' - f' {conduit.sending_component()}, which is not present in' - ' the model.') + f"Conduit {conduit} refers to a component named" + f" {conduit.sending_component()}, which is not present in" + " the model." + ) else: snd_cmp = self.components[conduit.sending_component()] cmp_sending_ports = snd_cmp.ports.sending_port_names() if conduit.sending_port() not in cmp_sending_ports: errors.append( - f'Conduit {conduit} refers to a sending port named' - f' {conduit.sending_port()}, which is not present on' - f' sending component {snd_cmp.name}, or is not an O_I or' - ' O_F port.') + f"Conduit {conduit} refers to a sending port named" + f" {conduit.sending_port()}, which is not present on" + f" sending component {snd_cmp.name}, or is not an O_I or" + " O_F port." + ) return errors def _check_receiving_side( - self, conduit: Conduit, model_sending_ports: List[Identifier] - ) -> List[str]: + self, conduit: Conduit, model_sending_ports: List[Identifier] + ) -> List[str]: """Check the receiving side of a conduit.""" errors = list() if not conduit.receiving_component(): # to model output port if conduit.receiving_port() not in model_sending_ports: errors.append( - f'Conduit {conduit} refers to an incoming model port' - f' {conduit.receiving_port()} that does not exist.') + f"Conduit {conduit} refers to an incoming model port" + f" {conduit.receiving_port()} that does not exist." + ) else: # to component if conduit.receiving_component() not in self.components: errors.append( - f'Conduit {conduit} refers to a component named' - f' {conduit.receiving_component()}, which is not present in' - ' the model.') + f"Conduit {conduit} refers to a component named" + f" {conduit.receiving_component()}, which is not present in" + " the model." + ) else: rcvng_cmp = self.components[conduit.receiving_component()] - rcvr_recv_ports = ( - rcvng_cmp.ports.receiving_port_names() + - ['muscle_settings_in']) + rcvr_recv_ports = rcvng_cmp.ports.receiving_port_names() + [ + "muscle_settings_in" + ] if conduit.receiving_port() not in rcvr_recv_ports: errors.append( - f'Conduit {conduit} refers to a receiving port named' - f' {conduit.receiving_port()}, which is not present on' - f' receiving component {rcvng_cmp.name}, or is not an' - ' F_INIT or S port.') + f"Conduit {conduit} refers to a receiving port named" + f" {conduit.receiving_port()}, which is not present on" + f" receiving component {rcvng_cmp.name}, or is not an" + " F_INIT or S port." + ) return errors def _conduits_for_export(self) -> List[AnyConduit]: @@ -397,19 +417,25 @@ def _conduits_for_export(self) -> List[AnyConduit]: if len(conduits) == 1: conduit_list.append(conduits[0]) else: - conduit_list.append(MulticastConduit( + conduit_list.append( + MulticastConduit( str(sender), - [str(conduit._filters_receiver()) for conduit in conduits])) + [str(conduit._filters_receiver()) for conduit in conduits], + ) + ) return conduit_list def _yatiml_attributes(self) -> OrderedDict: - return OrderedDict([ - ('name', self.name), - ('ports', self.ports), - ('description', self.description), - ('supported_settings', self.supported_settings), - ('components', self.components), - ('conduits', self._conduits_for_export())]) + return OrderedDict( + [ + ("name", self.name), + ("ports", self.ports), + ("description", self.description), + ("supported_settings", self.supported_settings), + ("components", self.components), + ("conduits", self._conduits_for_export()), + ] + ) @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: @@ -418,14 +444,14 @@ def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: @classmethod def _yatiml_savorize(cls, node: yatiml.Node) -> None: - node.map_attribute_to_seq('components', 'name') - node.map_attribute_to_seq('conduits', 'sender', 'receiver') + node.map_attribute_to_seq("components", "name") + node.map_attribute_to_seq("conduits", "sender", "receiver") @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - node.index_attribute_to_map('components', 'name') + node.index_attribute_to_map("components", "name") - if len(node.get_attribute('conduits').seq_items()) == 0: - node.remove_attribute('conduits') + if len(node.get_attribute("conduits").seq_items()) == 0: + node.remove_attribute("conduits") - node.seq_attribute_to_map('conduits', 'sender', 'receiver') + node.seq_attribute_to_map("conduits", "sender", "receiver") diff --git a/ymmsl/v0_2/ports.py b/ymmsl/v0_2/ports.py index 1def0d6..5bfa6a0 100644 --- a/ymmsl/v0_2/ports.py +++ b/ymmsl/v0_2/ports.py @@ -36,39 +36,43 @@ class Timeline: Timelines have a technical representation as a list of References, and a string representation in which those References are joined using colons. """ + @overload def __init__(self, timeline: str) -> None: ... @overload def __init__( - self, timeline: Sequence[Union[str, Reference]], absolute: bool = True - ) -> None: ... + self, timeline: Sequence[Union[str, Reference]], absolute: bool = True + ) -> None: ... def __init__( - self, timeline: Union[str, Sequence[Union[str, Reference]]], - absolute: bool = True) -> None: + self, + timeline: Union[str, Sequence[Union[str, Reference]]], + absolute: bool = True, + ) -> None: """Create a Timeline. The argument is either a string describing the timeline, or a list of components that are each a string that is also a valid Reference. """ + def make_new_reference(x: Union[str, Reference]) -> Reference: return Reference(str(x)) if isinstance(timeline, str): - if timeline == '': + if timeline == "": self.absolute = False parts: Sequence[Union[str, Reference]] = [] - elif timeline == ':': + elif timeline == ":": self.absolute = True parts = [] else: self.absolute = False - if timeline[0] == ':': + if timeline[0] == ":": self.absolute = True timeline = timeline[1:] - parts = timeline.split(':') + parts = timeline.split(":") self._parts = list(map(make_new_reference, parts)) @@ -90,12 +94,12 @@ def __hash__(self) -> int: def __str__(self) -> str: """Return the string representation of this Timeline.""" - anchor = ':' if self.absolute else '' - return anchor + ':'.join(map(str, self._parts)) + anchor = ":" if self.absolute else "" + return anchor + ":".join(map(str, self._parts)) def __repr__(self) -> str: """Return a representation of the object.""" - return f'Timeline({self.__str__()})' + return f"Timeline({self.__str__()})" def __len__(self) -> int: """Return the number of parts in the Timeline.""" @@ -105,12 +109,13 @@ def __getitem__(self, index: int) -> Reference: """Return the index'th item in the timeline.""" return self._parts[index] - def __add__(self, other: Any) -> 'Timeline': + def __add__(self, other: Any) -> "Timeline": """Concatenate this timeline with another (relative!) Timeline.""" if isinstance(other, Timeline): if other.absolute: raise ValueError( - 'Cannot concatenate an absolute Timeline onto another one') + "Cannot concatenate an absolute Timeline onto another one" + ) return Timeline(self._parts + other._parts, self.absolute) return NotImplemented @@ -126,9 +131,10 @@ class Port: operator: The MMSL operator in which this port is used timeline: The timeline this port is on, relative to its component. """ + def __init__( - self, name: Identifier, operator: Operator, - timeline: Optional[Timeline] = None) -> None: + self, name: Identifier, operator: Operator, timeline: Optional[Timeline] = None + ) -> None: """Create a Port. Args: @@ -140,15 +146,17 @@ def __init__( self.name = name self.operator = operator if timeline is None: - timeline = Timeline('') + timeline = Timeline("") self.timeline = timeline def __eq__(self, other: Any) -> bool: """Compare Port objects by value.""" if isinstance(other, Port): return ( - self.name == other.name and self.operator == other.operator and - self.timeline == other.timeline) + self.name == other.name + and self.operator == other.operator + and self.timeline == other.timeline + ) return NotImplemented @@ -156,8 +164,7 @@ def __eq__(self, other: Any) -> bool: _PortsSubAttrs = OrderedDict[str, Union[str, List[str]]] -_PortsAttrs = OrderedDict[ - str, Union[str, List[str], _PortsSubAttrs]] +_PortsAttrs = OrderedDict[str, Union[str, List[str], _PortsSubAttrs]] def _ensure_identifier(port_name: Union[str, Identifier]) -> Identifier: @@ -202,21 +209,26 @@ class Ports: On the Python side, this class acts like a dictionary mapping port names to Port objects. """ + @overload def __init__( - self, f_init: Union[None, str, List[str]] = None, - o_i: Union[None, str, List[str]] = None, - s: Union[None, str, List[str]] = None, - o_f: Union[None, str, List[str]] = None) -> None: ... + self, + f_init: Union[None, str, List[str]] = None, + o_i: Union[None, str, List[str]] = None, + s: Union[None, str, List[str]] = None, + o_f: Union[None, str, List[str]] = None, + ) -> None: ... @overload def __init__(self, f_init: List[Port]) -> None: ... def __init__( - self, f_init: Union[None, str, List[str], List[Port]] = None, - o_i: Union[None, str, List[str]] = None, - s: Union[None, str, List[str]] = None, - o_f: Union[None, str, List[str]] = None) -> None: + self, + f_init: Union[None, str, List[str], List[Port]] = None, + o_i: Union[None, str, List[str]] = None, + s: Union[None, str, List[str]] = None, + o_f: Union[None, str, List[str]] = None, + ) -> None: """Create a Ports declaration. This can be called in two ways, either with four sets of ports, or with a single @@ -244,8 +256,9 @@ def __init__( if is_port_list or (is_empty_list and others_none): if not others_none: raise ValueError( - 'Both a list of ports and per-operator ports were given. Please' - ' use either, not both.') + "Both a list of ports and per-operator ports were given. Please" + " use either, not both." + ) self._ports = {p.name: p for p in cast(List[Port], f_init)} else: @@ -278,8 +291,10 @@ def sending_port_names(self) -> List[Identifier]: These are ports associated with O_I or O_F operators. """ return [ - p.name for p in self._ports.values() - if p.operator in (Operator.O_I, Operator.O_F)] + p.name + for p in self._ports.values() + if p.operator in (Operator.O_I, Operator.O_F) + ] def receiving_port_names(self) -> List[Identifier]: """Return the names of all the receiving ports. @@ -287,12 +302,14 @@ def receiving_port_names(self) -> List[Identifier]: These are ports associated with F_INIT or S operators. """ return [ - p.name for p in self._ports.values() - if p.operator in (Operator.F_INIT, Operator.S)] + p.name + for p in self._ports.values() + if p.operator in (Operator.F_INIT, Operator.S) + ] def _add_ports( - self, op: Operator, ports: Union[None, str, List[str]], - timeline: str = '') -> None: + self, op: Operator, ports: Union[None, str, List[str]], timeline: str = "" + ) -> None: """Add the described ports to self._ports, helper function""" if ports is None: name_list = [] @@ -304,26 +321,28 @@ def _add_ports( for name in name_list: if name in self._ports: raise RuntimeError( - f'Invalid ports specification: port "{name}" is' - ' specified more than once. Port names must be unique' - ' within the object they are on.') + f'Invalid ports specification: port "{name}" is' + " specified more than once. Port names must be unique" + " within the object they are on." + ) try: port_id = Identifier(name) except ValueError as e: raise ValueError( - f'Port name "{name}" is not a valid identifier. {e}') from None + f'Port name "{name}" is not a valid identifier. {e}' + ) from None self._ports[port_id] = Port(port_id, op, Timeline(timeline)) def _yatiml_init( - self, - _yatiml_extra: OrderedDict, - f_init: Union[None, str, List[str]] = None, - o_i: Union[None, str, List[str]] = None, - s: Union[None, str, List[str]] = None, - o_f: Union[None, str, List[str]] = None - ) -> None: + self, + _yatiml_extra: OrderedDict, + f_init: Union[None, str, List[str]] = None, + o_i: Union[None, str, List[str]] = None, + s: Union[None, str, List[str]] = None, + o_f: Union[None, str, List[str]] = None, + ) -> None: """Alternative initialisation when loading from YAML.""" self._ports = dict() self._add_ports(Operator.F_INIT, f_init) @@ -332,38 +351,43 @@ def _yatiml_init( self._add_ports(Operator.S, s) for tl_tag, tl_ports in _yatiml_extra.items(): - if not tl_tag.startswith('+'): + if not tl_tag.startswith("+"): raise RuntimeError( - f'Invalid operator "{tl_tag}". Please use f_init, o_i, s, or' - ' o_f, or prepend a + to the timeline name.') + f'Invalid operator "{tl_tag}". Please use f_init, o_i, s, or' + " o_f, or prepend a + to the timeline name." + ) - if 'o_i' in tl_ports: - self._add_ports(Operator.O_I, tl_ports['o_i'], tl_tag[1:]) + if "o_i" in tl_ports: + self._add_ports(Operator.O_I, tl_ports["o_i"], tl_tag[1:]) - if 's' in tl_ports: - self._add_ports(Operator.S, tl_ports['s'], tl_tag[1:]) + if "s" in tl_ports: + self._add_ports(Operator.S, tl_ports["s"], tl_tag[1:]) - additional_keys = set(tl_ports.keys()) - {'o_i', 's'} + additional_keys = set(tl_ports.keys()) - {"o_i", "s"} if additional_keys: raise RuntimeError( - f'Found additional keys {",".join(additional_keys)} under' - f' timeline {tl_tag}. Only o_i and s can be specified here.') + f"Found additional keys {','.join(additional_keys)} under" + f" timeline {tl_tag}. Only o_i and s can be specified here." + ) def _yatiml_attributes(self) -> OrderedDict: def add_ports( - attrs: OrderedDict, op: Operator, key: str, timeline: Timeline) -> None: + attrs: OrderedDict, op: Operator, key: str, timeline: Timeline + ) -> None: op_ports = [ - p for p in self._ports.values() - if p.operator == op and p.timeline == timeline] + p + for p in self._ports.values() + if p.operator == op and p.timeline == timeline + ] names = [str(p.name) for p in op_ports] - if len(names) > 5 or len(' '.join(names)) > 60: + if len(names) > 5 or len(" ".join(names)) > 60: attrs[key] = names elif names: - attrs[key] = ' '.join(names) + attrs[key] = " ".join(names) def unique(timelines: List[Timeline]) -> List[Timeline]: - result = list() # keeps the order - seen = set() # fast lookup + result = list() # keeps the order + seen = set() # fast lookup for timeline in timelines: if timeline not in seen: result.append(timeline) @@ -371,18 +395,18 @@ def unique(timelines: List[Timeline]) -> List[Timeline]: return result attrs: _PortsAttrs = OrderedDict() - add_ports(attrs, Operator.F_INIT, 'f_init', Timeline('')) + add_ports(attrs, Operator.F_INIT, "f_init", Timeline("")) - timelines = [p.timeline for p in self._ports.values() if p.timeline != ''] + timelines = [p.timeline for p in self._ports.values() if p.timeline != ""] if not timelines: - add_ports(attrs, Operator.O_I, 'o_i', Timeline('')) - add_ports(attrs, Operator.S, 's', Timeline('')) + add_ports(attrs, Operator.O_I, "o_i", Timeline("")) + add_ports(attrs, Operator.S, "s", Timeline("")) else: for timeline in unique(timelines): timeline_attrs: _PortsSubAttrs = OrderedDict() - add_ports(timeline_attrs, Operator.O_I, 'o_i', timeline) - add_ports(timeline_attrs, Operator.S, 's', timeline) - attrs[f'+{timeline}'] = timeline_attrs + add_ports(timeline_attrs, Operator.O_I, "o_i", timeline) + add_ports(timeline_attrs, Operator.S, "s", timeline) + attrs[f"+{timeline}"] = timeline_attrs - add_ports(attrs, Operator.O_F, 'o_f', Timeline('')) + add_ports(attrs, Operator.O_F, "o_f", Timeline("")) return attrs diff --git a/ymmsl/v0_2/program.py b/ymmsl/v0_2/program.py index 257237a..33f498c 100644 --- a/ymmsl/v0_2/program.py +++ b/ymmsl/v0_2/program.py @@ -1,4 +1,5 @@ """Definitions for how to start programs.""" + from pathlib import Path from typing import Dict, List, Optional, Union, cast @@ -62,24 +63,24 @@ class Program(Implementation): keeps_state_for_next_use: Does this program keep state for the next iteration of the reuse loop. See :class:`KeepsStateForNextUse`. """ + def __init__( - self, - name: str, - ports: Optional[Ports] = None, - description: str = '', - supported_settings: Optional[SupportedSettings] = None, - base_env: Optional[BaseEnv] = None, - modules: Union[str, List[str], None] = None, - virtual_env: Optional[Path] = None, - env: Optional[Dict[str, str]] = None, - execution_model: ExecutionModel = ExecutionModel.DIRECT, - executable: Optional[Path] = None, - args: Union[str, List[str], None] = None, - script: Union[str, List[str], None] = None, - can_share_resources: bool = True, - keeps_state_for_next_use: KeepsStateForNextUse - = KeepsStateForNextUse.NECESSARY - ) -> None: + self, + name: str, + ports: Optional[Ports] = None, + description: str = "", + supported_settings: Optional[SupportedSettings] = None, + base_env: Optional[BaseEnv] = None, + modules: Union[str, List[str], None] = None, + virtual_env: Optional[Path] = None, + env: Optional[Dict[str, str]] = None, + execution_model: ExecutionModel = ExecutionModel.DIRECT, + executable: Optional[Path] = None, + args: Union[str, List[str], None] = None, + script: Union[str, List[str], None] = None, + can_share_resources: bool = True, + keeps_state_for_next_use: KeepsStateForNextUse = KeepsStateForNextUse.NECESSARY, + ) -> None: """Create a Program description. A Program normally has an ``executable`` and any other needed arguments, with @@ -127,22 +128,26 @@ def __init__( err_arg.append('"args"') if err_arg: raise RuntimeError( - 'When creating a Program, script was specified together with' - f' arguments {", ".join(err_arg)}, which is not supported, as' - ' they are supposed to be inside the script if there is one.' - ' Please use either a script or the arguments listed above.') + "When creating a Program, script was specified together with" + f" arguments {', '.join(err_arg)}, which is not supported, as" + " they are supposed to be inside the script if there is one." + " Please use either a script or the arguments listed above." + ) if ( - executable is None and script is None and - execution_model != ExecutionModel.MANUAL): + executable is None + and script is None + and execution_model != ExecutionModel.MANUAL + ): raise RuntimeError( - f'In {name}, neither a script nor an executable was given. Please' - ' specify either a script, or the other parameters.') + f"In {name}, neither a script nor an executable was given. Please" + " specify either a script, or the other parameters." + ) self.base_env = base_env if base_env else BaseEnv.MANAGER if isinstance(modules, str): - self.modules: Optional[List[str]] = modules.split(' ') + self.modules: Optional[List[str]] = modules.split(" ") else: self.modules = modules @@ -161,7 +166,7 @@ def __init__( self.args = args if isinstance(script, list): - self.script: Optional[str] = '\n'.join(script) + '\n' + self.script: Optional[str] = "\n".join(script) + "\n" else: self.script = script @@ -176,35 +181,36 @@ def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: @classmethod def _yatiml_savorize(cls, node: yatiml.Node) -> None: - if node.has_attribute('env'): - env_node = node.get_attribute('env') + if node.has_attribute("env"): + env_node = node.get_attribute("env") if env_node.is_mapping(): for _, value_node in env_node.yaml_node.value: if isinstance(value_node, yaml.ScalarNode): - if value_node.tag == 'tag:yaml.org,2002:int': - value_node.tag = 'tag:yaml.org,2002:str' - if value_node.tag == 'tag:yaml.org,2002:float': - value_node.tag = 'tag:yaml.org,2002:str' - if value_node.tag == 'tag:yaml.org,2002:bool': - value_node.tag = 'tag:yaml.org,2002:str' + if value_node.tag == "tag:yaml.org,2002:int": + value_node.tag = "tag:yaml.org,2002:str" + if value_node.tag == "tag:yaml.org,2002:float": + value_node.tag = "tag:yaml.org,2002:str" + if value_node.tag == "tag:yaml.org,2002:bool": + value_node.tag = "tag:yaml.org,2002:str" _yatiml_defaults = { - 'base_env': 'manager', - 'execution_model': 'direct', - 'keeps_state_for_next_use': 'necessary'} + "base_env": "manager", + "execution_model": "direct", + "keeps_state_for_next_use": "necessary", + } @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - if node.has_attribute('script'): - script_node = node.get_attribute('script') + if node.has_attribute("script"): + script_node = node.get_attribute("script") if script_node.is_scalar(str): text = cast(str, script_node.get_value()) - if '\n' in text: - cast(yaml.ScalarNode, script_node.yaml_node).style = '|' + if "\n" in text: + cast(yaml.ScalarNode, script_node.yaml_node).style = "|" node.remove_attributes_with_default_values(cls) - if node.has_attribute('env'): - env_attr = node.get_attribute('env') + if node.has_attribute("env"): + env_attr = node.get_attribute("env") if env_attr.is_mapping(): if env_attr.is_empty(): - node.remove_attribute('env') + node.remove_attribute("env") diff --git a/ymmsl/v0_2/resolver.py b/ymmsl/v0_2/resolver.py index fc073a1..475f4ed 100644 --- a/ymmsl/v0_2/resolver.py +++ b/ymmsl/v0_2/resolver.py @@ -37,10 +37,11 @@ class ResolutionContext: generate descriptive error messages. It also contains a pre-parsed version of the ymmsl search path for convenience. """ + def __init__(self) -> None: """Create a ResolutionContext.""" - if 'YMMSL_PATH' in os.environ: - self.ymmsl_path = list(map(Path, os.environ['YMMSL_PATH'].split(':'))) + if "YMMSL_PATH" in os.environ: + self.ymmsl_path = list(map(Path, os.environ["YMMSL_PATH"].split(":"))) else: self.ymmsl_path = list() @@ -76,20 +77,21 @@ def trace(self) -> str: imp_mod = self._imports[i].module imp_name = self._imports[i].name result.append( - f'While importing {imp_name} from {imp_mod} in {file_path}:') + f"While importing {imp_name} from {imp_mod} in {file_path}:" + ) else: - result.append(f'In {file_path}:') + result.append(f"In {file_path}:") - return '\n'.join(result) + '\n' + return "\n".join(result) + "\n" def search_paths(self, rel_path: Path, indent_depth: int) -> str: """Return a description of the search path. Returns a multiline string indented by indent_depth spaces. """ - return '\n'.join([ - ' ' * indent_depth + f'{p / rel_path}' - for p in self.ymmsl_path]) + return "\n".join( + [" " * indent_depth + f"{p / rel_path}" for p in self.ymmsl_path] + ) def resolve(module: Reference, config: Configuration) -> None: @@ -108,13 +110,13 @@ def resolve(module: Reference, config: Configuration) -> None: RuntimeError: if an error occurs due to an invalid configuration. This will leave config in a broken state, so reload it if you want to try again. """ - overwritten = do_resolve(Path('
'), module, config, ResolutionContext()) + overwritten = do_resolve(Path("
"), module, config, ResolutionContext()) remove_overwritten_implementations(config, overwritten) def do_resolve( - file: ModuleSource, module: Reference, config: Configuration, - ctx: ResolutionContext) -> Set[Reference]: + file: ModuleSource, module: Reference, config: Configuration, ctx: ResolutionContext +) -> Set[Reference]: """Implementation of resolve(). This is separate so we don't keep parsing YMMSL_PATH over and over again. When we @@ -126,17 +128,17 @@ def do_resolve( module: The module corresponding to this configuration config: The configuration to resolve """ - _logger.debug(f'Resolving {module}') + _logger.debug(f"Resolving {module}") ctx.push_module(file, module) overwritten_implementations = resolve_impls(module, config, ctx) ctx.pop_module() - _logger.debug(f'Done resolving {module}') + _logger.debug(f"Done resolving {module}") return overwritten_implementations def resolve_impls( - module: Reference, config: Configuration, ctx: ResolutionContext - ) -> Set[Reference]: + module: Reference, config: Configuration, ctx: ResolutionContext +) -> Set[Reference]: """Resolve implementations. This resolves imports of implementations and then applies any local custom @@ -159,12 +161,14 @@ def resolve_impls( return overwritten_impls -T = TypeVar('T', bound='Implementation') +T = TypeVar("T", bound="Implementation") def rename_local_impls( - impls: MutableMapping[Reference, T], module: Reference, - ylocals: Dict[Reference, Reference]) -> None: + impls: MutableMapping[Reference, T], + module: Reference, + ylocals: Dict[Reference, Reference], +) -> None: """Rename local implementations to their full name. Update the given dict of implementations, changing names from p, q and r to a.b.c.p, @@ -183,8 +187,8 @@ def rename_local_impls( def resolve_impl_imports( - config: Configuration, ylocals: Dict[Reference, Reference], - ctx: ResolutionContext) -> None: + config: Configuration, ylocals: Dict[Reference, Reference], ctx: ResolutionContext +) -> None: """Resolve any imports of implementations. This takes all import statements in config that import implementations, loads the @@ -194,10 +198,11 @@ def resolve_impl_imports( """ for imp_st in config.imports: ctx.push_import(imp_st) - _logger.debug(f'Processing import {imp_st.module} implementation {imp_st.name}') + _logger.debug(f"Processing import {imp_st.module} implementation {imp_st.name}") if imp_st.kind == ImportKind.IMPLEMENTATION: imp_cfg, loaded_file = load_resolve_module( - imp_st.module, imp_st.module_path(), ctx) + imp_st.module, imp_st.module_path(), ctx + ) ctx.push_module(loaded_file, imp_st.module) @@ -210,21 +215,24 @@ def resolve_impl_imports( if imp_st.name in ylocals: msg = ctx.trace() - msg += f' Implementation {imp_st.name} is both defined and' - msg += ' imported. Please remove\n' - msg += ' or rename one of the definitions to avoid ambiguity.' + msg += f" Implementation {imp_st.name} is both defined and" + msg += " imported. Please remove\n" + msg += " or rename one of the definitions to avoid ambiguity." raise RuntimeError(msg) ylocals[Reference([imp_st.name])] = imp_st.full_name() - _logger.debug(f'Imported {imp_st.full_name()} as {imp_st.name}') + _logger.debug(f"Imported {imp_st.full_name()} as {imp_st.name}") ctx.pop_module() ctx.pop_import() def apply_custom_implementations( - config: Configuration, module: Reference, ylocals: Dict[Reference, Reference], - ctx: ResolutionContext) -> Set[Reference]: + config: Configuration, + module: Reference, + ylocals: Dict[Reference, Reference], + ctx: ResolutionContext, +) -> Set[Reference]: """Applies custom implementations and removes them. Custom implementations are keyed by the model plus full component path, and map to @@ -237,19 +245,21 @@ def apply_custom_implementations( module: Name of the module it is in ylocals: Map from local to global names """ + def impl_hint_msg( - unknown: Reference, known_impls: Optional[List[str]] = None) -> str: + unknown: Reference, known_impls: Optional[List[str]] = None + ) -> str: if known_impls is None: known_impls = [str(k) for k in ylocals.keys()] matches = get_close_matches(str(unknown), known_impls) if len(matches) == 1: - msg = f'Did you mean {matches[0]}?\n' + msg = f"Did you mean {matches[0]}?\n" elif len(matches) > 1: - msg = 'Did you mean any of these?\n' - msg += indent('- ' + '\n- '.join(matches), 8 * ' ') + msg = "Did you mean any of these?\n" + msg += indent("- " + "\n- ".join(matches), 8 * " ") else: - msg = 'Possible values are:\n' - msg += indent('- ' + '\n- '.join(known_impls), 8 * ' ') + msg = "Possible values are:\n" + msg += indent("- " + "\n- ".join(known_impls), 8 * " ") return msg overwritten_implementations = set() @@ -263,10 +273,13 @@ def set_overwritten(config: Configuration, implementation: Reference) -> None: seen.add(impl) overwritten_implementations.add(impl) if impl in config.models: - queue.extend([ - c.implementation - for c in config.models[impl].components.values() - if c.implementation is not None and c.implementation not in seen]) + queue.extend( + [ + c.implementation + for c in config.models[impl].components.values() + if c.implementation is not None and c.implementation not in seen + ] + ) # Pre-copy any models that will be updated, if they were imported and we therefore # cannot modify them in place without interfering with other uses of the same model. @@ -274,15 +287,17 @@ def set_overwritten(config: Configuration, implementation: Reference) -> None: base_model_name = Reference([key[0]]) if ylocals.get(base_model_name) not in config.models: raise RuntimeError( - ctx.trace() + - f' Unknown model "{base_model_name}" in custom_implementations' - f' "{key}: {value}". {impl_hint_msg(base_model_name)}') + ctx.trace() + + f' Unknown model "{base_model_name}" in custom_implementations' + f' "{key}: {value}". {impl_hint_msg(base_model_name)}' + ) if value is not None and value not in ylocals: raise RuntimeError( - ctx.trace() + - f' Unknown implementation "{value}" in custom_implementations' - f' "{key}: {value}". {impl_hint_msg(value)}') + ctx.trace() + + f' Unknown implementation "{value}" in custom_implementations' + f' "{key}: {value}". {impl_hint_msg(value)}' + ) # Before modifying it, we copy the model from its original a.b.c.Model to # .Model so that the changes don't affect other uses of it, or the @@ -305,7 +320,8 @@ def set_overwritten(config: Configuration, implementation: Reference) -> None: new_impl = ylocals[value] if value is not None else None _logger.debug( - f'Processing custom implementation {base_model_name} {path} {new_impl}') + f"Processing custom implementation {base_model_name} {path} {new_impl}" + ) m = config.models[ylocals[base_model_name]] # Now we can walk down the components and copy-and-rename the models along the @@ -313,65 +329,69 @@ def set_overwritten(config: Configuration, implementation: Reference) -> None: for i, c in enumerate(path[:-1]): if not isinstance(c, Identifier): raise RuntimeError( - ctx.trace() + - f' Invalid path in custom_implementations "{key}". Only' - ' component names are allowed here.') + ctx.trace() + + f' Invalid path in custom_implementations "{key}". Only' + " component names are allowed here." + ) component = Reference([c]) if component not in m.components: raise RuntimeError( - ctx.trace() + - f' In custom_implementations {key}: {value}:' - f' {base_model_name + path[:i]} is implemented by {m.name},' - f' which does not have a component named {str(component)}.') + ctx.trace() + f" In custom_implementations {key}: {value}:" + f" {base_model_name + path[:i]} is implemented by {m.name}," + f" which does not have a component named {str(component)}." + ) orig_impl = m.components[component].implementation if orig_impl in ylocals: orig_impl = ylocals[orig_impl] if orig_impl is None or orig_impl not in config.models: raise RuntimeError( - ctx.trace() + - f' In custom_implementations "{key}: {value}":' - f' {base_model_name + path[:i+1]} has implementation' - f' {orig_impl}, which does not have a component named' - f' {path[i+1]} in it.') - - if (base_model_name + path[:i+1]) not in copied_paths: - new_name = module + Identifier(( - str(orig_impl) + '__customised_for__' + - str(ylocals[base_model_name] + path[:i+1]) - ).replace('.', '_')) + ctx.trace() + f' In custom_implementations "{key}: {value}":' + f" {base_model_name + path[: i + 1]} has implementation" + f" {orig_impl}, which does not have a component named" + f" {path[i + 1]} in it." + ) + + if (base_model_name + path[: i + 1]) not in copied_paths: + new_name = module + Identifier( + ( + str(orig_impl) + + "__customised_for__" + + str(ylocals[base_model_name] + path[: i + 1]) + ).replace(".", "_") + ) new_submodel = copy(config.models[orig_impl]) new_submodel.components = copy(new_submodel.components) new_submodel.name = new_name config.models[new_name] = new_submodel m.components[component] = copy(m.components[component]) m.components[component].implementation = new_name - _logger.debug(f'Set {new_name} as impl of {m.name} {component}') + _logger.debug(f"Set {new_name} as impl of {m.name} {component}") set_overwritten(config, orig_impl) m = new_submodel - copied_paths.add(base_model_name + path[:i+1]) + copied_paths.add(base_model_name + path[: i + 1]) else: m = config.models[orig_impl] if not isinstance(path[-1], Identifier): raise RuntimeError( - ctx.trace() + - f' In custom_implementations "{key}: {value}":' - f' [i] instance selectors are invalid in custom_implementations.') + ctx.trace() + f' In custom_implementations "{key}: {value}":' + f" [i] instance selectors are invalid in custom_implementations." + ) if path[-1] not in m.components: raise RuntimeError( - ctx.trace() + - f' In custom_implementations "{key}: {value}":' - f' {base_model_name + path[:-1]} is implemented by {m.name}, which' - f' does not have a component named {path[-1]}. ' + - impl_hint_msg( - Reference([path[-1]]), [str(k) for k in m.components.keys()]) - ) + ctx.trace() + f' In custom_implementations "{key}: {value}":' + f" {base_model_name + path[:-1]} is implemented by {m.name}, which" + f" does not have a component named {path[-1]}. " + + impl_hint_msg( + Reference([path[-1]]), [str(k) for k in m.components.keys()] + ) + ) component = Reference([path[-1]]) new_component = copy(m.components[component]) - _logger.debug(f'In {m.name} replacing {component} with {new_impl}') + _logger.debug(f"In {m.name} replacing {component} with {new_impl}") if new_component.implementation is not None: if new_component.implementation in config.models: set_overwritten(config, new_component.implementation) @@ -384,11 +404,12 @@ def set_overwritten(config: Configuration, implementation: Reference) -> None: def update_local_implementations( - config: Configuration, ylocals: Dict[Reference, Reference]) -> None: + config: Configuration, ylocals: Dict[Reference, Reference] +) -> None: """Updates names of local implementations to their full names.""" - _logger.debug('Updating local implementations') + _logger.debug("Updating local implementations") for model in config.models.values(): - _logger.debug(f'Updating impls in model {model.name}') + _logger.debug(f"Updating impls in model {model.name}") for cmp in model.components.values(): if cmp.implementation: if cmp.implementation in ylocals: @@ -397,7 +418,8 @@ def update_local_implementations( def remove_overwritten_implementations( - config: Configuration, overwritten: Set[Reference]) -> None: + config: Configuration, overwritten: Set[Reference] +) -> None: """Remove implementations that are no longer needed. This can happen when a custom_implementation overwrites the implementation of a @@ -410,8 +432,8 @@ def remove_overwritten_implementations( reference to a different implementation. """ referred_to = { - c.implementation for m in config.models.values() - for c in m.components.values()} + c.implementation for m in config.models.values() for c in m.components.values() + } roots = set(config.models.keys()) - referred_to - overwritten used = set() @@ -422,13 +444,15 @@ def remove_overwritten_implementations( seen.add(cur_impl) used.add(cur_impl) if cur_impl in config.models: - queue.extend([ - c.implementation - for c in config.models[cur_impl].components.values() - if c.implementation is not None + queue.extend( + [ + c.implementation + for c in config.models[cur_impl].components.values() + if c.implementation is not None and c.implementation not in seen and c.implementation in config.models - ]) + ] + ) for m in list(config.models.keys()): if m in overwritten and m not in used: @@ -436,8 +460,8 @@ def remove_overwritten_implementations( def find_impls( - config: Configuration, name: Reference, ctx: ResolutionContext - ) -> List[Implementation]: + config: Configuration, name: Reference, ctx: ResolutionContext +) -> List[Implementation]: """Find and return an implementation and its dependencies. This searches config for the implementation with the given name, and returns it and @@ -459,8 +483,8 @@ def find_impls( def find_impl( - config: Configuration, name: Reference, ctx: ResolutionContext - ) -> Implementation: + config: Configuration, name: Reference, ctx: ResolutionContext +) -> Implementation: """Find a model or program in a configuration.""" if name in config.programs: return config.programs[name] @@ -468,20 +492,21 @@ def find_impl( return config.models[name] else: impls = [ - str(k[-1]) - for k in list(config.models.keys()) + list(config.programs.keys())] + str(k[-1]) + for k in list(config.models.keys()) + list(config.programs.keys()) + ] matches = get_close_matches(str(name[-1]), impls) msg = ctx.trace() - msg += f' Implementation {name[-1]} not found.' + msg += f" Implementation {name[-1]} not found." if matches: if len(matches) == 1: - msg += f' Did you mean {matches[0]}?\n' + msg += f" Did you mean {matches[0]}?\n" else: - msg += ' Did you mean any of these?\n' - msg += indent('- ' + '\n- '.join(matches), 8 * ' ') + msg += " Did you mean any of these?\n" + msg += indent("- " + "\n- ".join(matches), 8 * " ") else: - msg += ' Available implementations in this module:\n' - msg += indent('- ' + '\n- '.join(impls), 8 * ' ') + msg += " Available implementations in this module:\n" + msg += indent("- " + "\n- ".join(impls), 8 * " ") raise RuntimeError(msg) @@ -490,7 +515,8 @@ def find_impl( def _load_from_entrypoints( - module: Reference) -> Optional[Tuple[Configuration, EntryPoint]]: + module: Reference, +) -> Optional[Tuple[Configuration, EntryPoint]]: # Find entry point entrypoints = entry_points(group="ymmsl.module", name=str(module)) if not entrypoints: @@ -503,7 +529,7 @@ def _load_from_entrypoints( ", ".join( f"'{ep.value}' (from {ep.dist.name if ep.dist else ''})" for ep in entrypoints - ) + ), ) entrypoint = next(iter(entrypoints)) @@ -523,8 +549,8 @@ def _load_from_entrypoints( def _load_from_ymmsl_path( - module_path: Path, ymmsl_path: list[Path] - ) -> Optional[Tuple[Configuration, Path]]: + module_path: Path, ymmsl_path: list[Path] +) -> Optional[Tuple[Configuration, Path]]: for yp in ymmsl_path: try: loaded_file = yp / module_path @@ -536,8 +562,8 @@ def _load_from_ymmsl_path( def load_resolve_module( - module: Reference, module_path: Path, ctx: ResolutionContext - ) -> Tuple[Configuration, ModuleSource]: + module: Reference, module_path: Path, ctx: ResolutionContext +) -> Tuple[Configuration, ModuleSource]: """Load and resolve an imported ymmsl file. Caches the result, without functools.lru_cache because ctx shouldn't hash. @@ -551,19 +577,19 @@ def load_resolve_module( if module_path not in ymmsl_cache: # TODO: relative to working directory? try: - config_and_loaded_file = ( - _load_from_entrypoints(module) - or _load_from_ymmsl_path(module_path, ctx.ymmsl_path) - ) + config_and_loaded_file = _load_from_entrypoints( + module + ) or _load_from_ymmsl_path(module_path, ctx.ymmsl_path) if config_and_loaded_file is None: msg = ctx.trace() - msg += f' Failed to find a file {module_path} for module {module}.\n' - msg += ' Based on the YMMSL_PATH environment variable and Python' - msg += ' entry points. I\'ve searched at:\n' + msg += f" Failed to find a file {module_path} for module {module}.\n" + msg += " Based on the YMMSL_PATH environment variable and Python" + msg += " entry points. I've searched at:\n" msg += ctx.search_paths(module_path, 8) - msg += '\n and in entry points:\n' - msg += '\n'.join( - 8*' ' + ep.name for ep in entry_points(group="ymmsl.module")) + msg += "\n and in entry points:\n" + msg += "\n".join( + 8 * " " + ep.name for ep in entry_points(group="ymmsl.module") + ) raise RuntimeError(msg) config, loaded_file = config_and_loaded_file @@ -572,7 +598,7 @@ def load_resolve_module( except RecognitionError as e: msg = ctx.trace() - msg += indent(str(e), ' ' * 4) + msg += indent(str(e), " " * 4) raise RuntimeError(msg) from None return ymmsl_cache[module_path] diff --git a/ymmsl/v0_2/resources.py b/ymmsl/v0_2/resources.py index 88522b9..026dab8 100644 --- a/ymmsl/v0_2/resources.py +++ b/ymmsl/v0_2/resources.py @@ -1,6 +1,6 @@ from ymmsl.v0_1.execution import ( # noqa: F401 - MPICoresResReq, - MPINodesResReq, - ResourceRequirements, - ThreadedResReq, + MPICoresResReq, + MPINodesResReq, + ResourceRequirements, + ThreadedResReq, ) diff --git a/ymmsl/v0_2/supported_settings.py b/ymmsl/v0_2/supported_settings.py index 9fdfabc..01cf7cd 100644 --- a/ymmsl/v0_2/supported_settings.py +++ b/ymmsl/v0_2/supported_settings.py @@ -10,13 +10,13 @@ class SettingType(Enum): - STR = 'str' - INT = 'int' - FLOAT = 'float' - BOOL = 'bool' - LIST_INT = '[int]' - LIST_FLOAT = '[float]' - LIST_LIST_FLOAT = '[[float]]' + STR = "str" + INT = "int" + FLOAT = "float" + BOOL = "bool" + LIST_INT = "[int]" + LIST_FLOAT = "[float]" + LIST_LIST_FLOAT = "[[float]]" def __str__(self) -> str: return self.value @@ -46,9 +46,9 @@ def to_str(node: yatiml.Node) -> str: elif node.is_sequence(): items = node.seq_items() if len(items) != 1: - raise yatiml.SeasoningError('Invalid setting type') - return 'LIST_' + to_str(items[0]) - raise yatiml.SeasoningError('Invalid setting type') + raise yatiml.SeasoningError("Invalid setting type") + return "LIST_" + to_str(items[0]) + raise yatiml.SeasoningError("Invalid setting type") node.set_value(to_str(node)) @@ -63,19 +63,22 @@ def to_node(typ: str, start_mark: yaml.Mark, end_mark: yaml.Mark) -> yaml.Node: rather than a multiline sequence with dashes, and finally inserts the lowercased remaining part. """ - if not typ.startswith('LIST_'): + if not typ.startswith("LIST_"): return yaml.ScalarNode( - 'tag:yaml.org,2002:str', typ.lower(), start_mark, end_mark) + "tag:yaml.org,2002:str", typ.lower(), start_mark, end_mark + ) else: subnode = to_node(typ[5:], start_mark, end_mark) seq = yaml.SequenceNode( - 'tag:yaml.org,2002:seq', [subnode], start_mark, end_mark) + "tag:yaml.org,2002:seq", [subnode], start_mark, end_mark + ) seq.flow_style = True return seq val = cast(str, node.get_value()) node.yaml_node = to_node( - val, node.yaml_node.start_mark, node.yaml_node.end_mark) + val, node.yaml_node.start_mark, node.yaml_node.end_mark + ) class SupportedSetting: @@ -87,9 +90,13 @@ class SupportedSetting: typ: Type of the setting description: Description of what this sets, allowed values, etc. """ + def __init__( - self, name: Union[str, Identifier], typ: Union[str, SettingType], - description: str) -> None: + self, + name: Union[str, Identifier], + typ: Union[str, SettingType], + description: str, + ) -> None: """Create a SupportedSetting. Args: @@ -112,8 +119,10 @@ def __eq__(self, other: Any) -> bool: if not isinstance(other, SupportedSetting): return NotImplemented return ( - self.name == other.name and self.typ == other.typ and - self.description == other.description) + self.name == other.name + and self.typ == other.typ + and self.description == other.description + ) @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: @@ -130,13 +139,15 @@ def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: def _yatiml_savorize(cls, node: yatiml.Node) -> None: """Rename to avoid the Python keyword""" if node.is_mapping(): - if node.has_attribute('type'): - node.rename_attribute('type', 'typ') + if node.has_attribute("type"): + node.rename_attribute("type", "typ") def _yatiml_init( - self, name: Identifier, typ: Optional[SettingType] = None, - description: Optional[Union[str, List[str], List[List[str]]]] = None - ) -> None: + self, + name: Identifier, + typ: Optional[SettingType] = None, + description: Optional[Union[str, List[str], List[List[str]]]] = None, + ) -> None: """ Yeah, sorry. I wanted nice syntax for the users and couldn't find a cleaner way. @@ -177,45 +188,51 @@ def _yatiml_init( bracket makes this a sequence, and those aren't allowed to have anything after the corresponding closing square bracket. """ + def list_to_setting_type( - description: Union[List[str], List[List[str]]]) -> SettingType: + description: Union[List[str], List[List[str]]], + ) -> SettingType: """convert ['something'] or [['something']] to a SettingType""" if len(description) == 0: raise ValueError(f'Invalid setting type [] for "{name}"') if isinstance(description[0], str): - return SettingType(f'[{description[0]}]') + return SettingType(f"[{description[0]}]") else: if len(description[0]) == 0: raise ValueError(f'Invalid setting type [[]] for "{name}"') - return SettingType(f'[[{description[0][0]}]]') + return SettingType(f"[[{description[0][0]}]]") if isinstance(description, list): # description contains only a type, a funny one if typ is not None: raise ValueError( - f'The description of "{name}" gives a type, rather than a' - ' description') + f'The description of "{name}" gives a type, rather than a' + " description" + ) typ = list_to_setting_type(description) - description = '' + description = "" elif isinstance(description, str): if typ is None: # description must contain a type, and perhaps a description if ( - description is None or - isinstance(description, str) and description.strip() == ''): + description is None + or isinstance(description, str) + and description.strip() == "" + ): raise ValueError( - f'Neither a type nor a description was given for "{name}"') + f'Neither a type nor a description was given for "{name}"' + ) pieces = description.split(maxsplit=1) try: typ = SettingType(pieces[0]) - description = pieces[1] if len(pieces) > 1 else '' + description = pieces[1] if len(pieces) > 1 else "" except KeyError as exc: raise ValueError( - 'If type is not given, then description must start with' + "If type is not given, then description must start with" f' the setting\'s type, which is not the case for "{name}"' ) from exc # else typ is the type and description the description, so nothing to do @@ -223,44 +240,38 @@ def list_to_setting_type( # description is None, which is okay as long as we have a type if typ is None: raise ValueError( - f'Neither a type nor a description was given for "{name}"') - description = '' + f'Neither a type nor a description was given for "{name}"' + ) + description = "" SupportedSetting.__init__(self, name, typ, description) def _yatiml_attributes(self) -> Dict: """Create output data for YAML serialisation.""" - if self.description.strip() == '': + if self.description.strip() == "": # Put type into the description field so that # SupportedSettings._yatiml_sweeten maps this to name: type - return { - 'name': self.name, - 'description': self.typ} + return {"name": self.name, "description": self.typ} # If the description is short enough, put it on a single line too short = len(self.name) + len(self.description) < 72 - if short and '\n' not in self.description: + if short and "\n" not in self.description: desc = self.typ.value if self.description: - desc += f' {self.description}' - return { - 'name': self.name, - 'description': desc} + desc += f" {self.description}" + return {"name": self.name, "description": desc} # For long or multiline descriptions, use type: and description: subkeys - return { - 'name': self.name, - 'type': self.typ, - 'description': self.description} + return {"name": self.name, "type": self.typ, "description": self.description} @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: """If the description is multiple lines, format it block style""" - desc_node = node.get_attribute('description') + desc_node = node.get_attribute("description") if desc_node.is_scalar(str): ynode = cast(yaml.ScalarNode, desc_node.yaml_node) - if '\n' in cast(str, desc_node.get_value()): - ynode.style = '|' + if "\n" in cast(str, desc_node.get_value()): + ynode.style = "|" # ensure PyYAML actually uses block style ynode.value = remove_trailing_whitespace(ynode.value) @@ -298,11 +309,13 @@ class SupportedSettings(MutableMapping): description of D needs to be quoted to avoid it being invalid YAML. If there is only the type, then the quotes can be omitted. """ + def __init__( - self, - supported_settings: Union[ - Mapping[str, str], List[SupportedSetting], None] = None - ) -> None: + self, + supported_settings: Union[ + Mapping[str, str], List[SupportedSetting], None + ] = None, + ) -> None: """Create a SupportedSettings object. If given, the argument must be a dictionary mapping strings to strings as in the @@ -335,7 +348,7 @@ def __eq__(self, other: Any) -> bool: def __str__(self) -> str: """Represent as a string, omitting the descriptions.""" - return ', '.join([f'{s.name}: {s.typ}' for s in self._store.values()]) + return ", ".join([f"{s.name}: {s.typ}" for s in self._store.values()]) def __getitem__(self, key: Union[str, Identifier]) -> SupportedSetting: """Returns a supported setting, implements supported_settings[name].""" @@ -344,8 +357,8 @@ def __getitem__(self, key: Union[str, Identifier]) -> SupportedSetting: return self._store[key] def __setitem__( - self, key: Union[str, Identifier], value: Union[str, SupportedSetting] - ) -> None: + self, key: Union[str, Identifier], value: Union[str, SupportedSetting] + ) -> None: """Sets a value, implements supported_settings[name] = typ, desc.""" if isinstance(key, str): key = Identifier(key) @@ -367,7 +380,7 @@ def __len__(self) -> int: """Returns the number of settings.""" return len(self._store) - def copy(self) -> 'SupportedSettings': + def copy(self) -> "SupportedSettings": """Makes a shallow copy of these supported settings and returns it.""" new_settings = SupportedSettings() new_settings._store = self._store.copy() @@ -377,10 +390,11 @@ def _to_supported_setting(self, name: Identifier, arg: str) -> SupportedSetting: """Parse a string into a SupportedSetting.""" pieces = arg.split(maxsplit=1) if len(pieces) == 0: - raise RuntimeError(f'Empty description for setting {name}') + raise RuntimeError(f"Empty description for setting {name}") return SupportedSetting( - name, SettingType(pieces[0]), '' if len(pieces) == 1 else pieces[1]) + name, SettingType(pieces[0]), "" if len(pieces) == 1 else pieces[1] + ) @classmethod def _yatiml_recognize(cls, node: yatiml.UnknownNode) -> None: @@ -391,25 +405,24 @@ def _yatiml_savorize(cls, node: yatiml.Node) -> None: # Nest the input inside of a new mapping with a supported_settings key sup_set = node.yaml_node node.make_mapping() - node.set_attribute('supported_settings', sup_set) + node.set_attribute("supported_settings", sup_set) # And then convert it to a list of SupportedSetting mappings # SupportedSettings can take a type for description, see its _yatiml_init - node.map_attribute_to_seq('supported_settings', 'name', 'description') + node.map_attribute_to_seq("supported_settings", "name", "description") def _yatiml_init( - self, supported_settings: Optional[List[SupportedSetting]] = None - ) -> None: + self, supported_settings: Optional[List[SupportedSetting]] = None + ) -> None: # Take that list of supported settings and initialise the object SupportedSettings.__init__(self, supported_settings) def _yatiml_attributes(self) -> Dict: # Put everything into a mapping under the supported_settings key, # so that we can use seq_attribute_to_map below. - return { - 'supported_settings': list(self._store.values())} + return {"supported_settings": list(self._store.values())} @classmethod def _yatiml_sweeten(cls, node: yatiml.Node) -> None: - node.seq_attribute_to_map('supported_settings', 'name', 'description') + node.seq_attribute_to_map("supported_settings", "name", "description") # use the created map directly again node.yaml_node.value = node.yaml_node.value[0][1].value diff --git a/ymmsl/v0_2/tests/conftest.py b/ymmsl/v0_2/tests/conftest.py index 975f020..d728e86 100644 --- a/ymmsl/v0_2/tests/conftest.py +++ b/ymmsl/v0_2/tests/conftest.py @@ -29,720 +29,895 @@ @pytest.fixture def model() -> Model: return Model( - 'test_model', - Ports('in', o_f='out'), - 'Test model for loading/dumping', - SupportedSettings({'eta': 'float'}), - [ - Component( - 'ic', Ports(o_f='out'), 'Creates initial state', - 'initial_conditions'), - Component( - 'smc', Ports('initial_state', 'cell_positions', 'wss_in'), - 'Simulates smooth muscle cells', 'smc'), - Component( - 'bf', Ports('initial_domain', o_f='wss_out'), - 'Simulates blood flow', 'blood_flow'), - Component( - 'smc2bf', Ports('in', o_f='out'), 'Grids domain', 'smc2bf'), - Component( - 'bf2smc', Ports('in', o_f='out'), 'Interpolates wss', 'bf2smc')], - [ - Conduit('ic.out', 'smc.initial_state'), - Conduit('smc.cell_positions', 'smc2bf.in'), - Conduit('smc2bf.out', 'bf.initial_domain'), - Conduit('bf.wss_out', 'bf2smc.in'), - Conduit('bf2smc.out', 'smc.wss_in')]) + "test_model", + Ports("in", o_f="out"), + "Test model for loading/dumping", + SupportedSettings({"eta": "float"}), + [ + Component( + "ic", Ports(o_f="out"), "Creates initial state", "initial_conditions" + ), + Component( + "smc", + Ports("initial_state", "cell_positions", "wss_in"), + "Simulates smooth muscle cells", + "smc", + ), + Component( + "bf", + Ports("initial_domain", o_f="wss_out"), + "Simulates blood flow", + "blood_flow", + ), + Component("smc2bf", Ports("in", o_f="out"), "Grids domain", "smc2bf"), + Component("bf2smc", Ports("in", o_f="out"), "Interpolates wss", "bf2smc"), + ], + [ + Conduit("ic.out", "smc.initial_state"), + Conduit("smc.cell_positions", "smc2bf.in"), + Conduit("smc2bf.out", "bf.initial_domain"), + Conduit("bf.wss_out", "bf2smc.in"), + Conduit("bf2smc.out", "smc.wss_in"), + ], + ) @pytest.fixture def model_text() -> str: return ( - 'name: test_model\n' - 'ports:\n' - ' f_init: in\n' - ' o_f: out\n' - 'description: |\n' - ' Test model for loading/dumping\n' - 'supported_settings:\n' - ' eta: float\n' - 'components:\n' - ' ic:\n' - ' ports:\n' - ' o_f: out\n' - ' description: |\n' - ' Creates initial state\n' - ' implementation: initial_conditions\n' - ' smc:\n' - ' ports:\n' - ' f_init: initial_state\n' - ' o_i: cell_positions\n' - ' s: wss_in\n' - ' description: |\n' - ' Simulates smooth muscle cells\n' - ' implementation: smc\n' - ' bf:\n' - ' ports:\n' - ' f_init: initial_domain\n' - ' o_f: wss_out\n' - ' description: |\n' - ' Simulates blood flow\n' - ' implementation: blood_flow\n' - ' smc2bf:\n' - ' ports:\n' - ' f_init: in\n' - ' o_f: out\n' - ' description: |\n' - ' Grids domain\n' - ' implementation: smc2bf\n' - ' bf2smc:\n' - ' ports:\n' - ' f_init: in\n' - ' o_f: out\n' - ' description: |\n' - ' Interpolates wss\n' - ' implementation: bf2smc\n' - 'conduits:\n' - ' ic.out: smc.initial_state\n' - ' smc.cell_positions: smc2bf.in\n' - ' smc2bf.out: bf.initial_domain\n' - ' bf.wss_out: bf2smc.in\n' - ' bf2smc.out: smc.wss_in\n' - ) + "name: test_model\n" + "ports:\n" + " f_init: in\n" + " o_f: out\n" + "description: |\n" + " Test model for loading/dumping\n" + "supported_settings:\n" + " eta: float\n" + "components:\n" + " ic:\n" + " ports:\n" + " o_f: out\n" + " description: |\n" + " Creates initial state\n" + " implementation: initial_conditions\n" + " smc:\n" + " ports:\n" + " f_init: initial_state\n" + " o_i: cell_positions\n" + " s: wss_in\n" + " description: |\n" + " Simulates smooth muscle cells\n" + " implementation: smc\n" + " bf:\n" + " ports:\n" + " f_init: initial_domain\n" + " o_f: wss_out\n" + " description: |\n" + " Simulates blood flow\n" + " implementation: blood_flow\n" + " smc2bf:\n" + " ports:\n" + " f_init: in\n" + " o_f: out\n" + " description: |\n" + " Grids domain\n" + " implementation: smc2bf\n" + " bf2smc:\n" + " ports:\n" + " f_init: in\n" + " o_f: out\n" + " description: |\n" + " Interpolates wss\n" + " implementation: bf2smc\n" + "conduits:\n" + " ic.out: smc.initial_state\n" + " smc.cell_positions: smc2bf.in\n" + " smc2bf.out: bf.initial_domain\n" + " bf.wss_out: bf2smc.in\n" + " bf2smc.out: smc.wss_in\n" + ) @pytest.fixture def model_multicast() -> Model: return Model( - 'test_model', None, 'Test model for multicast conduits', None, - [ - Component('a', Ports(o_f='out'), 'Creates data', 'a'), - Component('b', Ports('in'), 'Receives data', 'b'), - Component('c', Ports('in'), 'Receives data', 'b'), - ], - [ - Conduit('a.out', 'b.in'), - Conduit('a.out', 'c.in'), - ] - ) + "test_model", + None, + "Test model for multicast conduits", + None, + [ + Component("a", Ports(o_f="out"), "Creates data", "a"), + Component("b", Ports("in"), "Receives data", "b"), + Component("c", Ports("in"), "Receives data", "b"), + ], + [ + Conduit("a.out", "b.in"), + Conduit("a.out", "c.in"), + ], + ) @pytest.fixture def model_multicast_text() -> str: return ( - 'name: test_model\n' - 'description: |\n' - ' Test model for multicast conduits\n' - 'components:\n' - ' a:\n' - ' ports:\n' - ' o_f: out\n' - ' description: |\n' - ' Creates data\n' - ' implementation: a\n' - ' b:\n' - ' ports:\n' - ' f_init: in\n' - ' description: |\n' - ' Receives data\n' - ' implementation: b\n' - ' c:\n' - ' ports:\n' - ' f_init: in\n' - ' description: |\n' - ' Receives data\n' - ' implementation: b\n' - 'conduits:\n' - ' a.out:\n' - ' - b.in\n' - ' - c.in\n' - ) + "name: test_model\n" + "description: |\n" + " Test model for multicast conduits\n" + "components:\n" + " a:\n" + " ports:\n" + " o_f: out\n" + " description: |\n" + " Creates data\n" + " implementation: a\n" + " b:\n" + " ports:\n" + " f_init: in\n" + " description: |\n" + " Receives data\n" + " implementation: b\n" + " c:\n" + " ports:\n" + " f_init: in\n" + " description: |\n" + " Receives data\n" + " implementation: b\n" + "conduits:\n" + " a.out:\n" + " - b.in\n" + " - c.in\n" + ) @pytest.fixture def model_with_filters() -> Model: return Model( - 'test_model_conduit_filters', - Ports(), - 'Test model for loading/dumping conduit filters', - SupportedSettings(), - [ - Component( - 'init', Ports(o_f='macro_out micro_out'), 'Creates initial states', - 'init'), - Component( - 'macro1', Ports('init', 'bc_out', 'bc_in', 'final'), - 'First macro model', 'macro1'), - Component( - 'micro1', Ports('init_state init_bc', o_f='final_bc final_state'), - 'First micro model', 'micro1'), - Component( - 'macro2', Ports('init_state', 'bc_out', 'bc_in'), - 'Second macro model', 'macro2'), - Component( - 'micro2', Ports('init_state init_bc', o_f='final_bc'), - 'Second micro model', 'micro2'), - ], - [ - Conduit('init.macro_out', 'macro1.init'), - Conduit('init.micro_out', 'micro1.init_state', 'pad'), - Conduit('macro1.bc_out', 'micro1.init_bc'), - Conduit('micro1.final_bc', 'macro1.bc_in'), - Conduit('macro1.final', 'macro2.init_state'), - Conduit('micro1.final_state', 'micro2.init_state', 'last pad'), - Conduit('macro2.bc_out', 'micro2.init_bc'), - Conduit('micro2.final_bc', 'macro2.bc_in'), - ]) + "test_model_conduit_filters", + Ports(), + "Test model for loading/dumping conduit filters", + SupportedSettings(), + [ + Component( + "init", + Ports(o_f="macro_out micro_out"), + "Creates initial states", + "init", + ), + Component( + "macro1", + Ports("init", "bc_out", "bc_in", "final"), + "First macro model", + "macro1", + ), + Component( + "micro1", + Ports("init_state init_bc", o_f="final_bc final_state"), + "First micro model", + "micro1", + ), + Component( + "macro2", + Ports("init_state", "bc_out", "bc_in"), + "Second macro model", + "macro2", + ), + Component( + "micro2", + Ports("init_state init_bc", o_f="final_bc"), + "Second micro model", + "micro2", + ), + ], + [ + Conduit("init.macro_out", "macro1.init"), + Conduit("init.micro_out", "micro1.init_state", "pad"), + Conduit("macro1.bc_out", "micro1.init_bc"), + Conduit("micro1.final_bc", "macro1.bc_in"), + Conduit("macro1.final", "macro2.init_state"), + Conduit("micro1.final_state", "micro2.init_state", "last pad"), + Conduit("macro2.bc_out", "micro2.init_bc"), + Conduit("micro2.final_bc", "macro2.bc_in"), + ], + ) @pytest.fixture def model_with_filters_text() -> str: return ( - 'name: test_model_conduit_filters\n' - 'description: |\n' - ' Test model for loading/dumping conduit filters\n' - 'components:\n' - ' init:\n' - ' ports:\n' - ' o_f: macro_out micro_out\n' - ' description: |\n' - ' Creates initial states\n' - ' implementation: init\n' - ' macro1:\n' - ' ports:\n' - ' f_init: init\n' - ' o_i: bc_out\n' - ' s: bc_in\n' - ' o_f: final\n' - ' description: |\n' - ' First macro model\n' - ' implementation: macro1\n' - ' micro1:\n' - ' ports:\n' - ' f_init: init_state init_bc\n' - ' o_f: final_bc final_state\n' - ' description: |\n' - ' First micro model\n' - ' implementation: micro1\n' - ' macro2:\n' - ' ports:\n' - ' f_init: init_state\n' - ' o_i: bc_out\n' - ' s: bc_in\n' - ' description: |\n' - ' Second macro model\n' - ' implementation: macro2\n' - ' micro2:\n' - ' ports:\n' - ' f_init: init_state init_bc\n' - ' o_f: final_bc\n' - ' description: |\n' - ' Second micro model\n' - ' implementation: micro2\n' - 'conduits:\n' - ' init.macro_out: macro1.init\n' - ' init.micro_out: pad micro1.init_state\n' - ' macro1.bc_out: micro1.init_bc\n' - ' micro1.final_bc: macro1.bc_in\n' - ' macro1.final: macro2.init_state\n' - ' micro1.final_state: last pad micro2.init_state\n' - ' macro2.bc_out: micro2.init_bc\n' - ' micro2.final_bc: macro2.bc_in\n' - ) + "name: test_model_conduit_filters\n" + "description: |\n" + " Test model for loading/dumping conduit filters\n" + "components:\n" + " init:\n" + " ports:\n" + " o_f: macro_out micro_out\n" + " description: |\n" + " Creates initial states\n" + " implementation: init\n" + " macro1:\n" + " ports:\n" + " f_init: init\n" + " o_i: bc_out\n" + " s: bc_in\n" + " o_f: final\n" + " description: |\n" + " First macro model\n" + " implementation: macro1\n" + " micro1:\n" + " ports:\n" + " f_init: init_state init_bc\n" + " o_f: final_bc final_state\n" + " description: |\n" + " First micro model\n" + " implementation: micro1\n" + " macro2:\n" + " ports:\n" + " f_init: init_state\n" + " o_i: bc_out\n" + " s: bc_in\n" + " description: |\n" + " Second macro model\n" + " implementation: macro2\n" + " micro2:\n" + " ports:\n" + " f_init: init_state init_bc\n" + " o_f: final_bc\n" + " description: |\n" + " Second micro model\n" + " implementation: micro2\n" + "conduits:\n" + " init.macro_out: macro1.init\n" + " init.micro_out: pad micro1.init_state\n" + " macro1.bc_out: micro1.init_bc\n" + " micro1.final_bc: macro1.bc_in\n" + " macro1.final: macro2.init_state\n" + " micro1.final_state: last pad micro2.init_state\n" + " macro2.bc_out: micro2.init_bc\n" + " micro2.final_bc: macro2.bc_in\n" + ) @pytest.fixture def test_program() -> Program: return Program( - 'macro', Ports(['init'], ['out1', 'out2'], ['in1', 'in2'], ['final']), - 'description', SupportedSettings({'alpha': 'float'}), BaseEnv.LOGIN, - ['gcc/13.3.0', 'FFTW/3.2.1'], Path('/home/user/.venv'), - {'SETTING': 'something', 'VARIABLE': '42'}, ExecutionModel.INTELMPI, - Path('python3'), ['/home/user/script.py'], None, False, - KeepsStateForNextUse.HELPFUL) + "macro", + Ports(["init"], ["out1", "out2"], ["in1", "in2"], ["final"]), + "description", + SupportedSettings({"alpha": "float"}), + BaseEnv.LOGIN, + ["gcc/13.3.0", "FFTW/3.2.1"], + Path("/home/user/.venv"), + {"SETTING": "something", "VARIABLE": "42"}, + ExecutionModel.INTELMPI, + Path("python3"), + ["/home/user/script.py"], + None, + False, + KeepsStateForNextUse.HELPFUL, + ) @pytest.fixture def test_program_text() -> str: return ( - 'name: macro\n' - 'ports:\n' - ' f_init: init\n' - ' o_i: out1 out2\n' - ' s: in1 in2\n' - ' o_f: final\n' - 'description: |\n' - ' description\n' - 'supported_settings:\n' - ' alpha: float\n' - 'base_env: login\n' - 'modules:\n' - '- gcc/13.3.0\n' - '- FFTW/3.2.1\n' - 'virtual_env: /home/user/.venv\n' - 'env:\n' - ' SETTING: something\n' - ' VARIABLE: \'42\'\n' - 'execution_model: intelmpi\n' - 'executable: python3\n' - 'args:\n' - '- /home/user/script.py\n' - 'can_share_resources: false\n' - 'keeps_state_for_next_use: helpful\n' - ) + "name: macro\n" + "ports:\n" + " f_init: init\n" + " o_i: out1 out2\n" + " s: in1 in2\n" + " o_f: final\n" + "description: |\n" + " description\n" + "supported_settings:\n" + " alpha: float\n" + "base_env: login\n" + "modules:\n" + "- gcc/13.3.0\n" + "- FFTW/3.2.1\n" + "virtual_env: /home/user/.venv\n" + "env:\n" + " SETTING: something\n" + " VARIABLE: '42'\n" + "execution_model: intelmpi\n" + "executable: python3\n" + "args:\n" + "- /home/user/script.py\n" + "can_share_resources: false\n" + "keeps_state_for_next_use: helpful\n" + ) @pytest.fixture def config_custom_implementations() -> Configuration: return Configuration( - 'testing io of custom implementations', None, None, { - Reference('c1'): Reference('program1'), - Reference('c2.init_model'): Reference('initer2')}) + "testing io of custom implementations", + None, + None, + { + Reference("c1"): Reference("program1"), + Reference("c2.init_model"): Reference("initer2"), + }, + ) @pytest.fixture def config_custom_implementations_text() -> str: return ( - 'ymmsl_version: v0.2\n' - 'description: |\n' - ' testing io of custom implementations\n' - 'custom_implementations:\n' - ' c1: program1\n' - ' c2.init_model: initer2\n' - ) + "ymmsl_version: v0.2\n" + "description: |\n" + " testing io of custom implementations\n" + "custom_implementations:\n" + " c1: program1\n" + " c2.init_model: initer2\n" + ) @pytest.fixture def config_update_checkpoint() -> Configuration: description = "Multiline description for\nthis workflow" resources = [ - ThreadedResReq(Reference('ic'), 4), - ThreadedResReq(Reference('smc'), 4), - MPICoresResReq(Reference('bf'), 4), - ThreadedResReq(Reference('smc2bf'), 1), - ThreadedResReq(Reference('bf2smc'), 1)] + ThreadedResReq(Reference("ic"), 4), + ThreadedResReq(Reference("smc"), 4), + MPICoresResReq(Reference("bf"), 4), + ThreadedResReq(Reference("smc2bf"), 1), + ThreadedResReq(Reference("bf2smc"), 1), + ] checkpoints = Checkpoints( - True, - [CheckpointRangeRule(every=100), - CheckpointAtRule([10, 20, 50])], - [CheckpointRangeRule(start=0, stop=10, every=2), - CheckpointRangeRule(start=10, every=5)]) - resume = {Ref('ic'): Path('/path/to/snapshots/ic.pack'), - Ref('smc'): Path('/path/to/snapshots/smc.pack'), - Ref('bf'): Path('/path/to/snapshots/bf.pack'), - Ref('smc2bf'): Path('/path/to/snapshots/smc2bf.pack'), - Ref('bf2smc'): Path('/path/to/snapshots/bf2smc.pack')} + True, + [CheckpointRangeRule(every=100), CheckpointAtRule([10, 20, 50])], + [ + CheckpointRangeRule(start=0, stop=10, every=2), + CheckpointRangeRule(start=10, every=5), + ], + ) + resume = { + Ref("ic"): Path("/path/to/snapshots/ic.pack"), + Ref("smc"): Path("/path/to/snapshots/smc.pack"), + Ref("bf"): Path("/path/to/snapshots/bf.pack"), + Ref("smc2bf"): Path("/path/to/snapshots/smc2bf.pack"), + Ref("bf2smc"): Path("/path/to/snapshots/bf2smc.pack"), + } return Configuration( - description, None, None, None, None, None, resources, checkpoints, resume) + description, None, None, None, None, None, resources, checkpoints, resume + ) @pytest.fixture def config_duplicate_implementations() -> Configuration: - model1 = Model('impl1', None, 'description') - model2 = Model('impl2', None, 'description') - program1 = Program('impl2', script='impl2') - program2 = Program('impl3', script='impl3') + model1 = Model("impl1", None, "description") + model2 = Model("impl2", None, "description") + program1 = Program("impl2", script="impl2") + program2 = Program("impl3", script="impl3") return Configuration( - 'testing duplicate implementation names', None, - [model1, model2], None, None, [program1, program2]) + "testing duplicate implementation names", + None, + [model1, model2], + None, + None, + [program1, program2], + ) @pytest.fixture def config_consistent_impl_ports() -> Configuration: model1 = Model( - 'implementations_test', - None, 'description', None, - [ - Component( - 'no_implementation', Ports(o_i=['out'], s=['in']), 'description', - None), - Component( - 'impl_no_ports', Ports(f_init=['in'], o_f=['out']), 'description', - 'no_ports'), - ]) + "implementations_test", + None, + "description", + None, + [ + Component( + "no_implementation", Ports(o_i=["out"], s=["in"]), "description", None + ), + Component( + "impl_no_ports", + Ports(f_init=["in"], o_f=["out"]), + "description", + "no_ports", + ), + ], + ) model2 = Model( - 'implementations_test2', - None, 'description', None, - [ - Component( - 'impl_with_ports', Ports(f_init=['in'], o_f=['out']), 'description', - 'with_ports'), - ]) + "implementations_test2", + None, + "description", + None, + [ + Component( + "impl_with_ports", + Ports(f_init=["in"], o_f=["out"]), + "description", + "with_ports", + ), + ], + ) programs = [ - Program( - 'no_ports', None, 'description', None, - script='/home/user/models/bin/modela'), - Program( - 'with_ports', Ports(f_init=['in'], o_f=['out']), - base_env=BaseEnv.LOGIN, - modules=['gcc-6.3.0', 'openmpi-1.10'], - execution_model=ExecutionModel.OPENMPI, - executable=Path('/home/user/models/bin/modelc')), - ] + Program( + "no_ports", None, "description", None, script="/home/user/models/bin/modela" + ), + Program( + "with_ports", + Ports(f_init=["in"], o_f=["out"]), + base_env=BaseEnv.LOGIN, + modules=["gcc-6.3.0", "openmpi-1.10"], + execution_model=ExecutionModel.OPENMPI, + executable=Path("/home/user/models/bin/modelc"), + ), + ] resources = [ - ThreadedResReq(Reference('implementations_test.impl_no_ports'), 1), - MPICoresResReq(Reference('implementations_test2.impl_with_ports'), 4, 4), - ] + ThreadedResReq(Reference("implementations_test.impl_no_ports"), 1), + MPICoresResReq(Reference("implementations_test2.impl_with_ports"), 4, 4), + ] return Configuration( - 'impl_ports', None, [model1, model2], None, None, programs, resources) + "impl_ports", None, [model1, model2], None, None, programs, resources + ) @pytest.fixture def config_inconsistent_impl_ports() -> Configuration: model1 = Model( - 'implementations_test_broken', - None, 'description', None, - [ - Component( - 'no_implementation', Ports(o_i=['out'], s=['in']), 'description', - None), - Component( - 'missing_implementation', Ports(), 'description', - 'implementation_does_not_exist'), - Component( - 'impl_ports_mismatch', Ports(f_init=['in'], o_f=['out']), - 'description', 'ports1'), - Component( - 'model_as_impl', Ports(f_init=['inm'], o_f=['outm', 'out']), - 'description', 'implementations_test2'), - ]) + "implementations_test_broken", + None, + "description", + None, + [ + Component( + "no_implementation", Ports(o_i=["out"], s=["in"]), "description", None + ), + Component( + "missing_implementation", + Ports(), + "description", + "implementation_does_not_exist", + ), + Component( + "impl_ports_mismatch", + Ports(f_init=["in"], o_f=["out"]), + "description", + "ports1", + ), + Component( + "model_as_impl", + Ports(f_init=["inm"], o_f=["outm", "out"]), + "description", + "implementations_test2", + ), + ], + ) model2 = Model( - 'implementations_test2', - Ports(f_init=['inm1'], o_i=['outm'], o_f=['out']), - 'description', None, - [ - Component( - 'impl_also_wrong', Ports(o_i=['out'], s=['in']), 'description', - 'ports2'), - Component( - 'impl_extra_ports', Ports(f_init=['in']), 'description', 'ports3'), - ]) + "implementations_test2", + Ports(f_init=["inm1"], o_i=["outm"], o_f=["out"]), + "description", + None, + [ + Component( + "impl_also_wrong", Ports(o_i=["out"], s=["in"]), "description", "ports2" + ), + Component( + "impl_extra_ports", Ports(f_init=["in"]), "description", "ports3" + ), + ], + ) programs = [ - Program( - 'ports1', Ports(s=['test']), - script='/home/user/models/bin/modela'), - Program( - 'ports2', Ports(f_init=['in'], o_f=['out'], s=['wrong']), - base_env=BaseEnv.LOGIN, - modules=['gcc-6.3.0', 'openmpi-1.10'], - execution_model=ExecutionModel.OPENMPI, - executable=Path('/home/user/models/bin/modelb')), - Program( - 'ports3', Ports(f_init=['in'], s=['test']), - script='/home/user/models/bin/modelc'), - ] + Program("ports1", Ports(s=["test"]), script="/home/user/models/bin/modela"), + Program( + "ports2", + Ports(f_init=["in"], o_f=["out"], s=["wrong"]), + base_env=BaseEnv.LOGIN, + modules=["gcc-6.3.0", "openmpi-1.10"], + execution_model=ExecutionModel.OPENMPI, + executable=Path("/home/user/models/bin/modelb"), + ), + Program( + "ports3", + Ports(f_init=["in"], s=["test"]), + script="/home/user/models/bin/modelc", + ), + ] resources = [ - ThreadedResReq( - Reference('implementations_test_broken.missing_implementation'), 1), - ThreadedResReq( - Reference('implementations_test_broken.impl_ports_mismatch'), 1), - MPICoresResReq( - Reference('implementations_test_broken.model_as_impl.impl_also_wrong'), - 4, 4), - ThreadedResReq( - Reference('implementations_test_broken.model_as_impl.impl_extra_ports'), - 1), - ] + ThreadedResReq( + Reference("implementations_test_broken.missing_implementation"), 1 + ), + ThreadedResReq(Reference("implementations_test_broken.impl_ports_mismatch"), 1), + MPICoresResReq( + Reference("implementations_test_broken.model_as_impl.impl_also_wrong"), 4, 4 + ), + ThreadedResReq( + Reference("implementations_test_broken.model_as_impl.impl_extra_ports"), 1 + ), + ] return Configuration( - 'impl_ports', None, [model1, model2], None, None, programs, resources) + "impl_ports", None, [model1, model2], None, None, programs, resources + ) @pytest.fixture def config_consistent_custom_impls() -> Configuration: model1 = Model( - 'test_model', None, 'Testing consistent (custom) implementations', None, - [ - Component('c1', Ports(), 'description', None), - Component('c2', Ports(), 'description', 'submodel')]) + "test_model", + None, + "Testing consistent (custom) implementations", + None, + [ + Component("c1", Ports(), "description", None), + Component("c2", Ports(), "description", "submodel"), + ], + ) model2 = Model( - 'submodel', None, 'description', None, - [ - Component('init_model', Ports(), 'description', 'initer1'), - Component('optional', Ports(), 'description', None, True), - Component('optional2', Ports(), 'description', 'program1', True)]) + "submodel", + None, + "description", + None, + [ + Component("init_model", Ports(), "description", "initer1"), + Component("optional", Ports(), "description", None, True), + Component("optional2", Ports(), "description", "program1", True), + ], + ) programs = [ - Program('program1', executable=Path('program1')), - Program('initer2', executable=Path('initer2'))] + Program("program1", executable=Path("program1")), + Program("initer2", executable=Path("initer2")), + ] resources = [ - ThreadedResReq(Reference('test_model.c1'), 1), - ThreadedResReq(Reference('test_model.c2.init_model'), 1), - ThreadedResReq(Reference('test_model.c2.optional2'), 1), - ] + ThreadedResReq(Reference("test_model.c1"), 1), + ThreadedResReq(Reference("test_model.c2.init_model"), 1), + ThreadedResReq(Reference("test_model.c2.optional2"), 1), + ] return Configuration( - 'testing consistency of custom implementations', None, [model1, model2], { - Reference('test_model.c1'): Reference('program1'), - Reference('test_model.c2.init_model'): Reference('initer2')}, - None, programs, resources) + "testing consistency of custom implementations", + None, + [model1, model2], + { + Reference("test_model.c1"): Reference("program1"), + Reference("test_model.c2.init_model"): Reference("initer2"), + }, + None, + programs, + resources, + ) @pytest.fixture def config_inconsistent_custom_impls() -> Configuration: model1 = Model( - 'test_model', None, 'description', None, - [ - Component('c1', Ports(), 'description', None), - Component('c2', Ports(), 'description', 'submodel')]) + "test_model", + None, + "description", + None, + [ + Component("c1", Ports(), "description", None), + Component("c2", Ports(), "description", "submodel"), + ], + ) model2 = Model( - 'submodel', None, 'description', None, - [Component('init_model', Ports(), 'description', 'initer1')]) + "submodel", + None, + "description", + None, + [Component("init_model", Ports(), "description", "initer1")], + ) resources = [ - ThreadedResReq(Reference('test_model.c1'), 1), - ThreadedResReq(Reference('test_model.c2.init_model'), 1), - ] + ThreadedResReq(Reference("test_model.c1"), 1), + ThreadedResReq(Reference("test_model.c2.init_model"), 1), + ] return Configuration( - 'testing consistency of custom implementations', None, [model1, model2], { - Reference('test_model.cl'): Reference('program1'), - Reference('test_model.c2.init_model'): Reference('initer2')}, - None, None, resources) + "testing consistency of custom implementations", + None, + [model1, model2], + { + Reference("test_model.cl"): Reference("program1"), + Reference("test_model.c2.init_model"): Reference("initer2"), + }, + None, + None, + resources, + ) @pytest.fixture def config_consistent_settings() -> Configuration: model1 = Model( - 'supported_settings_test', - None, - 'description', - None, - [ - Component('c1', Ports(), 'description', 'a'), - Component( - 'submodel', Ports(), 'description', 'supported_settings_test2'), - ]) + "supported_settings_test", + None, + "description", + None, + [ + Component("c1", Ports(), "description", "a"), + Component("submodel", Ports(), "description", "supported_settings_test2"), + ], + ) model2 = Model( - 'supported_settings_test2', - None, - 'description', - SupportedSettings({'delta': 'bool'}), - [ - Component('c1', Ports(), 'description', 'b'), - Component('c2', Ports(), 'description', 'c', False, 10)], - []) - - settings = Settings({ - 'alpha': 3.2, - 'beta': 10, - 'gamma': 'text', - 'delta': 'text', - 'submodel.delta': False, - 'epsilon': [10, 11]}) + "supported_settings_test2", + None, + "description", + SupportedSettings({"delta": "bool"}), + [ + Component("c1", Ports(), "description", "b"), + Component("c2", Ports(), "description", "c", False, 10), + ], + [], + ) + + settings = Settings( + { + "alpha": 3.2, + "beta": 10, + "gamma": "text", + "delta": "text", + "submodel.delta": False, + "epsilon": [10, 11], + } + ) program_a = Program( - 'a', Ports(), 'a', SupportedSettings({'alpha': 'float', 'beta': 'int'}), - execution_model=ExecutionModel.DIRECT, executable=Path('a')) + "a", + Ports(), + "a", + SupportedSettings({"alpha": "float", "beta": "int"}), + execution_model=ExecutionModel.DIRECT, + executable=Path("a"), + ) program_b = Program( - 'b', Ports(), 'b', SupportedSettings({'gamma': 'str'}), - execution_model=ExecutionModel.INTELMPI, executable=Path('b')) + "b", + Ports(), + "b", + SupportedSettings({"gamma": "str"}), + execution_model=ExecutionModel.INTELMPI, + executable=Path("b"), + ) program_c = Program( - 'c', Ports(), 'c', SupportedSettings({'delta': 'bool'}), - execution_model=ExecutionModel.OPENMPI, executable=Path('c')) + "c", + Ports(), + "c", + SupportedSettings({"delta": "bool"}), + execution_model=ExecutionModel.OPENMPI, + executable=Path("c"), + ) programs = [program_a, program_b, program_c] resources = [ - ThreadedResReq(Reference('supported_settings_test.c1'), 1), - MPICoresResReq(Reference('supported_settings_test.submodel.c1'), 16), - MPICoresResReq(Reference('supported_settings_test.submodel.c2'), 4, 4)] + ThreadedResReq(Reference("supported_settings_test.c1"), 1), + MPICoresResReq(Reference("supported_settings_test.submodel.c1"), 16), + MPICoresResReq(Reference("supported_settings_test.submodel.c2"), 4, 4), + ] return Configuration( - 'config3', None, [model1, model2], None, settings, programs, resources) + "config3", None, [model1, model2], None, settings, programs, resources + ) @pytest.fixture def config_consistent_resources() -> Configuration: model1 = Model( - 'resources_test', - None, - 'description', - None, - [ - Component('singlethreaded', Ports(), 'description', 'a'), - Component('multithreaded', Ports(), 'description', 'b'), - Component('submodel', Ports(), 'description', 'resources_test2'), - ]) + "resources_test", + None, + "description", + None, + [ + Component("singlethreaded", Ports(), "description", "a"), + Component("multithreaded", Ports(), "description", "b"), + Component("submodel", Ports(), "description", "resources_test2"), + ], + ) model2 = Model( - 'resources_test2', - None, - 'description', - None, - [ - Component('mpi_cores1', Ports(), 'description', 'c'), - Component('mpi_cores2', Ports(), 'description', 'd'), - Component('mpi_nodes1', Ports(), 'description', 'c'), - Component('mpi_nodes2', Ports(), 'description', 'd')], - []) + "resources_test2", + None, + "description", + None, + [ + Component("mpi_cores1", Ports(), "description", "c"), + Component("mpi_cores2", Ports(), "description", "d"), + Component("mpi_nodes1", Ports(), "description", "c"), + Component("mpi_nodes2", Ports(), "description", "d"), + ], + [], + ) em = { - 'a': ExecutionModel.DIRECT, - 'b': ExecutionModel.DIRECT, - 'c': ExecutionModel.OPENMPI, - 'd': ExecutionModel.OPENMPI} - programs = [Program(x, script='script', execution_model=em[x]) for x in 'abcd'] + "a": ExecutionModel.DIRECT, + "b": ExecutionModel.DIRECT, + "c": ExecutionModel.OPENMPI, + "d": ExecutionModel.OPENMPI, + } + programs = [Program(x, script="script", execution_model=em[x]) for x in "abcd"] resources = [ - ThreadedResReq(Reference('resources_test.singlethreaded'), 1), - ThreadedResReq(Reference('resources_test.multithreaded'), 8), - MPICoresResReq(Reference('resources_test.submodel.mpi_cores1'), 16), - MPICoresResReq(Reference('resources_test.submodel.mpi_cores2'), 4, 4), - MPINodesResReq(Reference('resources_test.submodel.mpi_nodes1'), 10, 16), - MPINodesResReq(Reference('resources_test.submodel.mpi_nodes2'), 10, 4, 4)] + ThreadedResReq(Reference("resources_test.singlethreaded"), 1), + ThreadedResReq(Reference("resources_test.multithreaded"), 8), + MPICoresResReq(Reference("resources_test.submodel.mpi_cores1"), 16), + MPICoresResReq(Reference("resources_test.submodel.mpi_cores2"), 4, 4), + MPINodesResReq(Reference("resources_test.submodel.mpi_nodes1"), 10, 16), + MPINodesResReq(Reference("resources_test.submodel.mpi_nodes2"), 10, 4, 4), + ] return Configuration( - 'consistent_resources', None, [model1, model2], None, None, programs, - resources) + "consistent_resources", None, [model1, model2], None, None, programs, resources + ) @pytest.fixture def config_inconsistent_resources() -> Configuration: model1 = Model( - 'got_resources', None, 'description', None, - [ - Component('singlethreaded', Ports(), 'description', 'a'), - Component('with_mpi', Ports(), 'description', 'b'), - Component('without_mpi', Ports(), 'description', 'a'), - ]) + "got_resources", + None, + "description", + None, + [ + Component("singlethreaded", Ports(), "description", "a"), + Component("with_mpi", Ports(), "description", "b"), + Component("without_mpi", Ports(), "description", "a"), + ], + ) model2 = Model( - 'missing_resources', None, 'description', None, - [Component('singlethreaded2', Ports(), 'description', 'a')]) + "missing_resources", + None, + "description", + None, + [Component("singlethreaded2", Ports(), "description", "a")], + ) programs = [ - Program('a', script='a', execution_model=ExecutionModel.DIRECT), - Program('b', script='b', execution_model=ExecutionModel.INTELMPI)] + Program("a", script="a", execution_model=ExecutionModel.DIRECT), + Program("b", script="b", execution_model=ExecutionModel.INTELMPI), + ] resources = [ - ThreadedResReq(Ref('got_resources.singlethreaded'), 1), - ThreadedResReq(Ref('got_resources.with_mpi'), 2), - MPICoresResReq(Ref('got_resources.without_mpi'), 10)] + ThreadedResReq(Ref("got_resources.singlethreaded"), 1), + ThreadedResReq(Ref("got_resources.with_mpi"), 2), + MPICoresResReq(Ref("got_resources.without_mpi"), 10), + ] return Configuration( - 'test_config', None, [model1, model2], None, None, programs, resources) + "test_config", None, [model1, model2], None, None, programs, resources + ) @pytest.fixture def config_consistent_partial_resources() -> Configuration: model1 = Model( - 'resources_test', - None, - 'description', - None, - [ - Component('singlethreaded', Ports(), 'description', 'a'), - Component('multithreaded', Ports(), 'description', 'b'), - ]) + "resources_test", + None, + "description", + None, + [ + Component("singlethreaded", Ports(), "description", "a"), + Component("multithreaded", Ports(), "description", "b"), + ], + ) model2 = Model( - 'resources_test2', - None, - 'description', - None, - [ - Component('mpi_cores1', Ports(), 'description', 'c'), - Component('mpi_cores2', Ports(), 'description', 'd'), - Component('mpi_nodes1', Ports(), 'description', 'c'), - Component('mpi_nodes2', Ports(), 'description', 'd')], - []) + "resources_test2", + None, + "description", + None, + [ + Component("mpi_cores1", Ports(), "description", "c"), + Component("mpi_cores2", Ports(), "description", "d"), + Component("mpi_nodes1", Ports(), "description", "c"), + Component("mpi_nodes2", Ports(), "description", "d"), + ], + [], + ) em = { - 'a': ExecutionModel.DIRECT, - 'b': ExecutionModel.DIRECT, - 'c': ExecutionModel.OPENMPI, - 'd': ExecutionModel.OPENMPI} - programs = [Program(x, script='script', execution_model=em[x]) for x in 'abcd'] + "a": ExecutionModel.DIRECT, + "b": ExecutionModel.DIRECT, + "c": ExecutionModel.OPENMPI, + "d": ExecutionModel.OPENMPI, + } + programs = [Program(x, script="script", execution_model=em[x]) for x in "abcd"] resources = [ - ThreadedResReq(Reference('resources_test.singlethreaded'), 1), - ThreadedResReq(Reference('resources_test.multithreaded'), 4), - MPICoresResReq(Reference('resources_test2.mpi_cores1'), 16), - MPINodesResReq(Reference('resources_test2.mpi_nodes1'), 10, 16), - MPINodesResReq(Reference('resources_test2.mpi_nodes2'), 10, 4, 4)] + ThreadedResReq(Reference("resources_test.singlethreaded"), 1), + ThreadedResReq(Reference("resources_test.multithreaded"), 4), + MPICoresResReq(Reference("resources_test2.mpi_cores1"), 16), + MPINodesResReq(Reference("resources_test2.mpi_nodes1"), 10, 16), + MPINodesResReq(Reference("resources_test2.mpi_nodes2"), 10, 4, 4), + ] return Configuration( - 'consistent_resources', None, [model1, model2], None, None, programs, - resources) + "consistent_resources", None, [model1, model2], None, None, programs, resources + ) @pytest.fixture def config_component_loop() -> Configuration: main = Model( - 'main', - Ports(), - 'Test model with a nested model loop', - None, - [ - Component( - 'init', Ports(o_f='final'), - 'Calculates initial conditions', 'submodel1'), - Component( - 'macro', Ports(f_init='init', o_i='out', s='in'), 'Macro model', - 'submodel1'), - Component( - 'micro', Ports('init', o_f='final'), - 'Micro model', 'micro'), - ], [ - Conduit('init.final', 'macro.init'), - Conduit('macro.out', 'micro.init'), - Conduit('micro.out', 'macro.init'), - ]) + "main", + Ports(), + "Test model with a nested model loop", + None, + [ + Component( + "init", Ports(o_f="final"), "Calculates initial conditions", "submodel1" + ), + Component( + "macro", + Ports(f_init="init", o_i="out", s="in"), + "Macro model", + "submodel1", + ), + Component("micro", Ports("init", o_f="final"), "Micro model", "micro"), + ], + [ + Conduit("init.final", "macro.init"), + Conduit("macro.out", "micro.init"), + Conduit("micro.out", "macro.init"), + ], + ) submodel1 = Model( - 'submodel1', - Ports(f_init='init', o_i='out', s='in'), - 'Implements the macro model', - None, - [ - Component( - 'first', Ports(f_init='init', o_f='final'), 'First model', - 'submodel2'), - Component( - 'second', Ports(f_init='init', o_i='out', s='in'), 'Second model', - 'second') - ], [ - Conduit('init', 'first.init'), - Conduit('first.final', 'second.init'), - Conduit('second.out', 'out'), - Conduit('second.in', 'in') - ]) + "submodel1", + Ports(f_init="init", o_i="out", s="in"), + "Implements the macro model", + None, + [ + Component( + "first", Ports(f_init="init", o_f="final"), "First model", "submodel2" + ), + Component( + "second", + Ports(f_init="init", o_i="out", s="in"), + "Second model", + "second", + ), + ], + [ + Conduit("init", "first.init"), + Conduit("first.final", "second.init"), + Conduit("second.out", "out"), + Conduit("second.in", "in"), + ], + ) submodel2 = Model( - 'submodel2', - Ports(f_init='init', o_f='final'), - 'Processes the input a bit', - None, - [Component('micro', Ports('init', o_f='final'), 'Ooops...', 'submodel1')], - [ - Conduit('init', 'micro.init'), - Conduit('micro.final', 'final') - ]) + "submodel2", + Ports(f_init="init", o_f="final"), + "Processes the input a bit", + None, + [Component("micro", Ports("init", o_f="final"), "Ooops...", "submodel1")], + [Conduit("init", "micro.init"), Conduit("micro.final", "final")], + ) programs = [ - Program('micro', script='micro', execution_model=ExecutionModel.DIRECT), - Program('second', script='second', execution_model=ExecutionModel.INTELMPI)] + Program("micro", script="micro", execution_model=ExecutionModel.DIRECT), + Program("second", script="second", execution_model=ExecutionModel.INTELMPI), + ] resources = [ - ThreadedResReq(Ref('micro'), 1), - ThreadedResReq(Ref('second'), 2),] + ThreadedResReq(Ref("micro"), 1), + ThreadedResReq(Ref("second"), 2), + ] return Configuration( - 'config_Component_loop', None, [main, submodel1, submodel2], None, None, - programs, resources) + "config_Component_loop", + None, + [main, submodel1, submodel2], + None, + None, + programs, + resources, + ) diff --git a/ymmsl/v0_2/tests/test_component.py b/ymmsl/v0_2/tests/test_component.py index 321e7d1..26ebbb5 100644 --- a/ymmsl/v0_2/tests/test_component.py +++ b/ymmsl/v0_2/tests/test_component.py @@ -4,60 +4,66 @@ def test_component_declaration() -> None: - test_decl = Component('test', Ports(), 'description', 'ns.model') + test_decl = Component("test", Ports(), "description", "ns.model") assert isinstance(test_decl.name, Reference) - assert str(test_decl.name) == 'test' + assert str(test_decl.name) == "test" assert len(test_decl.ports) == 0 - assert test_decl.description == 'description' + assert test_decl.description == "description" assert test_decl.optional is False assert isinstance(test_decl.implementation, Reference) - assert str(test_decl.implementation) == 'ns.model' + assert str(test_decl.implementation) == "ns.model" assert test_decl.multiplicity == [] - assert str(test_decl) == 'test' + assert str(test_decl) == "test" def test_component_multiplicity() -> None: - test_decl = Component('test', Ports(), 'description', 'ns.model', False, 10) + test_decl = Component("test", Ports(), "description", "ns.model", False, 10) assert isinstance(test_decl.name, Reference) - assert str(test_decl.name) == 'test' + assert str(test_decl.name) == "test" assert test_decl.multiplicity == [10] - assert str(test_decl) == 'test[0:10]' + assert str(test_decl) == "test[0:10]" - test_decl = Component('test', Ports(), 'description', 'ns2.model2', False, [1, 2]) + test_decl = Component("test", Ports(), "description", "ns2.model2", False, [1, 2]) assert isinstance(test_decl.name, Reference) - assert str(test_decl.name) == 'test' - assert str(test_decl.implementation) == 'ns2.model2' + assert str(test_decl.name) == "test" + assert str(test_decl.implementation) == "ns2.model2" assert test_decl.multiplicity == [1, 2] - assert str(test_decl) == 'test[0:1][0:2]' + assert str(test_decl) == "test[0:1][0:2]" with pytest.raises(ValueError): - test_decl = Component('test', Ports(), 'description', 'ns2.model2[1]') + test_decl = Component("test", Ports(), "description", "ns2.model2[1]") def test_component_instances() -> None: - c1 = Component('test', Ports(), 'description', 'model') - assert c1.instances() == [Reference('test')] + c1 = Component("test", Ports(), "description", "model") + assert c1.instances() == [Reference("test")] - c2 = Component('test', Ports(), 'description', 'model', False, 3) + c2 = Component("test", Ports(), "description", "model", False, 3) assert c2.instances() == [ - Reference('test[0]'), Reference('test[1]'), Reference('test[2]')] + Reference("test[0]"), + Reference("test[1]"), + Reference("test[2]"), + ] - c3 = Component('test', Ports(), 'description', 'model', False, [2, 2]) + c3 = Component("test", Ports(), "description", "model", False, [2, 2]) assert c3.instances() == [ - Reference('test[0][0]'), Reference('test[0][1]'), - Reference('test[1][0]'), Reference('test[1][1]')] + Reference("test[0][0]"), + Reference("test[0][1]"), + Reference("test[1][0]"), + Reference("test[1][1]"), + ] def test_component_ports() -> None: - c = Component('test', Ports(['init'], ['out'], ['in'], ['final']), 'description') + c = Component("test", Ports(["init"], ["out"], ["in"], ["final"]), "description") assert isinstance(c.ports, Ports) - assert c.ports['init'] == Port(Identifier('init'), Operator.F_INIT, Timeline('')) - assert c.ports['out'] == Port(Identifier('out'), Operator.O_I, Timeline('')) - assert c.ports['in'] == Port(Identifier('in'), Operator.S, Timeline('')) - assert c.ports['final'] == Port(Identifier('final'), Operator.O_F, Timeline('')) + assert c.ports["init"] == Port(Identifier("init"), Operator.F_INIT, Timeline("")) + assert c.ports["out"] == Port(Identifier("out"), Operator.O_I, Timeline("")) + assert c.ports["in"] == Port(Identifier("in"), Operator.S, Timeline("")) + assert c.ports["final"] == Port(Identifier("final"), Operator.O_F, Timeline("")) diff --git a/ymmsl/v0_2/tests/test_configuration.py b/ymmsl/v0_2/tests/test_configuration.py index 9575911..a34c8e4 100644 --- a/ymmsl/v0_2/tests/test_configuration.py +++ b/ymmsl/v0_2/tests/test_configuration.py @@ -30,15 +30,15 @@ def test_configuration() -> None: setting_values: OrderedDict[str, SettingValue] = OrderedDict() settings = Settings(setting_values) - model1 = Model('model1', None, 'description', None, []) - model2 = Model('model2', None, 'description', None, []) - config = Configuration('description', None, [model1, model2], settings) + model1 = Model("model1", None, "description", None, []) + model2 = Model("model2", None, "description", None, []) + config = Configuration("description", None, [model1, model2], settings) - assert config.description == 'description' + assert config.description == "description" assert len(config.models) == 2 - assert config.models[Ref('model1')] is model1 - assert config.models[Ref('model2')] is model2 + assert config.models[Ref("model1")] is model1 + assert config.models[Ref("model2")] is model2 assert isinstance(config.settings, Settings) assert len(config.settings) == 0 @@ -46,58 +46,60 @@ def test_configuration() -> None: def test_create_duplicate_models() -> None: with pytest.raises(ValueError): - Configuration('desc', models=[Model('a'), Model('a')]) + Configuration("desc", models=[Model("a"), Model("a")]) def test_create_duplicate_programs() -> None: with pytest.raises(ValueError): Configuration( - 'desc', programs=[Program('p', script='p'), Program('p', script='q')]) + "desc", programs=[Program("p", script="p"), Program("p", script="q")] + ) def test_load_models() -> None: text = ( - 'ymmsl_version: v0.2\n' - 'description: Test loading multiple models\n' - 'models:\n' - ' macro_micro:\n' - ' components:\n' - ' macro:\n' - ' ports:\n' - ' o_i: out\n' - ' s: in\n' - ' description: the macro component\n' - ' implementation: macro_program\n' - ' micro:\n' - ' ports:\n' - ' f_init: init\n' - ' o_f: final\n' - ' description: the micro component\n' - ' implementation: micro_program\n' - ' do_nothing:\n' - ' components:\n' - ' nil:\n' - ' ports: {}\n' - ' description: a component that does nothing\n' - ' implementation: nil_program\n' - ) + "ymmsl_version: v0.2\n" + "description: Test loading multiple models\n" + "models:\n" + " macro_micro:\n" + " components:\n" + " macro:\n" + " ports:\n" + " o_i: out\n" + " s: in\n" + " description: the macro component\n" + " implementation: macro_program\n" + " micro:\n" + " ports:\n" + " f_init: init\n" + " o_f: final\n" + " description: the micro component\n" + " implementation: micro_program\n" + " do_nothing:\n" + " components:\n" + " nil:\n" + " ports: {}\n" + " description: a component that does nothing\n" + " implementation: nil_program\n" + ) configuration = load(text) assert isinstance(configuration, Configuration) assert len(configuration.models) == 2 - mm = configuration.models[Ref('macro_micro')] - assert mm.components[Ref('macro')].ports['out'] == Port( - Identifier('out'), Operator.O_I, Timeline('')) - dn = configuration.models[Ref('do_nothing')] - assert len(dn.components[Ref('nil')].ports) == 0 + mm = configuration.models[Ref("macro_micro")] + assert mm.components[Ref("macro")].ports["out"] == Port( + Identifier("out"), Operator.O_I, Timeline("") + ) + dn = configuration.models[Ref("do_nothing")] + assert len(dn.components[Ref("nil")].ports) == 0 def test_load_nil_settings() -> None: text = ( - 'ymmsl_version: v0.2\n' - 'description: test loading nil-valued settings\n' - 'settings:\n' + "ymmsl_version: v0.2\n" + "description: test loading nil-valued settings\n" + "settings:\n" ) configuration = load(text) @@ -113,23 +115,21 @@ def test_load_custom_implementations(config_custom_implementations_text: str) -> assert isinstance(configuration, Configuration) assert configuration.custom_implementations == { - Ref('c1'): Ref('program1'), - Ref('c2.init_model'): Ref('initer2') - } + Ref("c1"): Ref("program1"), + Ref("c2.init_model"): Ref("initer2"), + } def test_dump_custom_implementations( - config_custom_implementations: Configuration, - config_custom_implementations_text: str) -> None: + config_custom_implementations: Configuration, + config_custom_implementations_text: str, +) -> None: text = dump(config_custom_implementations) assert text == config_custom_implementations_text def test_load_no_settings() -> None: - text = ( - 'ymmsl_version: v0.2\n' - 'description: test loading with no settings\n' - ) + text = "ymmsl_version: v0.2\ndescription: test loading with no settings\n" configuration = load(text) @@ -140,59 +140,54 @@ def test_load_no_settings() -> None: def test_dump_empty_settings() -> None: - configuration = Configuration('empty settings!', None, None, Settings()) + configuration = Configuration("empty settings!", None, None, Settings()) text = dump(configuration) - assert text == ( - 'ymmsl_version: v0.2\n' - 'description: |\n' - ' empty settings!\n') + assert text == ("ymmsl_version: v0.2\ndescription: |\n empty settings!\n") def test_load_resources() -> None: text = ( - 'ymmsl_version: v0.2\n' - 'description: testing loading of resources\n' - 'resources:\n' - ' macro:\n' - ' threads: 10\n' - ' micro:\n' - ' threads: 1\n' - ) + "ymmsl_version: v0.2\n" + "description: testing loading of resources\n" + "resources:\n" + " macro:\n" + " threads: 10\n" + " micro:\n" + " threads: 1\n" + ) configuration = load(text) assert isinstance(configuration, Configuration) - assert configuration.resources[Ref('macro')].name == 'macro' - assert configuration.resources[Ref('macro')].threads == 10 # type: ignore - assert configuration.resources[Ref('micro')].name == 'micro' - assert configuration.resources[Ref('micro')].threads == 1 # type: ignore + assert configuration.resources[Ref("macro")].name == "macro" + assert configuration.resources[Ref("macro")].threads == 10 # type: ignore + assert configuration.resources[Ref("micro")].name == "micro" + assert configuration.resources[Ref("micro")].threads == 1 # type: ignore def test_dump_resources() -> None: - resources = [ - ThreadedResReq(Ref('macro'), 10), - ThreadedResReq(Ref('micro'), 1)] + resources = [ThreadedResReq(Ref("macro"), 10), ThreadedResReq(Ref("micro"), 1)] - configuration = Configuration('dumping resources', resources=resources) + configuration = Configuration("dumping resources", resources=resources) text = dump(configuration) assert text == ( - 'ymmsl_version: v0.2\n' - 'description: |\n' - ' dumping resources\n' - 'resources:\n' - ' macro:\n' - ' threads: 10\n' - ' micro:\n' - ' threads: 1\n' - ) + "ymmsl_version: v0.2\n" + "description: |\n" + " dumping resources\n" + "resources:\n" + " macro:\n" + " threads: 10\n" + " micro:\n" + " threads: 1\n" + ) def test_configuration_update_description() -> None: - description1 = '\n' - description2 = 'single line description\n' - description3 = 'multiline\ndescription\n' + description1 = "\n" + description2 = "single line description\n" + description3 = "multiline\ndescription\n" overlay1 = Configuration(description=description1) overlay2 = Configuration(description=description2) @@ -201,175 +196,224 @@ def test_configuration_update_description() -> None: base = Configuration(description=description1) base.update(overlay1) - assert base.description == '\n' + assert base.description == "\n" base.update(overlay2) assert base.description == description2 base.update(overlay3) - assert base.description == description2 + '\n' + description3 + assert base.description == description2 + "\n" + description3 base.update(overlay1) - assert base.description == description2 + '\n' + description3 + assert base.description == description2 + "\n" + description3 def test_configuration_update_model_error() -> None: base = Configuration( - 'Configuration for testing', None, [ - Model('model', Ports(), 'description', None, [ - Component('macro', Ports(o_i='out', s='in'), 'description'), - Component('micro', Ports(f_init='init', o_f='final'), 'description') - ], [ - Conduit('macro.out', 'micro.init'), - Conduit('micro.final', 'macro.in')] - ) - ]) + "Configuration for testing", + None, + [ + Model( + "model", + Ports(), + "description", + None, + [ + Component("macro", Ports(o_i="out", s="in"), "description"), + Component( + "micro", Ports(f_init="init", o_f="final"), "description" + ), + ], + [ + Conduit("macro.out", "micro.init"), + Conduit("micro.final", "macro.in"), + ], + ) + ], + ) overlay1 = Configuration( - 'Extra component', None, - [ - Model('model', None, 'description', None, - [Component('micro2', Ports(), 'description')], []) - ]) + "Extra component", + None, + [ + Model( + "model", + None, + "description", + None, + [Component("micro2", Ports(), "description")], + [], + ) + ], + ) with pytest.raises(RuntimeError): base.update(overlay1) overlay2 = Configuration( - 'Extra conduit', None, [Model('model', None, 'description', None, [], [ - Conduit('micro.final', 'macro.in2')])]) + "Extra conduit", + None, + [ + Model( + "model", + None, + "description", + None, + [], + [Conduit("micro.final", "macro.in2")], + ) + ], + ) with pytest.raises(RuntimeError): base.update(overlay2) def test_configuration_update_imports() -> None: - base = Configuration('', [ - ImportStatement('a.b.c', 'implementation', 'p'), - ImportStatement('d.b.c', 'implementation', 'q')]) + base = Configuration( + "", + [ + ImportStatement("a.b.c", "implementation", "p"), + ImportStatement("d.b.c", "implementation", "q"), + ], + ) - overlay = Configuration('', [ - ImportStatement('e.f.g', 'implementation', 's'), - ImportStatement('b.f.h', 'implementation', 't')]) + overlay = Configuration( + "", + [ + ImportStatement("e.f.g", "implementation", "s"), + ImportStatement("b.f.h", "implementation", "t"), + ], + ) base.update(overlay) - assert base.imports[0].module == 'a.b.c' - assert base.imports[0].name == 'p' - assert base.imports[1].module == 'd.b.c' - assert base.imports[1].name == 'q' - assert base.imports[2].module == 'e.f.g' - assert base.imports[2].name == 's' - assert base.imports[3].module == 'b.f.h' - assert base.imports[3].name == 't' + assert base.imports[0].module == "a.b.c" + assert base.imports[0].name == "p" + assert base.imports[1].module == "d.b.c" + assert base.imports[1].name == "q" + assert base.imports[2].module == "e.f.g" + assert base.imports[2].name == "s" + assert base.imports[3].module == "b.f.h" + assert base.imports[3].name == "t" def test_configuration_update_custom_implementations() -> None: base = Configuration( - 'Configuration for testing', custom_implementations={ - Ref('c1'): Ref('impl1')}) + "Configuration for testing", custom_implementations={Ref("c1"): Ref("impl1")} + ) overlay1 = Configuration( - 'Extra and override', custom_implementations={ - Ref('c1'): Ref('impl2'), - Ref('c2.a'): Ref('impl1')}) + "Extra and override", + custom_implementations={Ref("c1"): Ref("impl2"), Ref("c2.a"): Ref("impl1")}, + ) base.update(overlay1) - assert base.custom_implementations[Ref('c1')] == 'impl2' - assert base.custom_implementations[Ref('c2.a')] == 'impl1' + assert base.custom_implementations[Ref("c1")] == "impl2" + assert base.custom_implementations[Ref("c2.a")] == "impl1" def test_configuration_update_programs() -> None: base = Configuration( - 'Configuration for testing', - programs=[Program('impl1', executable=Path('impl1'))]) + "Configuration for testing", + programs=[Program("impl1", executable=Path("impl1"))], + ) overlay1 = Configuration( - 'Extra program', - programs=[Program('impl2', executable=Path('impl2'))]) + "Extra program", programs=[Program("impl2", executable=Path("impl2"))] + ) base.update(overlay1) - assert base.programs[Ref('impl1')].name == 'impl1' - assert base.programs[Ref('impl2')].name == 'impl2' + assert base.programs[Ref("impl1")].name == "impl1" + assert base.programs[Ref("impl2")].name == "impl2" overlay2 = Configuration( - 'Conflicting program', - programs=[Program('impl1', executable=Path('impl3'))]) + "Conflicting program", programs=[Program("impl1", executable=Path("impl3"))] + ) with pytest.raises(RuntimeError): base.update(overlay2) def test_configuration_update_resources_override() -> None: - resources1 = ThreadedResReq(Ref('my.macro'), 10) - resources2 = ThreadedResReq(Ref('my.micro'), 100) - base = Configuration('', resources=[resources1, resources2]) + resources1 = ThreadedResReq(Ref("my.macro"), 10) + resources2 = ThreadedResReq(Ref("my.micro"), 100) + base = Configuration("", resources=[resources1, resources2]) - resources3 = ThreadedResReq(Ref('my.micro'), 2) - overlay = Configuration('', resources=[resources3]) + resources3 = ThreadedResReq(Ref("my.micro"), 2) + overlay = Configuration("", resources=[resources3]) base.update(overlay) assert len(base.resources) == 2 - assert base.resources[Ref('my.macro')] == resources1 - assert base.resources[Ref('my.micro')] == resources3 + assert base.resources[Ref("my.macro")] == resources1 + assert base.resources[Ref("my.micro")] == resources3 def test_configuration_update_checkpoint( - config_update_checkpoint: Configuration) -> None: + config_update_checkpoint: Configuration, +) -> None: # Note: test_checkpoint.py tests merging of checkpoint definitions - base = Configuration('', checkpoints=Checkpoints( - wallclock_time=config_update_checkpoint.checkpoints.wallclock_time)) - overlay = Configuration('', checkpoints=Checkpoints( - simulation_time=config_update_checkpoint.checkpoints.simulation_time)) + base = Configuration( + "", + checkpoints=Checkpoints( + wallclock_time=config_update_checkpoint.checkpoints.wallclock_time + ), + ) + overlay = Configuration( + "", + checkpoints=Checkpoints( + simulation_time=config_update_checkpoint.checkpoints.simulation_time + ), + ) assert base.checkpoints.simulation_time == [] assert overlay.checkpoints.wallclock_time == [] base.update(overlay) - assert (base.checkpoints.simulation_time == overlay.checkpoints.simulation_time) + assert base.checkpoints.simulation_time == overlay.checkpoints.simulation_time def test_configuration_update_resume() -> None: - base = Configuration('', resume={Ref('a'): Path('a')}) - overlay = Configuration('', resume={Ref('b'): Path('b')}) + base = Configuration("", resume={Ref("a"): Path("a")}) + overlay = Configuration("", resume={Ref("b"): Path("b")}) base.update(overlay) assert len(base.resume) == 2 - assert base.resume[Ref('a')] == Path('a') - assert base.resume[Ref('b')] == Path('b') + assert base.resume[Ref("a")] == Path("a") + assert base.resume[Ref("b")] == Path("b") def test_configuration_update_resume_override() -> None: - base = Configuration('', resume={Ref('a'): Path('a'), Ref('b'): Path('b')}) - overlay = Configuration('', resume={Ref('b'): Path('b_update')}) + base = Configuration("", resume={Ref("a"): Path("a"), Ref("b"): Path("b")}) + overlay = Configuration("", resume={Ref("b"): Path("b_update")}) base.update(overlay) assert len(base.resume) == 2 - assert base.resume[Ref('a')] == Path('a') - assert base.resume[Ref('b')] == Path('b_update') + assert base.resume[Ref("a")] == Path("a") + assert base.resume[Ref("b")] == Path("b_update") def test_configuration_update_resources_add() -> None: - resources1 = ThreadedResReq(Ref('my.macro'), 10) - base = Configuration('', resources=[resources1]) + resources1 = ThreadedResReq(Ref("my.macro"), 10) + base = Configuration("", resources=[resources1]) - resources2 = ThreadedResReq(Ref('my.micro'), 2) - overlay = Configuration('', resources=[resources2]) + resources2 = ThreadedResReq(Ref("my.micro"), 2) + overlay = Configuration("", resources=[resources2]) base.update(overlay) assert len(base.resources) == 2 - assert base.resources[Ref('my.macro')] == resources1 - assert base.resources[Ref('my.micro')] == resources2 + assert base.resources[Ref("my.macro")] == resources1 + assert base.resources[Ref("my.micro")] == resources2 def test_configuration_load_default() -> None: - config = load('ymmsl_version: v0.2') + config = load("ymmsl_version: v0.2") assert isinstance(config, Configuration) - assert config.description == '' + assert config.description == "" assert config.imports == [] assert config.models == {} assert config.custom_implementations == {} @@ -382,83 +426,87 @@ def test_configuration_load_default() -> None: def test_configuration_dump_default() -> None: text = dump(Configuration()) - assert text == 'ymmsl_version: v0.2\n' + assert text == "ymmsl_version: v0.2\n" def test_check_duplicate_implementations( - config_duplicate_implementations: Configuration) -> None: + config_duplicate_implementations: Configuration, +) -> None: with pytest.raises(RuntimeError) as e: config_duplicate_implementations.check_consistent() - assert len(str(e.value).split('\n')) == 2 + assert len(str(e.value).split("\n")) == 2 -def test_check_component_loop( - config_component_loop: Configuration) -> None: +def test_check_component_loop(config_component_loop: Configuration) -> None: with pytest.raises(RuntimeError) as e: config_component_loop.check_consistent() - assert 'loop of components and models' in str(e) + assert "loop of components and models" in str(e) def test_check_consistent_implementation_ports( - config_consistent_impl_ports: Configuration) -> None: + config_consistent_impl_ports: Configuration, +) -> None: config_consistent_impl_ports.check_consistent() def test_check_inconsistent_implementation_ports( - config_inconsistent_impl_ports: Configuration) -> None: + config_inconsistent_impl_ports: Configuration, +) -> None: with pytest.raises(RuntimeError) as e: config_inconsistent_impl_ports.check_consistent() - assert len(str(e.value).split('\n')) == 8 + assert len(str(e.value).split("\n")) == 8 with pytest.raises(RuntimeError) as e: config_inconsistent_impl_ports.check_consistent(False) - assert len(str(e.value).split('\n')) == 7 + assert len(str(e.value).split("\n")) == 7 def test_check_consistent_custom_implementations( - config_consistent_custom_impls: Configuration) -> None: + config_consistent_custom_impls: Configuration, +) -> None: config_consistent_custom_impls.check_consistent() def test_check_inconsistent_custom_impls( - config_inconsistent_custom_impls: Configuration) -> None: + config_inconsistent_custom_impls: Configuration, +) -> None: with pytest.raises(RuntimeError) as e: config_inconsistent_custom_impls.check_consistent() - assert len(str(e.value).split('\n')) == 3 + assert len(str(e.value).split("\n")) == 3 def test_check_consistent_settings(config_consistent_settings: Configuration) -> None: config_consistent_settings.check_consistent() - config_consistent_settings.settings['submodel.c2[3].delta'] = 10 + config_consistent_settings.settings["submodel.c2[3].delta"] = 10 with pytest.raises(RuntimeError) as e: config_consistent_settings.check_consistent() - assert len(str(e.value).split('\n')) == 2 + assert len(str(e.value).split("\n")) == 2 - del config_consistent_settings.settings['submodel.c2[3].delta'] - config_consistent_settings.settings['alpha'] = [[1.2, 3.4], [5.6, 7.8]] + del config_consistent_settings.settings["submodel.c2[3].delta"] + config_consistent_settings.settings["alpha"] = [[1.2, 3.4], [5.6, 7.8]] with pytest.raises(RuntimeError) as e: config_consistent_settings.check_consistent() - assert len(str(e.value).split('\n')) == 2 + assert len(str(e.value).split("\n")) == 2 - config_consistent_settings.settings['alpha'] = 3.2 + config_consistent_settings.settings["alpha"] = 3.2 - model2 = config_consistent_settings.models[Ref('supported_settings_test2')] + model2 = config_consistent_settings.models[Ref("supported_settings_test2")] assert model2.supported_settings is not None - model2.supported_settings['delta'].typ = SettingType.INT + model2.supported_settings["delta"].typ = SettingType.INT with pytest.raises(RuntimeError) as e: config_consistent_settings.check_consistent() - assert len(str(e.value).split('\n')) == 2 + assert len(str(e.value).split("\n")) == 2 def test_check_consistent_resources(config_consistent_resources: Configuration) -> None: @@ -466,63 +514,63 @@ def test_check_consistent_resources(config_consistent_resources: Configuration) def test_check_inconsistent_resources( - config_inconsistent_resources: Configuration) -> None: + config_inconsistent_resources: Configuration, +) -> None: with pytest.raises(RuntimeError) as e: config_inconsistent_resources.check_consistent() - assert len(str(e.value).split('\n')) == 3 + assert len(str(e.value).split("\n")) == 3 config_inconsistent_resources.check_consistent(False) def test_check_partial_resources( - config_consistent_partial_resources: Configuration) -> None: - config_consistent_partial_resources.check_consistent(True, 'resources_test') + config_consistent_partial_resources: Configuration, +) -> None: + config_consistent_partial_resources.check_consistent(True, "resources_test") with pytest.raises(RuntimeError) as e: - config_consistent_partial_resources.check_consistent(True, 'resources_test2') + config_consistent_partial_resources.check_consistent(True, "resources_test2") - assert len(str(e.value).split('\n')) == 2 + assert len(str(e.value).split("\n")) == 2 def test_get_resources_defined() -> None: """Test that get_resources returns the defined resource for a component.""" - resources = [ - ThreadedResReq(Ref('macro'), 10), - ThreadedResReq(Ref('micro'), 1)] - config = Configuration('test', resources=resources) + resources = [ThreadedResReq(Ref("macro"), 10), ThreadedResReq(Ref("micro"), 1)] + config = Configuration("test", resources=resources) - res = config.get_resources(Ref('macro')) + res = config.get_resources(Ref("macro")) assert isinstance(res, ThreadedResReq) - assert res.name == Ref('macro') + assert res.name == Ref("macro") assert res.threads == 10 def test_get_resources_default() -> None: """Test that get_resources returns a default of 1 thread for a DIRECT component with no resources defined.""" - config = Configuration('test') + config = Configuration("test") - res = config.get_resources(Ref('micro')) + res = config.get_resources(Ref("micro")) assert isinstance(res, ThreadedResReq) - assert res.name == Ref('micro') + assert res.name == Ref("micro") assert res.threads == 1 def test_check_no_resources_for_non_mpi() -> None: """Non-MPI components without resources should pass check_consistent.""" ymmsl_text = ( - 'ymmsl_version: v0.2\n' - 'models:\n' - ' no_resources_test:\n' - ' components:\n' - ' worker:\n' - ' ports: {}\n' - ' description: description\n' - ' implementation: worker_prog\n' - 'programs:\n' - ' worker_prog:\n' - ' script: worker\n' - ) + "ymmsl_version: v0.2\n" + "models:\n" + " no_resources_test:\n" + " components:\n" + " worker:\n" + " ports: {}\n" + " description: description\n" + " implementation: worker_prog\n" + "programs:\n" + " worker_prog:\n" + " script: worker\n" + ) config = load_as(Configuration, ymmsl_text) config.check_consistent() diff --git a/ymmsl/v0_2/tests/test_imports.py b/ymmsl/v0_2/tests/test_imports.py index 731503d..270d267 100644 --- a/ymmsl/v0_2/tests/test_imports.py +++ b/ymmsl/v0_2/tests/test_imports.py @@ -16,73 +16,74 @@ def load_import() -> LoadImport: def test_load_import_statement1(load_import: LoadImport) -> None: - imp = load_import('from nlesc.examples import implementation example') + imp = load_import("from nlesc.examples import implementation example") - assert imp.module == 'nlesc.examples' + assert imp.module == "nlesc.examples" assert imp.kind == ImportKind.IMPLEMENTATION - assert imp.name == Identifier('example') + assert imp.name == Identifier("example") def test_load_import_statement2(load_import: LoadImport) -> None: - imp = load_import('from nlesc import implementation example') + imp = load_import("from nlesc import implementation example") - assert imp.module == 'nlesc' + assert imp.module == "nlesc" assert imp.kind == ImportKind.IMPLEMENTATION - assert imp.name == Identifier('example') + assert imp.name == Identifier("example") def test_save_import_statement() -> None: dumps_import = yatiml.dumps_function( - Identifier, ImportKind, ImportStatement, Reference) - imp = ImportStatement('nlesc.examples', 'implementation', 'example') + Identifier, ImportKind, ImportStatement, Reference + ) + imp = ImportStatement("nlesc.examples", "implementation", "example") text = dumps_import(imp) - assert text == 'from nlesc.examples import implementation example\n...\n' + assert text == "from nlesc.examples import implementation example\n...\n" def test_load_invalid_import1(load_import: LoadImport) -> None: with pytest.raises(RuntimeError): - load_import('frim nlesc.examples import implementation example') + load_import("frim nlesc.examples import implementation example") def test_load_invalid_import2(load_import: LoadImport) -> None: with pytest.raises(RuntimeError): - load_import('from nlesc,examples import implementation example') + load_import("from nlesc,examples import implementation example") def test_load_invalid_import3(load_import: LoadImport) -> None: with pytest.raises(RuntimeError): - load_import('from nlesc.examples ompirt implementation example') + load_import("from nlesc.examples ompirt implementation example") def test_load_invalid_import4(load_import: LoadImport) -> None: with pytest.raises(RuntimeError): - load_import('from nlesc.examples import impelmentation example') + load_import("from nlesc.examples import impelmentation example") def test_load_invalid_import5(load_import: LoadImport) -> None: with pytest.raises(RuntimeError): - load_import('from nlesc.examples import implementation 123example') + load_import("from nlesc.examples import implementation 123example") def test_load_invalid_import6(load_import: LoadImport) -> None: with pytest.raises(RuntimeError): - load_import('from nlesc.examples[1] import implementation example') + load_import("from nlesc.examples[1] import implementation example") def test_load_invalid_import7(load_import: LoadImport) -> None: with pytest.raises(RuntimeError): - load_import('from a.b import example') + load_import("from a.b import example") def test_load_invalid_import8(load_import: LoadImport) -> None: with pytest.raises(RuntimeError): - load_import('from a.b import implementation') + load_import("from a.b import implementation") def test_module_path() -> None: - imp = ImportStatement('nlesc.examples', 'implementation', 'test') - assert imp.module_path() == Path('nlesc/examples.ymmsl') + imp = ImportStatement("nlesc.examples", "implementation", "test") + assert imp.module_path() == Path("nlesc/examples.ymmsl") - imp = ImportStatement('nlesc', 'implementation', 'test') - assert imp.module_path() == Path('nlesc.ymmsl') + imp = ImportStatement("nlesc", "implementation", "test") + assert imp.module_path() == Path("nlesc.ymmsl") diff --git a/ymmsl/v0_2/tests/test_model.py b/ymmsl/v0_2/tests/test_model.py index 8772d15..d4944a1 100644 --- a/ymmsl/v0_2/tests/test_model.py +++ b/ymmsl/v0_2/tests/test_model.py @@ -22,18 +22,18 @@ def test_conduit_filter() -> None: def test_conduit_access() -> None: - conduit = Conduit('macro.out', 'micro.in') + conduit = Conduit("macro.out", "micro.in") - assert str(conduit) == 'Conduit(macro.out -> micro.in)' - assert conduit == Conduit('macro.out', 'micro.in') - assert conduit != Conduit('micro.in', 'macro.out') + assert str(conduit) == "Conduit(macro.out -> micro.in)" + assert conduit == Conduit("macro.out", "micro.in") + assert conduit != Conduit("micro.in", "macro.out") - assert conduit.sending_component() == 'macro' - assert conduit.sending_port() == 'out' - assert conduit.receiving_component() == 'micro' - assert conduit.receiving_port() == 'in' + assert conduit.sending_component() == "macro" + assert conduit.sending_port() == "out" + assert conduit.receiving_component() == "micro" + assert conduit.receiving_port() == "in" - conduit = Conduit('out', 'in') + conduit = Conduit("out", "in") assert conduit.sending_component() == Reference([]) assert conduit.receiving_component() == Reference([]) @@ -41,52 +41,38 @@ def test_conduit_access() -> None: def test_load_plain_conduit() -> None: load = yatiml.load_function(Conduit, ConduitFilter) - text = ( - 'sender: macro.out\n' - 'receiver: micro.in\n' - ) + text = "sender: macro.out\nreceiver: micro.in\n" conduit = load(text) - assert conduit.sender == 'macro.out' - assert conduit.receiver == 'micro.in' + assert conduit.sender == "macro.out" + assert conduit.receiver == "micro.in" def test_load_model_conduit() -> None: load = yatiml.load_function(Conduit, ConduitFilter) - text = ( - 'sender: out\n' - 'receiver: in\n' - ) + text = "sender: out\nreceiver: in\n" conduit = load(text) - assert conduit.sender == 'out' - assert conduit.receiver == 'in' + assert conduit.sender == "out" + assert conduit.receiver == "in" def test_load_filtered_conduit() -> None: load = yatiml.load_function(Conduit, ConduitFilter) - text = ( - 'sender: micro1.final\n' - 'receiver: micro2.init\n' - 'filters: last pad\n' - ) + text = "sender: micro1.final\nreceiver: micro2.init\nfilters: last pad\n" conduit = load(text) - assert conduit.sender == 'micro1.final' - assert conduit.receiver == 'micro2.init' + assert conduit.sender == "micro1.final" + assert conduit.receiver == "micro2.init" assert conduit.filters == [ConduitFilter.LAST, ConduitFilter.PAD] def test_load_invalid_filter() -> None: load = yatiml.load_function(Conduit, ConduitFilter) - text = ( - 'sender: micro1.final\n' - 'receiver: micro2.init\n' - 'filters: convert-data-format\n' - ) + text = "sender: micro1.final\nreceiver: micro2.init\nfilters: convert-data-format\n" with pytest.raises(yatiml.RecognitionError): load(text) @@ -94,54 +80,43 @@ def test_load_invalid_filter() -> None: def test_dump_conduit() -> None: dumps = yatiml.dumps_function(Conduit, ConduitFilter, Reference) - conduit = Conduit('c1.out', 'c2.in', [ConduitFilter.LAST, ConduitFilter.REPEAT]) + conduit = Conduit("c1.out", "c2.in", [ConduitFilter.LAST, ConduitFilter.REPEAT]) text = dumps(conduit) - assert text == ( - 'sender: c1.out\n' - 'receiver: last repeat c2.in\n') + assert text == ("sender: c1.out\nreceiver: last repeat c2.in\n") def test_multicast_conduits() -> None: - c1 = Conduit('macro.out', 'micro.init') - mc1 = MulticastConduit('micro.final', ['macro.in', 'micro2.init']) - m = Model('test_model', None, 'description', None, [], [c1, mc1]) + c1 = Conduit("macro.out", "micro.init") + mc1 = MulticastConduit("micro.final", ["macro.in", "micro2.init"]) + m = Model("test_model", None, "description", None, [], [c1, mc1]) assert m.conduits[0] is c1 - assert m.conduits[1].sender == 'micro.final' - assert m.conduits[1].receiver == 'macro.in' - assert m.conduits[2].sender == 'micro.final' - assert m.conduits[2].receiver == 'micro2.init' + assert m.conduits[1].sender == "micro.final" + assert m.conduits[1].receiver == "macro.in" + assert m.conduits[2].sender == "micro.final" + assert m.conduits[2].receiver == "micro2.init" def test_load_multicast_conduits() -> None: load = yatiml.load_function(MulticastConduit) - text = ( - 'sender: init.out\n' - 'receiver:\n' - '- c1.in\n' - '- repeat pad c2.in\n' - ) + text = "sender: init.out\nreceiver:\n- c1.in\n- repeat pad c2.in\n" conduit = load(text) - assert conduit.sender == 'init.out' - assert conduit.receiver == ['c1.in', 'repeat pad c2.in'] + assert conduit.sender == "init.out" + assert conduit.receiver == ["c1.in", "repeat pad c2.in"] assert conduit.as_conduits() == [ - Conduit('init.out', 'c1.in'), - Conduit('init.out', 'c2.in', [ConduitFilter.REPEAT, ConduitFilter.PAD])] + Conduit("init.out", "c1.in"), + Conduit("init.out", "c2.in", [ConduitFilter.REPEAT, ConduitFilter.PAD]), + ] def test_load_multicast_invalid_filter() -> None: load = yatiml.load_function(MulticastConduit) - text = ( - 'sender: init.out\n' - 'receiver:\n' - '- c1.in\n' - '- convert_data_format c2.in\n' - ) + text = "sender: init.out\nreceiver:\n- c1.in\n- convert_data_format c2.in\n" mc_conduit = load(text) with pytest.raises(RuntimeError): @@ -151,96 +126,129 @@ def test_load_multicast_invalid_filter() -> None: def test_dump_multicast_conduits() -> None: dump = yatiml.dumps_function(MulticastConduit) - conduit = MulticastConduit('init.out', ['c1.in', 'repeat pad c2.in']) + conduit = MulticastConduit("init.out", ["c1.in", "repeat pad c2.in"]) text = dump(conduit) - assert text == ( - 'sender: init.out\n' - 'receiver:\n' - '- c1.in\n' - '- repeat pad c2.in\n') + assert text == ("sender: init.out\nreceiver:\n- c1.in\n- repeat pad c2.in\n") def test_load_model(model_text: str) -> None: load_model = yatiml.load_function( - Model, Component, Conduit, ConduitFilter, Identifier, MulticastConduit, - Ports, Reference, SettingType, SupportedSetting, SupportedSettings) + Model, + Component, + Conduit, + ConduitFilter, + Identifier, + MulticastConduit, + Ports, + Reference, + SettingType, + SupportedSetting, + SupportedSettings, + ) m = load_model(model_text) - assert m.name == 'test_model' + assert m.name == "test_model" assert m.ports is not None - assert m.ports['in'] == Port(Identifier('in'), Operator.F_INIT, Timeline('')) - assert m.ports['out'] == Port(Identifier('out'), Operator.O_F, Timeline('')) - assert m.description == 'Test model for loading/dumping\n' + assert m.ports["in"] == Port(Identifier("in"), Operator.F_INIT, Timeline("")) + assert m.ports["out"] == Port(Identifier("out"), Operator.O_F, Timeline("")) + assert m.description == "Test model for loading/dumping\n" assert m.supported_settings is not None - assert m.supported_settings['eta'].typ == SettingType.FLOAT - assert m.components[Ref('ic')].name == Reference('ic') - assert m.components[Ref('smc')].name == Reference('smc') - assert m.components[Ref('bf')].name == Reference('bf') - assert m.components[Ref('smc2bf')].name == Reference('smc2bf') - assert m.components[Ref('bf2smc')].name == Reference('bf2smc') - assert m.conduits[0].sender == Reference('ic.out') - assert m.conduits[1].sender == Reference('smc.cell_positions') - assert m.conduits[2].receiver == Reference('bf.initial_domain') - assert m.conduits[3].receiver == Reference('bf2smc.in') - assert m.conduits[4].sender == Reference('bf2smc.out') + assert m.supported_settings["eta"].typ == SettingType.FLOAT + assert m.components[Ref("ic")].name == Reference("ic") + assert m.components[Ref("smc")].name == Reference("smc") + assert m.components[Ref("bf")].name == Reference("bf") + assert m.components[Ref("smc2bf")].name == Reference("smc2bf") + assert m.components[Ref("bf2smc")].name == Reference("bf2smc") + assert m.conduits[0].sender == Reference("ic.out") + assert m.conduits[1].sender == Reference("smc.cell_positions") + assert m.conduits[2].receiver == Reference("bf.initial_domain") + assert m.conduits[3].receiver == Reference("bf2smc.in") + assert m.conduits[4].sender == Reference("bf2smc.out") def test_load_model_with_multicast_conduits(model_multicast_text: str) -> None: load_model = yatiml.load_function( - Model, Component, Conduit, ConduitFilter, Identifier, MulticastConduit, - Ports, Reference, SettingType, SupportedSettings) + Model, + Component, + Conduit, + ConduitFilter, + Identifier, + MulticastConduit, + Ports, + Reference, + SettingType, + SupportedSettings, + ) m = load_model(model_multicast_text) - assert m.conduits[0].sender == 'a.out' - assert m.conduits[0].receiver == 'b.in' - assert m.conduits[1].sender == 'a.out' - assert m.conduits[1].receiver == 'c.in' + assert m.conduits[0].sender == "a.out" + assert m.conduits[0].receiver == "b.in" + assert m.conduits[1].sender == "a.out" + assert m.conduits[1].receiver == "c.in" def test_load_model_with_filters(model_with_filters_text: str) -> None: load_model = yatiml.load_function( - Model, Component, Conduit, ConduitFilter, Identifier, MulticastConduit, - Ports, Reference, SettingType, SupportedSettings) + Model, + Component, + Conduit, + ConduitFilter, + Identifier, + MulticastConduit, + Ports, + Reference, + SettingType, + SupportedSettings, + ) m = load_model(model_with_filters_text) - assert m.name == 'test_model_conduit_filters' + assert m.name == "test_model_conduit_filters" assert m.ports is not None assert len(m.ports) == 0 - assert m.description == 'Test model for loading/dumping conduit filters\n' + assert m.description == "Test model for loading/dumping conduit filters\n" assert m.supported_settings is None - assert m.components[Ref('init')].name == Reference('init') - assert m.components[Ref('macro1')].name == Reference('macro1') - assert m.components[Ref('micro1')].name == Reference('micro1') - assert m.components[Ref('macro2')].name == Reference('macro2') - assert m.components[Ref('micro2')].name == Reference('micro2') - - assert m.conduits[0].sender == Reference('init.macro_out') - assert m.conduits[0].receiver == Reference('macro1.init') + assert m.components[Ref("init")].name == Reference("init") + assert m.components[Ref("macro1")].name == Reference("macro1") + assert m.components[Ref("micro1")].name == Reference("micro1") + assert m.components[Ref("macro2")].name == Reference("macro2") + assert m.components[Ref("micro2")].name == Reference("micro2") + + assert m.conduits[0].sender == Reference("init.macro_out") + assert m.conduits[0].receiver == Reference("macro1.init") assert m.conduits[0].filters == [] - assert m.conduits[1].sender == Reference('init.micro_out') - assert m.conduits[1].receiver == Reference('micro1.init_state') + assert m.conduits[1].sender == Reference("init.micro_out") + assert m.conduits[1].receiver == Reference("micro1.init_state") assert m.conduits[1].filters == [ConduitFilter.PAD] - assert m.conduits[5].sender == Reference('micro1.final_state') - assert m.conduits[5].receiver == Reference('micro2.init_state') + assert m.conduits[5].sender == Reference("micro1.final_state") + assert m.conduits[5].receiver == Reference("micro2.init_state") assert m.conduits[5].filters == [ConduitFilter.LAST, ConduitFilter.PAD] def test_load_model_with_invalid_filters() -> None: load_model = yatiml.load_function( - Model, Component, Conduit, ConduitFilter, Identifier, MulticastConduit, - Ports, Reference, SettingType, SupportedSettings) + Model, + Component, + Conduit, + ConduitFilter, + Identifier, + MulticastConduit, + Ports, + Reference, + SettingType, + SupportedSettings, + ) text = ( - 'name: test_model_with_invalid_filters\n' - 'description: Testing invalid filters\n' - 'conduits:\n' - ' init.micro_out: invalid-filter micro1.init_state\n' - ) + "name: test_model_with_invalid_filters\n" + "description: Testing invalid filters\n" + "conduits:\n" + " init.micro_out: invalid-filter micro1.init_state\n" + ) with pytest.raises(yatiml.RecognitionError): load_model(text) @@ -248,75 +256,110 @@ def test_load_model_with_invalid_filters() -> None: def test_dump_model(model: Model, model_text: str) -> None: dumps_model = yatiml.dumps_function( - Model, Component, Conduit, Identifier, Implementation, MulticastConduit, - Ports, Reference, SettingType, SupportedSetting, SupportedSettings) + Model, + Component, + Conduit, + Identifier, + Implementation, + MulticastConduit, + Ports, + Reference, + SettingType, + SupportedSetting, + SupportedSettings, + ) text = dumps_model(model) assert text == model_text def test_dump_model_with_multicast_conduits( - model_multicast: Model, model_multicast_text: str) -> None: + model_multicast: Model, model_multicast_text: str +) -> None: dumps_model = yatiml.dumps_function( - Model, Component, Conduit, Identifier, Implementation, MulticastConduit, - Ports, Reference, SettingType, SupportedSettings) + Model, + Component, + Conduit, + Identifier, + Implementation, + MulticastConduit, + Ports, + Reference, + SettingType, + SupportedSettings, + ) text = dumps_model(model_multicast) assert text == model_multicast_text def test_dump_model_with_filters( - model_with_filters: Model, model_with_filters_text: str) -> None: + model_with_filters: Model, model_with_filters_text: str +) -> None: dumps_model = yatiml.dumps_function( - Model, Component, Conduit, ConduitFilter, Identifier, Implementation, - MulticastConduit, Ports, Reference, SettingType, SupportedSettings) + Model, + Component, + Conduit, + ConduitFilter, + Identifier, + Implementation, + MulticastConduit, + Ports, + Reference, + SettingType, + SupportedSettings, + ) text = dumps_model(model_with_filters) assert text == model_with_filters_text def test_consistent() -> None: - model_ports = Ports(f_init=['model_init'], o_f=['model_final']) + model_ports = Ports(f_init=["model_init"], o_f=["model_final"]) - macro_ports = Ports(f_init=['Minit'], o_i=['Mout'], s=['Min'], o_f=['Mfinal']) - macro = Component('macro', macro_ports, 'macro_impl') + macro_ports = Ports(f_init=["Minit"], o_i=["Mout"], s=["Min"], o_f=["Mfinal"]) + macro = Component("macro", macro_ports, "macro_impl") - micro_ports = Ports(f_init=['minit'], o_f=['mfinal']) - micro = Component('micro', micro_ports, 'micro_impl') + micro_ports = Ports(f_init=["minit"], o_f=["mfinal"]) + micro = Component("micro", micro_ports, "micro_impl") conduits = [ - Conduit('model_init', 'macro.Minit'), - Conduit('macro.Mout', 'micro.minit'), - Conduit('micro.mfinal', 'macro.Min'), - Conduit('macro.Mfinal', 'model_final')] + Conduit("model_init", "macro.Minit"), + Conduit("macro.Mout", "micro.minit"), + Conduit("micro.mfinal", "macro.Min"), + Conduit("macro.Mfinal", "model_final"), + ] model = Model( - 'with_conduits', model_ports, 'description', None, [macro, micro], conduits) + "with_conduits", model_ports, "description", None, [macro, micro], conduits + ) errors = model.check_consistent() assert not errors def test_conduits_inconsistent() -> None: - model_ports = Ports(f_init=['model_init'], o_f=['model_final']) + model_ports = Ports(f_init=["model_init"], o_f=["model_final"]) - macro_ports = Ports(f_init=['Minit'], o_i=['Mout'], s=['Min'], o_f=['Mfinal']) - macro = Component('macro', macro_ports, 'macro_impl') + macro_ports = Ports(f_init=["Minit"], o_i=["Mout"], s=["Min"], o_f=["Mfinal"]) + macro = Component("macro", macro_ports, "macro_impl") - micro_ports = Ports(f_init=['minit'], o_f=['mfinal']) - micro = Component('micro', micro_ports, 'micro_impl') + micro_ports = Ports(f_init=["minit"], o_f=["mfinal"]) + micro = Component("micro", micro_ports, "micro_impl") conduits = [ - Conduit('modelinit', 'macro.Minit'), - Conduit('macro.Mout', 'micro.m_init'), - Conduit('macro.Mout', 'miicro.minit'), - Conduit('macroo.Mout', 'micro.minit'), - Conduit('micro.m_final', 'macro.Min'), - Conduit('macro.Mfinal', 'modelfinal')] + Conduit("modelinit", "macro.Minit"), + Conduit("macro.Mout", "micro.m_init"), + Conduit("macro.Mout", "miicro.minit"), + Conduit("macroo.Mout", "micro.minit"), + Conduit("micro.m_final", "macro.Min"), + Conduit("macro.Mfinal", "modelfinal"), + ] model = Model( - 'bad_conduits', model_ports, 'description', None, [macro, micro], conduits) + "bad_conduits", model_ports, "description", None, [macro, micro], conduits + ) errors = model.check_consistent() assert len(errors) == 6 diff --git a/ymmsl/v0_2/tests/test_ports.py b/ymmsl/v0_2/tests/test_ports.py index cfb791c..438358b 100644 --- a/ymmsl/v0_2/tests/test_ports.py +++ b/ymmsl/v0_2/tests/test_ports.py @@ -8,61 +8,61 @@ def test_create_empty_timeline() -> None: - tl = Timeline('') + tl = Timeline("") assert tl.absolute is False assert len(tl._parts) == 0 def test_create_root_timeline() -> None: - tl = Timeline(':') + tl = Timeline(":") assert tl.absolute is True assert len(tl._parts) == 0 def test_create_absolute_timeline() -> None: - tl = Timeline(':timeline') + tl = Timeline(":timeline") assert tl.absolute is True assert len(tl._parts) == 1 - assert tl._parts[0] == 'timeline' + assert tl._parts[0] == "timeline" def test_create_relative_timeline() -> None: - tl = Timeline('timeline') + tl = Timeline("timeline") assert tl.absolute is False assert len(tl._parts) == 1 - assert tl._parts[0] == 'timeline' + assert tl._parts[0] == "timeline" def test_create_full_timeline() -> None: - tl = Timeline('c1.a:c2.b:c3:c4') + tl = Timeline("c1.a:c2.b:c3:c4") assert tl.absolute is False assert len(tl._parts) == 4 - assert tl._parts == ['c1.a', 'c2.b', 'c3', 'c4'] + assert tl._parts == ["c1.a", "c2.b", "c3", "c4"] def test_create_absolute_from_list_of_str() -> None: - tl = Timeline(['c1.a', 'c2.b', 'c3', 'c4'], True) + tl = Timeline(["c1.a", "c2.b", "c3", "c4"], True) assert tl.absolute is True assert len(tl._parts) == 4 - assert tl._parts == ['c1.a', 'c2.b', 'c3', 'c4'] + assert tl._parts == ["c1.a", "c2.b", "c3", "c4"] def test_create_relative_from_list_of_ref() -> None: - tl = Timeline([Ref('c1'), Ref('c2.a')], False) + tl = Timeline([Ref("c1"), Ref("c2.a")], False) assert tl.absolute is False assert len(tl._parts) == 2 - assert tl._parts == ['c1', 'c2.a'] + assert tl._parts == ["c1", "c2.a"] def test_timeline_equality() -> None: - tl1 = Timeline(':c0:c1:c2.a') - tl2 = Timeline(['c0', 'c1', 'c2.a'], True) + tl1 = Timeline(":c0:c1:c2.a") + tl2 = Timeline(["c0", "c1", "c2.a"], True) assert tl1 == tl2 - tl2._parts[1] = Reference('c1.x') + tl2._parts[1] = Reference("c1.x") assert tl1 != tl2 - tl1._parts[1] = Reference('c1.x') + tl1._parts[1] = Reference("c1.x") assert tl1 == tl2 tl2.absolute = False @@ -76,37 +76,37 @@ def test_timeline_equality() -> None: def test_timeline_to_str() -> None: - tl = Timeline([Ref('c1'), Ref('c2.a'), Ref('c3')], False) - assert str(tl) == 'c1:c2.a:c3' + tl = Timeline([Ref("c1"), Ref("c2.a"), Ref("c3")], False) + assert str(tl) == "c1:c2.a:c3" tl.absolute = True - assert str(tl) == ':c1:c2.a:c3' + assert str(tl) == ":c1:c2.a:c3" def test_timeline_indexing() -> None: - tl = Timeline(':c0:c1:c2.p:c3.q') + tl = Timeline(":c0:c1:c2.p:c3.q") assert len(tl) == 4 assert isinstance(tl[0], Reference) - assert tl[0] == 'c0' + assert tl[0] == "c0" assert isinstance(tl[1], Reference) - assert tl[1] == 'c1' + assert tl[1] == "c1" assert isinstance(tl[2], Reference) - assert tl[2] == 'c2.p' + assert tl[2] == "c2.p" assert isinstance(tl[3], Reference) - assert tl[3] == 'c3.q' + assert tl[3] == "c3.q" with pytest.raises(IndexError): tl[4] def test_timeline_concatenation() -> None: - tl1 = Timeline(':c1:c2.a') - tl2 = Timeline('c3.b:c4') + tl1 = Timeline(":c1:c2.a") + tl2 = Timeline("c3.b:c4") tl3 = tl1 + tl2 assert tl3.absolute is True assert len(tl3._parts) == 4 - assert tl3 == Timeline(':c1:c2.a:c3.b:c4') + assert tl3 == Timeline(":c1:c2.a:c3.b:c4") tl2.absolute = True @@ -115,8 +115,8 @@ def test_timeline_concatenation() -> None: def test_timeline_concatenate_empty() -> None: - tl1 = Timeline('c1:c2.a') - tl2 = Timeline('') + tl1 = Timeline("c1:c2.a") + tl2 = Timeline("") tl3 = tl1 + tl2 assert tl3 == tl1 @@ -131,57 +131,60 @@ def test_create_empty_ports() -> None: def test_create_ports_by_operator() -> None: - p = Ports('a', None, 'b c', ['d', 'e']) + p = Ports("a", None, "b c", ["d", "e"]) assert p._ports == { - 'a': Port(Identifier('a'), Operator.F_INIT, Timeline('')), - 'b': Port(Identifier('b'), Operator.S, Timeline('')), - 'c': Port(Identifier('c'), Operator.S, Timeline('')), - 'd': Port(Identifier('d'), Operator.O_F, Timeline('')), - 'e': Port(Identifier('e'), Operator.O_F, Timeline(''))} + "a": Port(Identifier("a"), Operator.F_INIT, Timeline("")), + "b": Port(Identifier("b"), Operator.S, Timeline("")), + "c": Port(Identifier("c"), Operator.S, Timeline("")), + "d": Port(Identifier("d"), Operator.O_F, Timeline("")), + "e": Port(Identifier("e"), Operator.O_F, Timeline("")), + } def test_create_ports_from_list() -> None: - t1 = Timeline('timeline1') - t2 = Timeline('timeline2') - - p = Ports([ - Port(Identifier('a'), Operator.F_INIT), - Port(Identifier('b'), Operator.O_F), - Port(Identifier('c'), Operator.O_I, t1), - Port(Identifier('d'), Operator.S, t1), - Port(Identifier('e'), Operator.O_I, t2), - Port(Identifier('f'), Operator.S, t2) - ]) + t1 = Timeline("timeline1") + t2 = Timeline("timeline2") + + p = Ports( + [ + Port(Identifier("a"), Operator.F_INIT), + Port(Identifier("b"), Operator.O_F), + Port(Identifier("c"), Operator.O_I, t1), + Port(Identifier("d"), Operator.S, t1), + Port(Identifier("e"), Operator.O_I, t2), + Port(Identifier("f"), Operator.S, t2), + ] + ) assert p._ports == { - 'a': Port(Identifier('a'), Operator.F_INIT), - 'b': Port(Identifier('b'), Operator.O_F), - 'c': Port(Identifier('c'), Operator.O_I, t1), - 'd': Port(Identifier('d'), Operator.S, t1), - 'e': Port(Identifier('e'), Operator.O_I, t2), - 'f': Port(Identifier('f'), Operator.S, t2) - } + "a": Port(Identifier("a"), Operator.F_INIT), + "b": Port(Identifier("b"), Operator.O_F), + "c": Port(Identifier("c"), Operator.O_I, t1), + "d": Port(Identifier("d"), Operator.S, t1), + "e": Port(Identifier("e"), Operator.O_I, t2), + "f": Port(Identifier("f"), Operator.S, t2), + } def test_ports_access() -> None: - p = Ports('a', None, 'b c', ['d', 'e']) + p = Ports("a", None, "b c", ["d", "e"]) - assert 'a' in p - assert Identifier('a') in p - assert 'z' not in p - assert Identifier('q') not in p + assert "a" in p + assert Identifier("a") in p + assert "z" not in p + assert Identifier("q") not in p - assert p['a'] == Port(Identifier('a'), Operator.F_INIT) - assert p['b'] == Port(Identifier('b'), Operator.S) - assert p[Identifier('c')] == Port(Identifier('c'), Operator.S) - assert p['d'] == Port(Identifier('d'), Operator.O_F) - assert p[Identifier('e')] == Port(Identifier('e'), Operator.O_F) + assert p["a"] == Port(Identifier("a"), Operator.F_INIT) + assert p["b"] == Port(Identifier("b"), Operator.S) + assert p[Identifier("c")] == Port(Identifier("c"), Operator.S) + assert p["d"] == Port(Identifier("d"), Operator.O_F) + assert p[Identifier("e")] == Port(Identifier("e"), Operator.O_F) def test_ports_iteration() -> None: - p = Ports('p', 'q r', ['s', 't'], 'u v') - names = 'pqrstuv' + p = Ports("p", "q r", ["s", "t"], "u v") + names = "pqrstuv" for port_name, ref in zip(p, names): assert port_name == ref @@ -190,61 +193,50 @@ def test_ports_iteration() -> None: def test_load_ports_simple() -> None: load = yatiml.load_function(Ports, Port, Operator, Timeline) - text = ( - 'f_init:\n' - '- a\n' - '- b\n' - 'o_i: c\n' - 'o_f: d e\n' - ) + text = "f_init:\n- a\n- b\no_i: c\no_f: d e\n" p = load(text) assert p._ports == { - 'a': Port(Identifier('a'), Operator.F_INIT), - 'b': Port(Identifier('b'), Operator.F_INIT), - 'c': Port(Identifier('c'), Operator.O_I), - 'd': Port(Identifier('d'), Operator.O_F), - 'e': Port(Identifier('e'), Operator.O_F) - } + "a": Port(Identifier("a"), Operator.F_INIT), + "b": Port(Identifier("b"), Operator.F_INIT), + "c": Port(Identifier("c"), Operator.O_I), + "d": Port(Identifier("d"), Operator.O_F), + "e": Port(Identifier("e"), Operator.O_F), + } def test_load_ports_with_timelines() -> None: load = yatiml.load_function(Ports, Port, Operator, Timeline) text = ( - 'f_init: a b\n' - 'o_f: c\n' - '+timeline1:\n' - ' o_i: d\n' - ' s: e f\n' - '+timeline2:\n' - ' o_i: g h\n' - ) + "f_init: a b\n" + "o_f: c\n" + "+timeline1:\n" + " o_i: d\n" + " s: e f\n" + "+timeline2:\n" + " o_i: g h\n" + ) p = load(text) assert p._ports == { - 'a': Port(Identifier('a'), Operator.F_INIT), - 'b': Port(Identifier('b'), Operator.F_INIT), - 'c': Port(Identifier('c'), Operator.O_F), - 'd': Port(Identifier('d'), Operator.O_I, Timeline('timeline1')), - 'e': Port(Identifier('e'), Operator.S, Timeline('timeline1')), - 'f': Port(Identifier('f'), Operator.S, Timeline('timeline1')), - 'g': Port(Identifier('g'), Operator.O_I, Timeline('timeline2')), - 'h': Port(Identifier('h'), Operator.O_I, Timeline('timeline2')) - } + "a": Port(Identifier("a"), Operator.F_INIT), + "b": Port(Identifier("b"), Operator.F_INIT), + "c": Port(Identifier("c"), Operator.O_F), + "d": Port(Identifier("d"), Operator.O_I, Timeline("timeline1")), + "e": Port(Identifier("e"), Operator.S, Timeline("timeline1")), + "f": Port(Identifier("f"), Operator.S, Timeline("timeline1")), + "g": Port(Identifier("g"), Operator.O_I, Timeline("timeline2")), + "h": Port(Identifier("h"), Operator.O_I, Timeline("timeline2")), + } def test_load_ports_invalid_operator() -> None: load = yatiml.load_function(Ports, Port, Operator, Timeline) - text = ( - 'f_init: a b\n' - 'o_i: d\n' - 's: e f\n' - 'o_g: g h\n' - ) + text = "f_init: a b\no_i: d\ns: e f\no_g: g h\n" with pytest.raises(yatiml.RecognitionError): load(text) @@ -253,14 +245,7 @@ def test_load_ports_invalid_operator() -> None: def test_load_ports_invalid_timeline() -> None: load = yatiml.load_function(Ports, Port, Operator, Timeline) - text = ( - 'f_init: a b\n' - '+timeline1:\n' - ' o_i: d\n' - ' s: e f\n' - 'timeline2:\n' - ' o_i: g h\n' - ) + text = "f_init: a b\n+timeline1:\n o_i: d\n s: e f\ntimeline2:\n o_i: g h\n" with pytest.raises(yatiml.RecognitionError): load(text) @@ -270,15 +255,15 @@ def test_load_ports_invalid_timeline_operator() -> None: load = yatiml.load_function(Ports, Port, Operator, Timeline) text = ( - 'f_init: a b\n' - '+timeline1:\n' - ' o_i: d\n' - ' s: e f\n' - '+timeline2:\n' - ' o_i: g h\n' - ' +timeline3:\n' - ' o_i: i\n' - ) + "f_init: a b\n" + "+timeline1:\n" + " o_i: d\n" + " s: e f\n" + "+timeline2:\n" + " o_i: g h\n" + " +timeline3:\n" + " o_i: i\n" + ) with pytest.raises(yatiml.RecognitionError): load(text) @@ -287,69 +272,72 @@ def test_load_ports_invalid_timeline_operator() -> None: def test_dump_simple_ports() -> None: dumps = yatiml.dumps_function(Ports, Port, Operator, Timeline) - p = Ports(['init'], ['out'], 'in', 'final') + p = Ports(["init"], ["out"], "in", "final") text = dumps(p) - assert text == ( - 'f_init: init\n' - 'o_i: out\n' - 's: in\n' - 'o_f: final\n' - ) + assert text == ("f_init: init\no_i: out\ns: in\no_f: final\n") def test_dump_long_ports() -> None: dumps = yatiml.dumps_function(Ports, Port, Operator, Timeline) p = Ports( - 'a b c d e f', None, None, [ - 'port_with_a_long_name', 'another_one_like_that', - 'this_is_really_going_to_cause_trouble']) + "a b c d e f", + None, + None, + [ + "port_with_a_long_name", + "another_one_like_that", + "this_is_really_going_to_cause_trouble", + ], + ) text = dumps(p) assert text == ( - 'f_init:\n' - '- a\n' - '- b\n' - '- c\n' - '- d\n' - '- e\n' - '- f\n' - 'o_f:\n' - '- port_with_a_long_name\n' - '- another_one_like_that\n' - '- this_is_really_going_to_cause_trouble\n' - ) + "f_init:\n" + "- a\n" + "- b\n" + "- c\n" + "- d\n" + "- e\n" + "- f\n" + "o_f:\n" + "- port_with_a_long_name\n" + "- another_one_like_that\n" + "- this_is_really_going_to_cause_trouble\n" + ) def test_dump_ports_with_timelines() -> None: dumps = yatiml.dumps_function(Ports, Port, Operator, Timeline) - p = Ports([ - Port(Identifier('init'), Operator.F_INIT), - Port(Identifier('out1'), Operator.O_I, Timeline('timeline1')), - Port(Identifier('out2'), Operator.O_I, Timeline('timeline2')), - Port(Identifier('in1'), Operator.S, Timeline('timeline1')), - Port(Identifier('in2'), Operator.S, Timeline('timeline2')), - Port(Identifier('final'), Operator.O_F) - ]) + p = Ports( + [ + Port(Identifier("init"), Operator.F_INIT), + Port(Identifier("out1"), Operator.O_I, Timeline("timeline1")), + Port(Identifier("out2"), Operator.O_I, Timeline("timeline2")), + Port(Identifier("in1"), Operator.S, Timeline("timeline1")), + Port(Identifier("in2"), Operator.S, Timeline("timeline2")), + Port(Identifier("final"), Operator.O_F), + ] + ) text = dumps(p) assert text == ( - 'f_init: init\n' - '+timeline1:\n' - ' o_i: out1\n' - ' s: in1\n' - '+timeline2:\n' - ' o_i: out2\n' - ' s: in2\n' - 'o_f: final\n' - ) + "f_init: init\n" + "+timeline1:\n" + " o_i: out1\n" + " s: in1\n" + "+timeline2:\n" + " o_i: out2\n" + " s: in2\n" + "o_f: final\n" + ) def test_sending_receiving_port_names() -> None: - p = Ports('a', 'b c', 'd e', 'f') + p = Ports("a", "b c", "d e", "f") - assert p.sending_port_names() == ['b', 'c', 'f'] - assert p.receiving_port_names() == ['a', 'd', 'e'] + assert p.sending_port_names() == ["b", "c", "f"] + assert p.receiving_port_names() == ["a", "d", "e"] diff --git a/ymmsl/v0_2/tests/test_program.py b/ymmsl/v0_2/tests/test_program.py index b85ac9d..71bc24b 100644 --- a/ymmsl/v0_2/tests/test_program.py +++ b/ymmsl/v0_2/tests/test_program.py @@ -17,77 +17,72 @@ def test_program_script_list() -> None: prog = Program( - name='test_prog', - ports=Ports(f_init='init', s='bc_in'), - script=[ - '#!/bin/bash', - '', - 'test_prog']) + name="test_prog", + ports=Ports(f_init="init", s="bc_in"), + script=["#!/bin/bash", "", "test_prog"], + ) assert isinstance(prog.name, Reference) - assert prog.name == 'test_prog' + assert prog.name == "test_prog" assert len(prog.ports) == 2 - assert prog.ports['init'].operator == Operator.F_INIT - assert prog.ports['bc_in'].operator == Operator.S - assert prog.script == '#!/bin/bash\n\ntest_prog\n' + assert prog.ports["init"].operator == Operator.F_INIT + assert prog.ports["bc_in"].operator == Operator.S + assert prog.script == "#!/bin/bash\n\ntest_prog\n" def test_program_executable() -> None: prog = Program( - name='test_prog', - ports=Ports(o_i=['out'], s=['in']), - description='Description of the program', - supported_settings=SupportedSettings({'a': '[int]', 'b': 'str'}), - base_env=BaseEnv.MANAGER, - modules=['python/3.10.0', 'gcc/13.3.0'], - virtual_env=Path('/home/user/envs/venv'), - env={ - 'VAR1': '1', - 'VAR2': 'Testing'}, - executable=Path('/home/user/software/my_submodel/bin/model'), - args='-v -a', - execution_model=ExecutionModel.OPENMPI, - can_share_resources=False) - - assert prog.name == 'test_prog' + name="test_prog", + ports=Ports(o_i=["out"], s=["in"]), + description="Description of the program", + supported_settings=SupportedSettings({"a": "[int]", "b": "str"}), + base_env=BaseEnv.MANAGER, + modules=["python/3.10.0", "gcc/13.3.0"], + virtual_env=Path("/home/user/envs/venv"), + env={"VAR1": "1", "VAR2": "Testing"}, + executable=Path("/home/user/software/my_submodel/bin/model"), + args="-v -a", + execution_model=ExecutionModel.OPENMPI, + can_share_resources=False, + ) + + assert prog.name == "test_prog" assert len(prog.ports) == 2 - assert prog.ports['out'].operator == Operator.O_I - assert prog.ports['in'].operator == Operator.S - assert prog.description == 'Description of the program' + assert prog.ports["out"].operator == Operator.O_I + assert prog.ports["in"].operator == Operator.S + assert prog.description == "Description of the program" assert prog.supported_settings is not None - assert prog.supported_settings['a'].typ == SettingType.LIST_INT - assert prog.supported_settings['b'].typ == SettingType.STR + assert prog.supported_settings["a"].typ == SettingType.LIST_INT + assert prog.supported_settings["b"].typ == SettingType.STR assert prog.base_env == BaseEnv.MANAGER - assert prog.modules == ['python/3.10.0', 'gcc/13.3.0'] - assert prog.virtual_env == Path('/home/user/envs/venv') + assert prog.modules == ["python/3.10.0", "gcc/13.3.0"] + assert prog.virtual_env == Path("/home/user/envs/venv") assert prog.env is not None - assert prog.env['VAR1'] == '1' - assert prog.env['VAR2'] == 'Testing' - assert prog.executable == Path('/home/user/software/my_submodel/bin/model') - assert prog.args == ['-v -a'] + assert prog.env["VAR1"] == "1" + assert prog.env["VAR2"] == "Testing" + assert prog.executable == Path("/home/user/software/my_submodel/bin/model") + assert prog.args == ["-v -a"] assert prog.execution_model == ExecutionModel.OPENMPI assert prog.can_share_resources is False def test_program_exclusive() -> None: with pytest.raises(RuntimeError): - Program(name='test', script='', executable=Path()) + Program(name="test", script="", executable=Path()) def test_program_script() -> None: - script = ( - '#!/bin/bash\n' - '\n' - 'mpirun my_model\n') + script = "#!/bin/bash\n\nmpirun my_model\n" prog = Program( - name='test_prog', - execution_model=ExecutionModel.OPENMPI, - can_share_resources=False, - keeps_state_for_next_use=KeepsStateForNextUse.HELPFUL, - script=script) - - assert prog.name == 'test_prog' + name="test_prog", + execution_model=ExecutionModel.OPENMPI, + can_share_resources=False, + keeps_state_for_next_use=KeepsStateForNextUse.HELPFUL, + script=script, + ) + + assert prog.name == "test_prog" assert prog.execution_model == ExecutionModel.OPENMPI assert prog.can_share_resources is False assert prog.keeps_state_for_next_use == KeepsStateForNextUse.HELPFUL @@ -97,44 +92,63 @@ def test_program_script() -> None: def test_program_script_invalid_args() -> None: with pytest.raises(RuntimeError): Program( - name='test_prog', - execution_model=ExecutionModel.DIRECT, - env={'TEST': 'NOT_ALLOWED'}, - script='test') + name="test_prog", + execution_model=ExecutionModel.DIRECT, + env={"TEST": "NOT_ALLOWED"}, + script="test", + ) def test_load_program(test_program_text: str) -> None: load = yatiml.load_function( - Program, BaseEnv, ExecutionModel, Identifier, Implementation, - KeepsStateForNextUse, Ports, Reference, SettingType, SupportedSetting, - SupportedSettings) + Program, + BaseEnv, + ExecutionModel, + Identifier, + Implementation, + KeepsStateForNextUse, + Ports, + Reference, + SettingType, + SupportedSetting, + SupportedSettings, + ) prog = load(test_program_text) - assert prog.name == 'macro' - assert prog.ports.sending_port_names() == ['final', 'out1', 'out2'] - assert prog.ports.receiving_port_names() == ['init', 'in1', 'in2'] + assert prog.name == "macro" + assert prog.ports.sending_port_names() == ["final", "out1", "out2"] + assert prog.ports.receiving_port_names() == ["init", "in1", "in2"] assert prog.base_env == BaseEnv.LOGIN assert prog.modules is not None assert len(prog.modules) == 2 - assert prog.modules[0] == 'gcc/13.3.0' - assert prog.modules[1] == 'FFTW/3.2.1' - assert prog.virtual_env == Path('/home/user/.venv') + assert prog.modules[0] == "gcc/13.3.0" + assert prog.modules[1] == "FFTW/3.2.1" + assert prog.virtual_env == Path("/home/user/.venv") assert len(prog.env) == 2 - assert prog.env['SETTING'] == 'something' - assert prog.env['VARIABLE'] == '42' + assert prog.env["SETTING"] == "something" + assert prog.env["VARIABLE"] == "42" assert prog.execution_model == ExecutionModel.INTELMPI - assert prog.executable == Path('python3') - assert prog.args == ['/home/user/script.py'] + assert prog.executable == Path("python3") + assert prog.args == ["/home/user/script.py"] assert prog.can_share_resources is False assert prog.keeps_state_for_next_use == KeepsStateForNextUse.HELPFUL def test_dump_programs(test_program: Program, test_program_text: str) -> None: dump = yatiml.dumps_function( - Program, BaseEnv, ExecutionModel, Identifier, Implementation, - KeepsStateForNextUse, Ports, Reference, SettingType, SupportedSetting, - SupportedSettings) + Program, + BaseEnv, + ExecutionModel, + Identifier, + Implementation, + KeepsStateForNextUse, + Ports, + Reference, + SettingType, + SupportedSetting, + SupportedSettings, + ) text = dump(test_program) assert text == test_program_text diff --git a/ymmsl/v0_2/tests/test_resolver.py b/ymmsl/v0_2/tests/test_resolver.py index 83df282..d39ec0c 100644 --- a/ymmsl/v0_2/tests/test_resolver.py +++ b/ymmsl/v0_2/tests/test_resolver.py @@ -23,494 +23,506 @@ @pytest.fixture def env_ymmsl_path() -> Generator[None, None, None]: cur_dir = Path(__file__).parents[0] - ymmsl1 = cur_dir / 'ymmsl1' - ymmsl2 = cur_dir / 'ymmsl2' - ymmsl3 = cur_dir / 'ymmsl3' # intentionally does not exist + ymmsl1 = cur_dir / "ymmsl1" + ymmsl2 = cur_dir / "ymmsl2" + ymmsl3 = cur_dir / "ymmsl3" # intentionally does not exist - os.environ['YMMSL_PATH'] = f'{ymmsl3}:{ymmsl2}:{ymmsl1}' + os.environ["YMMSL_PATH"] = f"{ymmsl3}:{ymmsl2}:{ymmsl1}" yield - del os.environ['YMMSL_PATH'] + del os.environ["YMMSL_PATH"] def test_resolve_imports(env_ymmsl_path: None) -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: Testing resolving imports\n' - 'imports:\n' - '- from a.d import implementation test_importing\n' - ) + "ymmsl_version: v0.2\n" + "description: Testing resolving imports\n" + "imports:\n" + "- from a.d import implementation test_importing\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) assert config.models == {} - resolve(Reference('test_resolve_imports'), config) + resolve(Reference("test_resolve_imports"), config) assert len(config.imports) == 0 assert len(config.models) == 1 - assert config.models[Reference('a.d.test_importing')].name == 'a.d.test_importing' + assert config.models[Reference("a.d.test_importing")].name == "a.d.test_importing" assert len(config.programs) == 2 - assert config.programs[Reference('a.d.micro')].executable == Path('python3') - assert config.programs[Reference('a.b.c.macro')].executable == Path('my_program') + assert config.programs[Reference("a.d.micro")].executable == Path("python3") + assert config.programs[Reference("a.b.c.macro")].executable == Path("my_program") def test_apply_custom_implementations_simple(env_ymmsl_path: None) -> None: # should import a model, and substitute a local program into it # then check that we have a single root model ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: Testing resolving imports with custom_implementations\n' - 'imports:\n' - '- from a.e import implementation test_model\n' - '- from a.b.c import implementation macro\n' - 'custom_implementations:\n' - ' test_model.macro: macro\n' - ) + "ymmsl_version: v0.2\n" + "description: Testing resolving imports with custom_implementations\n" + "imports:\n" + "- from a.e import implementation test_model\n" + "- from a.b.c import implementation macro\n" + "custom_implementations:\n" + " test_model.macro: macro\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) - resolve(Reference('test_resolve_imports'), config) + resolve(Reference("test_resolve_imports"), config) assert len(config.imports) == 0 assert len(config.models) == 1 - assert Reference('test_resolve_imports.test_model') in config.models - m = config.models[Reference('test_resolve_imports.test_model')] - c = m.components[Reference('macro')] - assert c.implementation == 'a.b.c.macro' + assert Reference("test_resolve_imports.test_model") in config.models + m = config.models[Reference("test_resolve_imports.test_model")] + c = m.components[Reference("macro")] + assert c.implementation == "a.b.c.macro" def test_apply_custom_implementations(env_ymmsl_path: None) -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: Testing resolving imports with custom_implementations\n' - 'imports:\n' - '- from a.g import implementation test_macro_micro\n' - ) + "ymmsl_version: v0.2\n" + "description: Testing resolving imports with custom_implementations\n" + "imports:\n" + "- from a.g import implementation test_macro_micro\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) - resolve(Reference('test_resolve_imports'), config) + resolve(Reference("test_resolve_imports"), config) assert len(config.imports) == 0 assert len(config.models) == 2 - model = config.models[Reference('a.g.test_macro_micro')] - assert model.name == 'a.g.test_macro_micro' + model = config.models[Reference("a.g.test_macro_micro")] + assert model.name == "a.g.test_macro_micro" - macro = model.components[Reference('macro')] - assert macro.implementation == \ - 'a.g.a_e_test_model__customised_for__a_g_test_macro_micro_macro' + macro = model.components[Reference("macro")] + assert ( + macro.implementation + == "a.g.a_e_test_model__customised_for__a_g_test_macro_micro_macro" + ) - micro = model.components[Reference('micro')] - assert micro.implementation == 'a.f.micro' + micro = model.components[Reference("micro")] + assert micro.implementation == "a.f.micro" model = config.models[ - Reference('a.g.a_e_test_model__customised_for__a_g_test_macro_micro_macro')] - assert model.name == \ - 'a.g.a_e_test_model__customised_for__a_g_test_macro_micro_macro' + Reference("a.g.a_e_test_model__customised_for__a_g_test_macro_micro_macro") + ] + assert ( + model.name == "a.g.a_e_test_model__customised_for__a_g_test_macro_micro_macro" + ) - macro = model.components[Reference('macro')] - assert macro.implementation == 'a.g.macro2' + macro = model.components[Reference("macro")] + assert macro.implementation == "a.g.macro2" assert len(config.programs) == 2 - assert config.programs[Reference('a.f.micro')].args == ['/home/user/micro.py'] - assert config.programs[Reference('a.g.macro2')].args == ['/home/user/macro2.py'] + assert config.programs[Reference("a.f.micro")].args == ["/home/user/micro.py"] + assert config.programs[Reference("a.g.macro2")].args == ["/home/user/macro2.py"] def test_apply_custom_implementations_double_use(env_ymmsl_path: None) -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: |\n' - ' Testing resolving imports with custom_implementations, using a model\n' - ' that uses the same submodel for two different components, to make sure\n' - ' that we can change an implementation in one without affecting the\n' - ' other\n' - 'imports:\n' - '- from a.h import implementation test_macro_macro\n' - ) + "ymmsl_version: v0.2\n" + "description: |\n" + " Testing resolving imports with custom_implementations, using a model\n" + " that uses the same submodel for two different components, to make sure\n" + " that we can change an implementation in one without affecting the\n" + " other\n" + "imports:\n" + "- from a.h import implementation test_macro_macro\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) - resolve(Reference('test_resolve_imports'), config) + resolve(Reference("test_resolve_imports"), config) assert len(config.imports) == 0 assert len(config.models) == 3 - model = config.models[Reference('a.h.test_macro_macro')] - assert model.name == 'a.h.test_macro_macro' + model = config.models[Reference("a.h.test_macro_macro")] + assert model.name == "a.h.test_macro_macro" - macro1 = model.components[Reference('macro1')] - assert macro1.implementation == 'a.e.test_model' + macro1 = model.components[Reference("macro1")] + assert macro1.implementation == "a.e.test_model" - macro2 = model.components[Reference('macro2')] - assert macro2.implementation == \ - 'a.h.a_e_test_model__customised_for__a_h_test_macro_macro_macro2' + macro2 = model.components[Reference("macro2")] + assert ( + macro2.implementation + == "a.h.a_e_test_model__customised_for__a_h_test_macro_macro_macro2" + ) - model = config.models[Reference('a.e.test_model')] - assert model.name == 'a.e.test_model' + model = config.models[Reference("a.e.test_model")] + assert model.name == "a.e.test_model" - macro = model.components[Reference('macro')] - assert macro.implementation == 'a.b.c.macro' + macro = model.components[Reference("macro")] + assert macro.implementation == "a.b.c.macro" model = config.models[ - Reference('a.h.a_e_test_model__customised_for__a_h_test_macro_macro_macro2') - ] - assert model.name == \ - 'a.h.a_e_test_model__customised_for__a_h_test_macro_macro_macro2' + Reference("a.h.a_e_test_model__customised_for__a_h_test_macro_macro_macro2") + ] + assert ( + model.name == "a.h.a_e_test_model__customised_for__a_h_test_macro_macro_macro2" + ) - macro = model.components[Reference('macro')] - assert macro.implementation == 'a.h.macro2' + macro = model.components[Reference("macro")] + assert macro.implementation == "a.h.macro2" assert len(config.programs) == 2 - assert config.programs[Reference('a.b.c.macro')].executable == Path('my_program') - assert config.programs[Reference('a.h.macro2')].args == ['/home/user/macro2.py'] + assert config.programs[Reference("a.b.c.macro")].executable == Path("my_program") + assert config.programs[Reference("a.h.macro2")].args == ["/home/user/macro2.py"] def test_apply_custom_implementations_set_none(env_ymmsl_path: None) -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: Testing resolving imports with custom_implementations\n' - 'models:\n' - ' macro_micro:\n' - ' components:\n' - ' macro:\n' - ' ports:\n' - ' o_i: out\n' - ' s: in\n' - ' description: Macro model\n' - ' implementation: macro\n' - ' micro:\n' - ' ports:\n' - ' f_init: in\n' - ' o_f: out\n' - ' description: Micro model\n' - ' implementation: micro\n' - 'custom_implementations:\n' - ' macro_micro.micro:\n' - ) + "ymmsl_version: v0.2\n" + "description: Testing resolving imports with custom_implementations\n" + "models:\n" + " macro_micro:\n" + " components:\n" + " macro:\n" + " ports:\n" + " o_i: out\n" + " s: in\n" + " description: Macro model\n" + " implementation: macro\n" + " micro:\n" + " ports:\n" + " f_init: in\n" + " o_f: out\n" + " description: Micro model\n" + " implementation: micro\n" + "custom_implementations:\n" + " macro_micro.micro:\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) - resolve(Reference('test_resolve_imports'), config) + resolve(Reference("test_resolve_imports"), config) - model = config.models[Reference('test_resolve_imports.macro_micro')] - assert model.components[Reference('micro')].implementation is None + model = config.models[Reference("test_resolve_imports.macro_micro")] + assert model.components[Reference("micro")].implementation is None def test_apply_custom_implementations_no_hidden_copies() -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: |\n' - ' Testing that all local references point to the same object\n' - 'models:\n' - ' A:\n' - ' description: Model A\n' - ' components:\n' - ' c1:\n' - ' ports: {}\n' - ' description: Component c1\n' - ' B:\n' - ' description: Model B\n' - ' components:\n' - ' c2:\n' - ' ports: {}\n' - ' description: Component c2\n' - ' c3:\n' - ' ports: {}\n' - ' description: Component c3\n' - 'programs:\n' - ' p:\n' - ' ports:\n' - ' description: A program\n' - ' executable: /home/user/p\n' - ) + "ymmsl_version: v0.2\n" + "description: |\n" + " Testing that all local references point to the same object\n" + "models:\n" + " A:\n" + " description: Model A\n" + " components:\n" + " c1:\n" + " ports: {}\n" + " description: Component c1\n" + " B:\n" + " description: Model B\n" + " components:\n" + " c2:\n" + " ports: {}\n" + " description: Component c2\n" + " c3:\n" + " ports: {}\n" + " description: Component c3\n" + "programs:\n" + " p:\n" + " ports:\n" + " description: A program\n" + " executable: /home/user/p\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) - config.models[Ref('A')].components[Ref('c1')].implementation = Ref('p') - config.custom_implementations[Reference('B.c2')] = Reference('A') - config.custom_implementations[Reference('B.c3')] = Reference('A') - - resolve(Reference('no_copies'), config) + config.models[Ref("A")].components[Ref("c1")].implementation = Ref("p") + config.custom_implementations[Reference("B.c2")] = Reference("A") + config.custom_implementations[Reference("B.c3")] = Reference("A") + resolve(Reference("no_copies"), config) # Same thing but using custom implementations for everything config = load(ymmsl) assert isinstance(config, Configuration) - config.custom_implementations[Reference('A.c1')] = Reference('p') - config.custom_implementations[Reference('B.c2')] = Reference('A') - config.custom_implementations[Reference('B.c3')] = Reference('A') + config.custom_implementations[Reference("A.c1")] = Reference("p") + config.custom_implementations[Reference("B.c2")] = Reference("A") + config.custom_implementations[Reference("B.c3")] = Reference("A") - resolve(Reference('no_copies'), config) + resolve(Reference("no_copies"), config) - b = config.models[Reference('no_copies.B')] - assert b.components[Reference('c2')].implementation == 'no_copies.A' - assert b.components[Reference('c3')].implementation == 'no_copies.A' + b = config.models[Reference("no_copies.B")] + assert b.components[Reference("c2")].implementation == "no_copies.A" + assert b.components[Reference("c3")].implementation == "no_copies.A" - a = config.models[Reference('no_copies.A')] - assert a.components[Reference('c1')].implementation == 'no_copies.p' + a = config.models[Reference("no_copies.A")] + assert a.components[Reference("c1")].implementation == "no_copies.p" # this should yield the same result as above, because all references to A point to # the same object config = load(ymmsl) assert isinstance(config, Configuration) - config.custom_implementations[Reference('B.c2')] = Reference('A') - config.custom_implementations[Reference('B.c3')] = Reference('A') - config.custom_implementations[Reference('A.c1')] = Reference('p') + config.custom_implementations[Reference("B.c2")] = Reference("A") + config.custom_implementations[Reference("B.c3")] = Reference("A") + config.custom_implementations[Reference("A.c1")] = Reference("p") - resolve(Reference('no_copies2'), config) + resolve(Reference("no_copies2"), config) - b = config.models[Reference('no_copies2.B')] - assert b.components[Reference('c2')].implementation == 'no_copies2.A' - assert b.components[Reference('c3')].implementation == 'no_copies2.A' + b = config.models[Reference("no_copies2.B")] + assert b.components[Reference("c2")].implementation == "no_copies2.A" + assert b.components[Reference("c3")].implementation == "no_copies2.A" - a = config.models[Reference('no_copies2.A')] - assert a.components[Reference('c1')].implementation == 'no_copies2.p' + a = config.models[Reference("no_copies2.A")] + assert a.components[Reference("c1")].implementation == "no_copies2.p" def test_apply_custom_implementations_everything_localised() -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: |\n' - ' Testing that local and imported models are treated the same when\n' - ' customised.\n' - 'imports:\n' - '- from a.e import implementation test_model\n' - 'models:\n' - ' A:\n' - ' description: Model A\n' - ' components:\n' - ' c1:\n' - ' ports: {}\n' - ' description: Component c1\n' - ' B:\n' - ' description: Model B\n' - ' components:\n' - ' macro:\n' - ' ports: {}\n' - ' description: Component c2\n' - 'programs:\n' - ' p:\n' - ' ports:\n' - ' description: A program\n' - ' executable: /home/user/p\n' - ) + "ymmsl_version: v0.2\n" + "description: |\n" + " Testing that local and imported models are treated the same when\n" + " customised.\n" + "imports:\n" + "- from a.e import implementation test_model\n" + "models:\n" + " A:\n" + " description: Model A\n" + " components:\n" + " c1:\n" + " ports: {}\n" + " description: Component c1\n" + " B:\n" + " description: Model B\n" + " components:\n" + " macro:\n" + " ports: {}\n" + " description: Component c2\n" + "programs:\n" + " p:\n" + " ports:\n" + " description: A program\n" + " executable: /home/user/p\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) - config.custom_implementations[Ref('A.c1')] = Ref('test_model') - config.custom_implementations[Ref('test_model.macro')] = Ref('p') + config.custom_implementations[Ref("A.c1")] = Ref("test_model") + config.custom_implementations[Ref("test_model.macro")] = Ref("p") - resolve(Reference('el'), config) + resolve(Reference("el"), config) - c1 = config.models[Ref('el.A')].components[Ref('c1')] - assert c1.implementation == 'el.test_model' + c1 = config.models[Ref("el.A")].components[Ref("c1")] + assert c1.implementation == "el.test_model" - test_model = config.models[Ref('el.test_model')] - assert test_model.components[Ref('macro')].implementation == 'el.p' + test_model = config.models[Ref("el.test_model")] + assert test_model.components[Ref("macro")].implementation == "el.p" config = load(ymmsl) assert isinstance(config, Configuration) - config.custom_implementations[Ref('A.c1')] = Ref('B') - config.custom_implementations[Ref('B.macro')] = Ref('p') + config.custom_implementations[Ref("A.c1")] = Ref("B") + config.custom_implementations[Ref("B.macro")] = Ref("p") - resolve(Reference('el'), config) + resolve(Reference("el"), config) - c1 = config.models[Ref('el.A')].components[Ref('c1')] - assert c1.implementation == 'el.B' + c1 = config.models[Ref("el.A")].components[Ref("c1")] + assert c1.implementation == "el.B" - b = config.models[Ref('el.B')] - assert b.components[Ref('macro')].implementation == 'el.p' + b = config.models[Ref("el.B")] + assert b.components[Ref("macro")].implementation == "el.p" def test_apply_custom_implementations_cut_branch(env_ymmsl_path: None) -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: |\n' - ' Testing that lopped-off branches are cleaned up properly.\n' - 'imports:\n' - '- from a.g import implementation test_macro_micro\n' - 'models:\n' - ' test_deeply_nested:\n' - ' description: Models within models within models...\n' - ' components:\n' - ' c1:\n' - ' ports:\n' - ' o_i: out\n' - ' s: in\n' - ' description: Macro model\n' - ' implementation: test_macro_micro\n' - 'programs:\n' - ' program3:\n' - ' ports:\n' - ' o_i: out\n' - ' s: in\n' - ' description: Alternative implementation\n' - ' executable: python3\n' - ' args: /home/user/program3.py\n' - 'custom_implementations:\n' - ' test_deeply_nested.c1: program3\n' - ) + "ymmsl_version: v0.2\n" + "description: |\n" + " Testing that lopped-off branches are cleaned up properly.\n" + "imports:\n" + "- from a.g import implementation test_macro_micro\n" + "models:\n" + " test_deeply_nested:\n" + " description: Models within models within models...\n" + " components:\n" + " c1:\n" + " ports:\n" + " o_i: out\n" + " s: in\n" + " description: Macro model\n" + " implementation: test_macro_micro\n" + "programs:\n" + " program3:\n" + " ports:\n" + " o_i: out\n" + " s: in\n" + " description: Alternative implementation\n" + " executable: python3\n" + " args: /home/user/program3.py\n" + "custom_implementations:\n" + " test_deeply_nested.c1: program3\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) - resolve(Reference('nested'), config) + resolve(Reference("nested"), config) assert len(config.models) == 1 def test_apply_custom_implementations_errors(env_ymmsl_path: None) -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: |\n' - ' Testing resolving imports with custom_implementations, using a model\n' - ' that uses the same submodel for two different components, to make sure\n' - ' that we can change an implementation in one without affecting the\n' - ' other\n' - 'imports:\n' - '- from a.h import implementation test_macro_macro\n' - 'programs:\n' - ' macro3:\n' - ' ports:\n' - ' o_i: out\n' - ' s: in\n' - ' description: Alternative macro model implementation\n' - ' executable: python3\n' - ' args: /home/user/macro3.py\n' - ) + "ymmsl_version: v0.2\n" + "description: |\n" + " Testing resolving imports with custom_implementations, using a model\n" + " that uses the same submodel for two different components, to make sure\n" + " that we can change an implementation in one without affecting the\n" + " other\n" + "imports:\n" + "- from a.h import implementation test_macro_macro\n" + "programs:\n" + " macro3:\n" + " ports:\n" + " o_i: out\n" + " s: in\n" + " description: Alternative macro model implementation\n" + " executable: python3\n" + " args: /home/user/macro3.py\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) - resolve(Reference('test_resolve_imports'), config) + resolve(Reference("test_resolve_imports"), config) config = load(ymmsl) assert isinstance(config, Configuration) - config.custom_implementations[Reference('test_macro_mAcro.macro')] = \ - Reference('macro3') + config.custom_implementations[Reference("test_macro_mAcro.macro")] = Reference( + "macro3" + ) with pytest.raises(RuntimeError): - resolve(Reference('test_resolve_imports'), config) + resolve(Reference("test_resolve_imports"), config) config = load(ymmsl) assert isinstance(config, Configuration) - config.custom_implementations[Reference('test_macro_macro.macro')] = \ - Reference('Macro3') + config.custom_implementations[Reference("test_macro_macro.macro")] = Reference( + "Macro3" + ) with pytest.raises(RuntimeError): - resolve(Reference('test_resolve_imports'), config) - + resolve(Reference("test_resolve_imports"), config) config = load(ymmsl) assert isinstance(config, Configuration) - config.custom_implementations[Reference('test_macro_macro.macro1[0]')] = \ - Reference('macro3') + config.custom_implementations[Reference("test_macro_macro.macro1[0]")] = Reference( + "macro3" + ) with pytest.raises(RuntimeError): - resolve(Reference('test_resolve_imports'), config) + resolve(Reference("test_resolve_imports"), config) config = load(ymmsl) assert isinstance(config, Configuration) - config.custom_implementations[Reference('test_macro_macro.macro3')] = \ - Reference('macro3') + config.custom_implementations[Reference("test_macro_macro.macro3")] = Reference( + "macro3" + ) with pytest.raises(RuntimeError): - resolve(Reference('test_resolve_imports'), config) + resolve(Reference("test_resolve_imports"), config) config = load(ymmsl) assert isinstance(config, Configuration) - config.custom_implementations[Reference('test_macro_macro.macro1.mAcro')] = \ - Reference('macro3') + config.custom_implementations[Reference("test_macro_macro.macro1.mAcro")] = ( + Reference("macro3") + ) with pytest.raises(RuntimeError): - resolve(Reference('test_resolve_imports'), config) + resolve(Reference("test_resolve_imports"), config) config = load(ymmsl) assert isinstance(config, Configuration) - config.custom_implementations[Reference('test_macro_macro.macro1.macro')] = \ - Reference('test_macro_macro') - config.custom_implementations[Reference('test_macro_macro.macro1.macro.mAcro')] = \ - Reference('macro3') + config.custom_implementations[Reference("test_macro_macro.macro1.macro")] = ( + Reference("test_macro_macro") + ) + config.custom_implementations[Reference("test_macro_macro.macro1.macro.mAcro")] = ( + Reference("macro3") + ) with pytest.raises(RuntimeError): - resolve(Reference('test_resolve_imports'), config) + resolve(Reference("test_resolve_imports"), config) def test_resolve_imports_module_not_found(env_ymmsl_path: None) -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: Testing missing modules\n' - 'imports:\n' - '- from a.doesnotexist import implementation test_error\n' - ) + "ymmsl_version: v0.2\n" + "description: Testing missing modules\n" + "imports:\n" + "- from a.doesnotexist import implementation test_error\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) with pytest.raises(RuntimeError) as e: - resolve(Reference('test_module_not_found'), config) + resolve(Reference("test_module_not_found"), config) - assert 'Failed to find a file' in str(e.value) + assert "Failed to find a file" in str(e.value) assert len(config.imports) == 1 def test_resolve_imports_broken_module(env_ymmsl_path: None) -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: Testing missing modules\n' - 'imports:\n' - '- from a.broken import implementation test_error\n' - ) + "ymmsl_version: v0.2\n" + "description: Testing missing modules\n" + "imports:\n" + "- from a.broken import implementation test_error\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) with pytest.raises(RuntimeError) as e: - resolve(Reference('test_broken_module'), config) + resolve(Reference("test_broken_module"), config) - assert 'model' in str(e.value) and 'models' in str(e.value) + assert "model" in str(e.value) and "models" in str(e.value) assert len(config.imports) == 1 def test_resolve_imports_implementation_not_found(env_ymmsl_path: None) -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: Testing missing modules\n' - 'imports:\n' - '- from a.d import implementation mucro\n' - ) + "ymmsl_version: v0.2\n" + "description: Testing missing modules\n" + "imports:\n" + "- from a.d import implementation mucro\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) with pytest.raises(RuntimeError) as e: - resolve(Reference('test_implementation_not_found'), config) + resolve(Reference("test_implementation_not_found"), config) - assert 'Implementation mucro not found' in str(e.value) + assert "Implementation mucro not found" in str(e.value) assert len(config.imports) == 1 def test_resolve_imports_no_shadowing(env_ymmsl_path: None) -> None: ymmsl = ( - 'ymmsl_version: v0.2\n' - 'description: Testing resolving imports\n' - 'imports:\n' - '- from a.d import implementation test_importing\n' - 'programs:\n' - ' test_importing:\n' - ' executable: python3\n' - ' args: /home/user/test_importing.py\n' - ) + "ymmsl_version: v0.2\n" + "description: Testing resolving imports\n" + "imports:\n" + "- from a.d import implementation test_importing\n" + "programs:\n" + " test_importing:\n" + " executable: python3\n" + " args: /home/user/test_importing.py\n" + ) config = load(ymmsl) assert isinstance(config, Configuration) with pytest.raises(RuntimeError) as e: - resolve(Reference('test_no_shadowing'), config) + resolve(Reference("test_no_shadowing"), config) - assert 'both defined and imported' in str(e.value) + assert "both defined and imported" in str(e.value) assert len(config.imports) == 1 @@ -561,7 +573,8 @@ def test_resolve_entrypoints(mock_entry_points: Mock) -> None: def test_resolve_entrypoints_duplicate_name( - mock_entry_points: Mock, caplog: pytest.LogCaptureFixture) -> None: + mock_entry_points: Mock, caplog: pytest.LogCaptureFixture +) -> None: config = load(""" ymmsl_version: v0.2 description: Test diff --git a/ymmsl/v0_2/tests/test_supported_settings.py b/ymmsl/v0_2/tests/test_supported_settings.py index fd47bc2..6ee0e5e 100644 --- a/ymmsl/v0_2/tests/test_supported_settings.py +++ b/ymmsl/v0_2/tests/test_supported_settings.py @@ -7,14 +7,14 @@ @pytest.fixture def supported_settings() -> SupportedSettings: settings = { - 'text': 'str', - 'alpha': 'int', - 'beta': 'float', - 'enable_something': 'bool Enables something', - 'things': '[int]', - 'numbers': '[float]', - 'square': '[[float]] Ensure it is not a rectangle' - } + "text": "str", + "alpha": "int", + "beta": "float", + "enable_something": "bool Enables something", + "things": "[int]", + "numbers": "[float]", + "square": "[[float]] Ensure it is not a rectangle", + } return SupportedSettings(settings) @@ -22,14 +22,14 @@ def supported_settings() -> SupportedSettings: @pytest.fixture def yaml_text() -> str: return ( - 'text: str\n' - 'alpha: int\n' - 'beta: float\n' - 'enable_something: bool Enables something\n' - 'things: [int]\n' - 'numbers: [float]\n' - 'square: \'[[float]] Ensure it is not a rectangle\'\n' - ) + "text: str\n" + "alpha: int\n" + "beta: float\n" + "enable_something: bool Enables something\n" + "things: [int]\n" + "numbers: [float]\n" + "square: '[[float]] Ensure it is not a rectangle'\n" + ) def test_create_supported_settings() -> None: @@ -39,77 +39,77 @@ def test_create_supported_settings() -> None: def test_create_supported_settings2(supported_settings: SupportedSettings) -> None: s = supported_settings._store - assert s[Identifier('text')].typ == SettingType.STR - assert s[Identifier('text')].description == '' - assert s[Identifier('alpha')].typ == SettingType.INT - assert s[Identifier('beta')].typ == SettingType.FLOAT - assert s[Identifier('enable_something')].typ == SettingType.BOOL - assert s[Identifier('enable_something')].description == 'Enables something' - assert s[Identifier('things')].typ == SettingType.LIST_INT - assert s[Identifier('numbers')].typ == SettingType.LIST_FLOAT - assert s[Identifier('square')].typ == SettingType.LIST_LIST_FLOAT - assert s[Identifier('square')].description == 'Ensure it is not a rectangle' + assert s[Identifier("text")].typ == SettingType.STR + assert s[Identifier("text")].description == "" + assert s[Identifier("alpha")].typ == SettingType.INT + assert s[Identifier("beta")].typ == SettingType.FLOAT + assert s[Identifier("enable_something")].typ == SettingType.BOOL + assert s[Identifier("enable_something")].description == "Enables something" + assert s[Identifier("things")].typ == SettingType.LIST_INT + assert s[Identifier("numbers")].typ == SettingType.LIST_FLOAT + assert s[Identifier("square")].typ == SettingType.LIST_LIST_FLOAT + assert s[Identifier("square")].description == "Ensure it is not a rectangle" def test_create_with_bad_name() -> None: with pytest.raises(ValueError): - SupportedSettings({'#test': 'str'}) + SupportedSettings({"#test": "str"}) def test_create_with_bad_type() -> None: with pytest.raises(ValueError): - SupportedSettings({'test': 'nosuchtype'}) + SupportedSettings({"test": "nosuchtype"}) def test_equality(supported_settings: SupportedSettings) -> None: settings2 = { - 'enable_something': 'bool Enables something', - 'square': '[[float]] Ensure it is not a rectangle', - 'alpha': 'int', - 'beta': 'float', - 'things': '[int]', - 'text': 'str', - 'numbers': '[float]', - } + "enable_something": "bool Enables something", + "square": "[[float]] Ensure it is not a rectangle", + "alpha": "int", + "beta": "float", + "things": "[int]", + "text": "str", + "numbers": "[float]", + } supported_settings2 = SupportedSettings(settings2) assert supported_settings == supported_settings2 def test_to_string() -> None: - supset = SupportedSettings({'a': 'str', 'b': 'int Description'}) - assert str(supset) == 'a: str, b: int' + supset = SupportedSettings({"a": "str", "b": "int Description"}) + assert str(supset) == "a: str, b: int" def test_item_access(supported_settings: SupportedSettings) -> None: - assert supported_settings['enable_something'].typ == SettingType.BOOL + assert supported_settings["enable_something"].typ == SettingType.BOOL - supported_settings['gamma'] = SupportedSetting('gamma', '[float]', '') - assert supported_settings._store[Identifier('gamma')].typ == SettingType.LIST_FLOAT - assert supported_settings[Identifier('gamma')].typ == SettingType.LIST_FLOAT + supported_settings["gamma"] = SupportedSetting("gamma", "[float]", "") + assert supported_settings._store[Identifier("gamma")].typ == SettingType.LIST_FLOAT + assert supported_settings[Identifier("gamma")].typ == SettingType.LIST_FLOAT - supported_settings['gamma'] = SupportedSetting('gamma', '[int]', 'description') - assert supported_settings._store[Identifier('gamma')].typ == SettingType.LIST_INT - assert supported_settings[Identifier('gamma')].description == 'description' + supported_settings["gamma"] = SupportedSetting("gamma", "[int]", "description") + assert supported_settings._store[Identifier("gamma")].typ == SettingType.LIST_INT + assert supported_settings[Identifier("gamma")].description == "description" def test_iteration(supported_settings: SupportedSettings) -> None: for name, setting in supported_settings: assert isinstance(name, Identifier) assert isinstance(setting, SupportedSetting) - if name == 'text': + if name == "text": assert setting.typ == SettingType.STR - if name == 'alpha': + if name == "alpha": assert setting.typ == SettingType.INT - if name == 'beta': + if name == "beta": assert setting.typ == SettingType.FLOAT - if name == 'enable_something': + if name == "enable_something": assert setting.typ == SettingType.BOOL - if name == 'things': + if name == "things": assert setting.typ == SettingType.LIST_INT - if name == 'numbers': + if name == "numbers": assert setting.typ == SettingType.LIST_FLOAT - if name == 'square': + if name == "square": assert setting.typ == SettingType.LIST_LIST_FLOAT @@ -117,82 +117,88 @@ def test_copy(supported_settings: SupportedSettings) -> None: supported_settings2 = supported_settings.copy() assert supported_settings2 is not supported_settings - assert supported_settings['text'] == supported_settings2['text'] + assert supported_settings["text"] == supported_settings2["text"] - supported_settings['text'] = SupportedSetting('text', 'int', '') - assert supported_settings['text'] != supported_settings2['text'] + supported_settings["text"] = SupportedSetting("text", "int", "") + assert supported_settings["text"] != supported_settings2["text"] -def test_load( - supported_settings: SupportedSettings, yaml_text: str) -> None: +def test_load(supported_settings: SupportedSettings, yaml_text: str) -> None: load = yatiml.load_function( - SupportedSettings, Identifier, SettingType, SupportedSetting) + SupportedSettings, Identifier, SettingType, SupportedSetting + ) loaded_settings = load(yaml_text) assert loaded_settings == supported_settings def test_dump(supported_settings: SupportedSettings, yaml_text: str) -> None: dumps = yatiml.dumps_function( - SupportedSettings, Identifier, SettingType, SupportedSetting) + SupportedSettings, Identifier, SettingType, SupportedSetting + ) dumped_settings = dumps(supported_settings) assert dumped_settings == yaml_text def test_load_descriptions() -> None: load = yatiml.load_function( - SupportedSettings, Identifier, SettingType, SupportedSetting) + SupportedSettings, Identifier, SettingType, SupportedSetting + ) s = load( - 'a: str With inline description\n' - 'b: \'[int] With inline description\'\n' - 'c: \'[[float]] With inline description\'\n' - 'd:\n' - ' type: str\n' - ' description: With single-line description\n' - 'e:\n' - ' type: [float]\n' - ' description: |-\n' - ' With multiline\n' - ' description\n' - ) + "a: str With inline description\n" + "b: '[int] With inline description'\n" + "c: '[[float]] With inline description'\n" + "d:\n" + " type: str\n" + " description: With single-line description\n" + "e:\n" + " type: [float]\n" + " description: |-\n" + " With multiline\n" + " description\n" + ) - assert s['a'].typ == SettingType.STR - assert s['a'].description == 'With inline description' + assert s["a"].typ == SettingType.STR + assert s["a"].description == "With inline description" - assert s['b'].typ == SettingType.LIST_INT - assert s['b'].description == 'With inline description' + assert s["b"].typ == SettingType.LIST_INT + assert s["b"].description == "With inline description" - assert s['c'].typ == SettingType.LIST_LIST_FLOAT - assert s['c'].description == 'With inline description' + assert s["c"].typ == SettingType.LIST_LIST_FLOAT + assert s["c"].description == "With inline description" - assert s['d'].typ == SettingType.STR - assert s['d'].description == 'With single-line description' + assert s["d"].typ == SettingType.STR + assert s["d"].description == "With single-line description" - assert s['e'].typ == SettingType.LIST_FLOAT - assert s['e'].description == 'With multiline\ndescription' + assert s["e"].typ == SettingType.LIST_FLOAT + assert s["e"].description == "With multiline\ndescription" def test_dump_descriptions() -> None: dumps = yatiml.dumps_function( - SupportedSettings, Identifier, SettingType, SupportedSetting) - - supported_settings = SupportedSettings({ - 'a': 'str With inline description', - 'b': '[int] With inline description', - 'c': '[[float]] With inline description', - 'd': 'str With single-line description', - 'e': '[float] With multiline\ndescription'}) + SupportedSettings, Identifier, SettingType, SupportedSetting + ) + + supported_settings = SupportedSettings( + { + "a": "str With inline description", + "b": "[int] With inline description", + "c": "[[float]] With inline description", + "d": "str With single-line description", + "e": "[float] With multiline\ndescription", + } + ) text = ( - 'a: str With inline description\n' - 'b: \'[int] With inline description\'\n' - 'c: \'[[float]] With inline description\'\n' - 'd: str With single-line description\n' - 'e:\n' - ' type: [float]\n' - ' description: |\n' - ' With multiline\n' - ' description\n' - ) + "a: str With inline description\n" + "b: '[int] With inline description'\n" + "c: '[[float]] With inline description'\n" + "d: str With single-line description\n" + "e:\n" + " type: [float]\n" + " description: |\n" + " With multiline\n" + " description\n" + ) assert dumps(supported_settings) == text