Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .git-blame-ignore-revs
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Ruff format
c69bbeff1d5171e230eb069112fdb65da1476a82
1 change: 1 addition & 0 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ commands =
mypy
pytest {posargs}
ruff check
ruff format --check

[gh-actions]
python =
Expand Down
17 changes: 13 additions & 4 deletions ymmsl/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
This package contains all the classes needed to represent a yMMSL file,
as well as to read and write yMMSL files.
"""

from importlib.metadata import version as package_version

from ymmsl.conversion.converter import DowngradeError, convert_to
Expand All @@ -13,10 +14,18 @@
from ymmsl.v0_2 import Operator, Settings

__version__ = package_version("ymmsl")
__author__ = 'Lourens Veen'
__email__ = 'l.veen@esciencecenter.nl'
__author__ = "Lourens Veen"
__email__ = "l.veen@esciencecenter.nl"


__all__ = [
'convert_to', 'Document', 'dump', 'DowngradeError', 'load', 'load_as',
'Operator', 'save', 'Settings']
"convert_to",
"Document",
"dump",
"DowngradeError",
"load",
"load_as",
"Operator",
"save",
"Settings",
]
83 changes: 52 additions & 31 deletions ymmsl/command_line.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,14 @@


def showwarning(
message: Union[Warning, str], category: Type[Warning], filename: str,
lineno: int, file: Optional[TextIO] = None, line: Optional[str] = None) -> None:
print(f'WARNING: {message}', file=file)
message: Union[Warning, str],
category: Type[Warning],
filename: str,
lineno: int,
file: Optional[TextIO] = None,
line: Optional[str] = None,
) -> None:
print(f"WARNING: {message}", file=file)


warnings.showwarning = showwarning
Expand All @@ -33,24 +38,37 @@ def ymmsl() -> None:


_version_tag_to_type: Dict[str, Type] = {
'v0.1': v0_1.PartialConfiguration,
'v0.2': v0_2.Configuration,
}
"v0.1": v0_1.PartialConfiguration,
"v0.2": v0_2.Configuration,
}


@ymmsl.command(short_help='Convert a yMMSL file to a newer version')
@ymmsl.command(short_help="Convert a yMMSL file to a newer version")
@click.argument(
'input_file', default='-', type=click.Path(
exists=True, file_okay=True, dir_okay=False, readable=True,
resolve_path=False, allow_dash=True))
"input_file",
default="-",
type=click.Path(
exists=True,
file_okay=True,
dir_okay=False,
readable=True,
resolve_path=False,
allow_dash=True,
),
)
@click.argument(
'output_file', required=False, type=click.Path(
file_okay=True, dir_okay=True, writable=True, resolve_path=False,
allow_dash=True))
@click.option(
'-t', '--to', default='v0.2', help='Version to convert to, e.g. "v0.2".')
def convert(
input_file: str, output_file: Optional[str], to: str) -> None:
"output_file",
required=False,
type=click.Path(
file_okay=True,
dir_okay=True,
writable=True,
resolve_path=False,
allow_dash=True,
),
)
@click.option("-t", "--to", default="v0.2", help='Version to convert to, e.g. "v0.2".')
def convert(input_file: str, output_file: Optional[str], to: str) -> None:
"""Convert a yMMSL file to a later version

When upgrading in place, and/or if an output file is specified and it exists, a
Expand Down Expand Up @@ -98,16 +116,17 @@ def convert(
won't work, because the shell will open the file to do the redirect, empty it, and
then run ymmsl convert, which then fails because the input is empty.
"""
if input_file != '-':
print(f'Converting {input_file}')
if input_file != "-":
print(f"Converting {input_file}")

if output_file is None:
output_file = input_file

if to not in _version_tag_to_type:
click.echo(
f'Invalid version {to} specified. Supported versions are v0.1 and v0.2',
err=True)
f"Invalid version {to} specified. Supported versions are v0.1 and v0.2",
err=True,
)
exit(1)

try:
Expand All @@ -116,22 +135,24 @@ def convert(
except DowngradeError:
click.echo()
click.echo(
f'It seems that {input_file} is of a newer version than the target'
f' version {to}.')
f"It seems that {input_file} is of a newer version than the target"
f" version {to}."
)
click.echo()
click.echo(
'This tool can only upgrade files, not downgrade them, so that does'
' not work.')
"This tool can only upgrade files, not downgrade them, so that does"
" not work."
)
exit(1)

if output_file != '-' and os.path.exists(output_file):
backup_file = output_file + '.bak'
if output_file != "-" and os.path.exists(output_file):
backup_file = output_file + ".bak"
if not os.path.exists(backup_file):
copyfile(output_file, backup_file)
print(f'Wrote backup file {backup_file}')
print(f"Wrote backup file {backup_file}")

with click.open_file(output_file, 'w') as output_stream:
with click.open_file(output_file, "w") as output_stream:
save(document, output_stream)

if input_file != '-':
print('Conversion complete')
if input_file != "-":
print("Conversion complete")
118 changes: 76 additions & 42 deletions ymmsl/conversion/convert_v0_1_to_v0_2.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,20 @@ def convert_v0_1_to_v0_2(config: v0_1.PartialConfiguration) -> v0_2.Configuratio
Returns:
The corresponding configuration expressed in yMMSL v0.2.
"""
description = 'Please add a description'
description = "Please add a description"
if config.description:
description = config.description
models = [convert_model(config.model)] if config.model is not None else None
settings = deepcopy(config.settings)
programs = [
convert_implementation(impl) for impl in config.implementations.values()]
convert_implementation(impl) for impl in config.implementations.values()
]
if programs:
warnings.warn(
'In yMMSL v0.2 implementations have become programs, and you can now'
' specify the ports of a program in the yMMSL description. If your'
' program has fixed ports then you should do this, because it will make'
' incorrect wiring easier to debug. While there, add a description too!',
"In yMMSL v0.2 implementations have become programs, and you can now"
" specify the ports of a program in the yMMSL description. If your"
" program has fixed ports then you should do this, because it will make"
" incorrect wiring easier to debug. While there, add a description too!",
stacklevel=2,
)

Expand All @@ -36,27 +37,40 @@ def convert_v0_1_to_v0_2(config: v0_1.PartialConfiguration) -> v0_2.Configuratio
resume = deepcopy(config.resume)

warnings.warn(
'Comments can unfortunately not be read by this converter, and so have been'
' ignored. Please copy them into an appropriate description field.',
"Comments can unfortunately not be read by this converter, and so have been"
" ignored. Please copy them into an appropriate description field.",
stacklevel=2,
)

return v0_2.Configuration(
description, None, models, None, settings, programs, resources, checkpoints,
resume)
description,
None,
models,
None,
settings,
programs,
resources,
checkpoints,
resume,
)


def convert_component(component: v0_1.Component) -> v0_2.Component:
"""Convert a v0.1 Component object to a v0.2 Component."""
ports = component.ports if component.ports else v0_1.Ports()
description = 'Please add a description'
description = "Please add a description"
implementation: Optional[str] = None
if component.implementation is not None:
implementation = str(component.implementation)

return v0_2.Component(
str(component.name), convert_ports(ports), description, implementation,
False, component.multiplicity)
str(component.name),
convert_ports(ports),
description,
implementation,
False,
component.multiplicity,
)


def convert_conduit(conduit: v0_1.Conduit) -> v0_2.Conduit:
Expand All @@ -77,27 +91,31 @@ def infer_ports(components: List[v0_2.Component], conduits: List[v0_2.Conduit])
for conduit in conduits:
if conduit.sending_component() == component.name:
component.ports[conduit.sending_port()] = v0_2.Port(
conduit.sending_port(), v0_2.Operator.O_F,
v0_2.Timeline(''))
conduit.sending_port(), v0_2.Operator.O_F, v0_2.Timeline("")
)
if component.name not in changed_components:
changed_components.append(component.name)

if conduit.receiving_component() == component.name:
component.ports[conduit.receiving_port()] = v0_2.Port(
conduit.receiving_port(), v0_2.Operator.F_INIT,
v0_2.Timeline(''))
conduit.receiving_port(),
v0_2.Operator.F_INIT,
v0_2.Timeline(""),
)
if component.name not in changed_components:
changed_components.append(component.name)

if changed_components:
ch_comp_list = '\n - ' + '\n - '.join(map(str, changed_components))
ch_comp_list = "\n - " + "\n - ".join(map(str, changed_components))
warnings.warn(
'In yMMSL v0.2 components are required to declare their ports. The'
' following components did not have a ports declaration, so one has'
' been added based on the connected conduits. THIS MAY BE WRONG,'
' because the operators have all been set to F_INIT and O_F, while they'
' may really be O_I or S. Please check these components and adjust'
f' as needed: {ch_comp_list}', stacklevel=4)
"In yMMSL v0.2 components are required to declare their ports. The"
" following components did not have a ports declaration, so one has"
" been added based on the connected conduits. THIS MAY BE WRONG,"
" because the operators have all been set to F_INIT and O_F, while they"
" may really be O_I or S. Please check these components and adjust"
f" as needed: {ch_comp_list}",
stacklevel=4,
)


def convert_model(model: v0_1.ModelReference) -> v0_2.Model:
Expand All @@ -109,14 +127,15 @@ def convert_model(model: v0_1.ModelReference) -> v0_2.Model:
Returns:
The corresponding configuration expressed in yMMSL v0.2.
"""
description = 'Please add a description'
description = "Please add a description"

if isinstance(model, v0_1.Model):
components = list(map(convert_component, model.components))
conduits = list(map(convert_conduit, model.conduits))
infer_ports(components, conduits)
return v0_2.Model(
str(model.name), None, description, None, components, conduits)
str(model.name), None, description, None, components, conduits
)
else:
return v0_2.Model(str(model.name), None, description, None, [], [])

Expand All @@ -133,7 +152,7 @@ def convert_implementation(impl: v0_1.Implementation) -> v0_2.Program:
Returns:
The corresponding program expressed in yMMSL v0.2.
"""
description = 'Please add a description'
description = "Please add a description"
base_env: Optional[v0_1.BaseEnv] = impl.base_env
env: Optional[Dict[str, str]] = impl.env

Expand All @@ -146,9 +165,21 @@ def convert_implementation(impl: v0_1.Implementation) -> v0_2.Program:
env = None

return v0_2.Program(
str(impl.name), None, description, None, base_env, impl.modules,
impl.virtual_env, env, execution_model, impl.executable, impl.args,
impl.script, impl.can_share_resources, impl.keeps_state_for_next_use)
str(impl.name),
None,
description,
None,
base_env,
impl.modules,
impl.virtual_env,
env,
execution_model,
impl.executable,
impl.args,
impl.script,
impl.can_share_resources,
impl.keeps_state_for_next_use,
)


def convert_ports(ports: v0_1.Ports) -> v0_2.Ports:
Expand All @@ -161,23 +192,26 @@ def convert_ports(ports: v0_1.Ports) -> v0_2.Ports:
The corresponding ports expressed in yMMSL v0.2.
"""
return v0_2.Ports(
list(map(str, ports.f_init)),
list(map(str, ports.o_i)),
list(map(str, ports.s)),
list(map(str, ports.o_f)))
list(map(str, ports.f_init)),
list(map(str, ports.o_i)),
list(map(str, ports.s)),
list(map(str, ports.o_f)),
)


def convert_resources(
resources: MutableMapping[v0_1.Reference, v0_1.ResourceRequirements],
models: Optional[List[v0_2.Model]]
) -> MutableMapping[v0_2.Reference, v0_2.ResourceRequirements]:
resources: MutableMapping[v0_1.Reference, v0_1.ResourceRequirements],
models: Optional[List[v0_2.Model]],
) -> MutableMapping[v0_2.Reference, v0_2.ResourceRequirements]:
if not models:
warnings.warn(
'When specifying resources in yMMSL v0.2, component names must be'
' prefixed with the name of the top (outermost) model, e.g. as'
' my_model.my_component rather than just my_component. This file'
' does not contain a model, so its name cannot be added automatically.'
' Please add the model name yourself.', stacklevel=3)
"When specifying resources in yMMSL v0.2, component names must be"
" prefixed with the name of the top (outermost) model, e.g. as"
" my_model.my_component rather than just my_component. This file"
" does not contain a model, so its name cannot be added automatically."
" Please add the model name yourself.",
stacklevel=3,
)
return deepcopy(resources)
else:
mname = models[0].name
Expand Down
Loading