diff --git a/pyproject.toml b/pyproject.toml index 3fb69a0..6caf712 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ classifiers = [ "Intended Audience :: Science/Research", "Topic :: Scientific/Engineering", ] -requires-python = ">=3.10" +requires-python = ">=3.11" dependencies = [ "click", "dacite", diff --git a/src/incatools/odk/config.py b/src/incatools/odk/config.py index ef94603..077e992 100644 --- a/src/incatools/odk/config.py +++ b/src/incatools/odk/config.py @@ -10,9 +10,23 @@ from typing import Any, Dict, List, Optional, TextIO, Tuple import yaml -from dacite import from_dict +from dacite import Config, from_dict -from .model import ImportGroup, ImportProduct, OntologyProject +from .model import ( + Compression, + ExportFormat, + ImportGroup, + ImportProduct, + OntologyProject, +) + + +def enum_representer(dumper, data): + return yaml.ScalarNode(tag="tag:yaml.org,2002:str", value=str(data)) + + +yaml.add_representer(ExportFormat, enum_representer) +yaml.add_representer(Compression, enum_representer) class ConfigurationError(Exception): @@ -109,7 +123,13 @@ def load_config( project = OntologyProject() else: obj, config_hash = load_config_dict(config_file) - project = from_dict(data_class=OntologyProject, data=obj) + project = from_dict( + data_class=OntologyProject, + data=obj, + config=Config( + type_hooks={ExportFormat: ExportFormat, Compression: Compression} + ), + ) project.config_hash = config_hash if title: project.title = title @@ -224,6 +244,27 @@ def update_config_dict(obj: Dict[str, Any]) -> None: if use_base: imp["use_variant"] = "base" + # Old-style export_formats + orig_export_formats = obj.get("export_formats") + export_formats_converted = False + if orig_export_formats is not None: + if ( + isinstance(orig_export_formats, list) + and len(orig_export_formats) > 0 + and isinstance(orig_export_formats[0], str) + ): + # Old-style format, converting to new-style + obj["export_formats"] = [{"format": fmt} for fmt in orig_export_formats] + export_formats_converted = True + + # gzip_main: Converted to Gzip compression enabled on all output + # formats, but only if export_formats was in old-style. + gzip_main = obj.pop("gzip_main", False) + if gzip_main and export_formats_converted: + for spec in obj["export_formats"]: + if spec["format"] != "db": + spec["compressions"] = ["none", "gz"] + def pop_key(obj: Dict[str, Any], path: str) -> Optional[str]: """Gets the value of a key in a nested dictionary structure. diff --git a/src/incatools/odk/model.py b/src/incatools/odk/model.py index 00ab09a..108f009 100644 --- a/src/incatools/odk/model.py +++ b/src/incatools/odk/model.py @@ -9,6 +9,7 @@ import logging from dataclasses import dataclass, field +from enum import StrEnum from typing import Any, Dict, List, Optional from dataclasses_json import dataclass_json @@ -21,6 +22,59 @@ Url = str +class Compression(StrEnum): + """Available compression formats for release artefacts.""" + + UNCOMPRESSED = "none" + GZIP = "gz" + + +class ExportFormat(StrEnum): + """Available export formats for release artefacts.""" + + OWL = "owl" + """RDF/XML OWL Format.""" + + OBO = "obo" + """OBO Flat File Format.""" + + JSON = "json" + """OBOGraph-Json Format.""" + + TTL = "ttl" + """RDF/Turtle Format.""" + + SEMSQL = "db" + """SemSQL Format.""" + + +@dataclass_json +@dataclass +class ExportSpecification(JsonSchemaMixin): + """Describes how a release artefact is to be exported. + + This is merely a container for (1) an Export Format and (2) a list + of Compression values. + """ + + format: ExportFormat = ExportFormat.OWL + """The export format.""" + + compressions: List[Compression] = field(default_factory=list) + """The compression(s) to use with the export format.""" + + def __post_init__(self): + if not self.compressions: + # Set default compressions according to the format; for now, + # all formats except SemSQL default to no-compression (for + # backwards compatibility). This will likely change in the + # future to enable compression by default for all formats. + if self.format == ExportFormat.SEMSQL: + self.compressions = [Compression.GZIP] + else: + self.compressions = [Compression.UNCOMPRESSED] + + @dataclass_json @dataclass class CommandSettings(JsonSchemaMixin): @@ -1110,9 +1164,6 @@ class OntologyProject(JsonSchemaMixin): create_obo_metadata: bool = True """Generates metadata files for the OBO Foundry.""" - gzip_main: bool = False - """Produces gzipped versions of the main artefacts in all formats.""" - release_artefacts: List[str] = field(default_factory=lambda: ["full", "base"]) """The types of release artefacts to produce. @@ -1135,14 +1186,13 @@ class OntologyProject(JsonSchemaMixin): release_materialize_object_properties: Optional[List[str]] = None """The object properties to materialise at release time.""" - export_formats: List[str] = field(default_factory=lambda: ["owl", "obo"]) - """The formats the release artefacts should be exported to. - - Allowed values: owl, obo, json, ttl, db. - - Note that here, ``owl`` means RDF/XML, ``json`` means OBOGraph-Json, - and ``db`` means SemSQL. - """ + export_formats: List[ExportSpecification] = field( + default_factory=lambda: [ + ExportSpecification(format=ExportFormat.OWL), + ExportSpecification(format=ExportFormat.OBO), + ] + ) + """The formats the release artefacts should be exported to.""" namespaces: Optional[List[str]] = None """The namespaces that are considered at home in this ontology. @@ -1256,5 +1306,8 @@ def derive_fields(self) -> None: self.public_release = "github" # Exporting to OWL is mandatory, even if not explicitly listed - if "owl" not in self.export_formats: - self.export_formats.append("owl") + if not [f for f in self.export_formats if f.format == ExportFormat.OWL]: + self.export_formats.append(ExportSpecification(format=ExportFormat.OWL)) + + # Ready-to-use list of export format names, for convenience + self.export_format_names = [f.format for f in self.export_formats] diff --git a/src/incatools/odk/templates/_dynamic_files.jinja2 b/src/incatools/odk/templates/_dynamic_files.jinja2 index b25922e..180cf73 100644 --- a/src/incatools/odk/templates/_dynamic_files.jinja2 +++ b/src/incatools/odk/templates/_dynamic_files.jinja2 @@ -302,20 +302,20 @@ homepage: {{ project.repo_url }} products: - id: {{ project.id }}.owl name: "{{ project.title }} main release in OWL format" -{%- for format in project.export_formats %} -{%- if format != "owl" %} - - id: {{ project.id }}.{{ format }} - name: "{{ project.title }} main release in {{ format|upper }} format" +{%- for spec in project.export_formats %} +{%- if spec.format != "owl" %} + - id: {{ project.id }}.{{ spec.format }} + name: "{{ project.title }} main release in {{ spec.format|upper }} format" {%- endif %} {%- endfor %} {%- for artefact in project.release_artefacts %} {%- if artefact != project.primary_release %} - id: {{ project.id }}/{{ project.id }}-{{ artefact }}.owl name: "{{ project.title }} {{ artefact|upper }} release in OWL format" -{%- for format in project.export_formats %} -{%- if format != "owl" %} - - id: {{ project.id }}/{{ project.id }}-{{ artefact }}.{{ format }} - name: "{{ project.title }} {{ artefact|upper }} release in {{ format|upper}} format" +{%- for spec in project.export_formats %} +{%- if spec.format != "owl" %} + - id: {{ project.id }}/{{ project.id }}-{{ artefact }}.{{ spec.format }} + name: "{{ project.title }} {{ artefact|upper }} release in {{ spec.format|upper}} format" {%- endif %} {%- endfor %} {%- endif %} @@ -349,9 +349,9 @@ base_url: /obo/{{ project.id }} products: {%- if project.github_org %} - {{ project.id }}.owl: https://github.com/{{ project.github_org }}/{{ project.repo }}/releases/latest/download/{{ project.id }}.owl -{%- for format in project.export_formats %} -{%- if format != "owl" %} -- {{ project.id }}.{{ format }}: https://github.com/{{ project.github_org }}/{{ project.repo }}/releases/latest/download/{{ project.id }}.{{ format }} +{%- for spec in project.export_formats %} +{%- if spec.format != "owl" %} +- {{ project.id }}.{{ spec.format }}: https://github.com/{{ project.github_org }}/{{ project.repo }}/releases/latest/download/{{ project.id }}.{{ spec.format }} {%- endif %} {%- endfor %} {%- else %} diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 283269f..80e20c0 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -40,7 +40,7 @@ ONTBASE = {{ project.uribase }}/{% if project.uribase_suffix i {{ project.id -}} {% endif %} EDIT_FORMAT = {{ project.edit_format|default('owl') }} -FORMATS ={% for format in project.export_formats|sort %} {{ format }}{% endfor %} +FORMATS ={% for format in project.export_format_names|sort %} {{ format }}{% endfor %} RELEASE_ARTEFACTS ={% for release in project.release_artefacts %} {# -#} {% if release.startswith("custom") -%} {{ release | replace("custom-", "") -}} @@ -1322,18 +1322,18 @@ $(SUBSETDIR)/%.owl: $(ONT).owl | $(SUBSETDIR) all_robot_plugins $(ANNOTATE_ONTOLOGY_VERSION) --output $@ .PRECIOUS: $(SUBSETDIR)/%.owl -{% if 'obo' in project.export_formats -%} +{% if 'obo' in project.export_format_names -%} $(SUBSETDIR)/%.obo: $(SUBSETDIR)/%.owl $(ROBOT) convert --input $< --check false --format obo \ $(OBO_FORMAT_OPTIONS) --output $@ {% endif -%} -{% if 'ttl' in project.export_formats -%} +{% if 'ttl' in project.export_format_names -%} $(SUBSETDIR)/%.ttl: $(SUBSETDIR)/%.owl $(ROBOT) convert --input $< --check false --format ttl --output $@ {% endif -%} -{% if 'json' in project.export_formats -%} +{% if 'json' in project.export_format_names -%} $(SUBSETDIR)/%.json: $(SUBSETDIR)/%.owl $(ROBOT) convert --input $< --check false --format json --output $@ @@ -1477,10 +1477,11 @@ ROBOT_RELEASE_IMPORT_MODE_BASE = $(ROBOT) remove --input $< --select imports --t {% endif %} MAIN_PRODUCTS = $(sort $(foreach a, $(RELEASE_ARTEFACTS), $(a)) $(ONT)) -MAIN_FILES = $(foreach p, $(MAIN_PRODUCTS), $(foreach f, $(FORMATS), $(p).$(f))){# -#} - {% if project.gzip_main %} \ - $(foreach f, $(FORMATS), $(ONT).$(f).gz){# -#} - {% endif %} +MAIN_FILES = $(foreach p, $(MAIN_PRODUCTS), {% for spec in project.export_formats -%} + {% for comp in spec.compressions -%} + $(p).{{ spec.format }}{% if comp != "none" %}.{{ comp }}{% endif %} {# -#} + {% endfor -%} + {% endfor -%}) .PHONY: all_main all_main: $(MAIN_FILES){# -#} @@ -1749,7 +1750,7 @@ $(ONT)-international.owl: $(ONT).owl $(TRANSLATIONS_OWL) {% endif -%} {% endfor -%} -{% if project.export_formats|length > 1 or project.export_formats[0] != "owl" -%} +{% if project.export_format_names|length > 1 or project.export_format_names[0] != "owl" -%} # Release artefacts: export formats # ---------------------------------------- @@ -1758,13 +1759,13 @@ $(ONT)-international.owl: $(ONT).owl $(TRANSLATIONS_OWL) variants ($(ONT)-base, $(ONT)-full, etc.), so that we do not have to duplicate all the code below (once for the primary artefact and one when we iterate over the variants. -#} -{% if "obo" in project.export_formats -%} +{% if "obo" in project.export_format_names -%} $(ONT).obo: $(ONT).owl $(ROBOT) convert --input $< --check false \ --format obo $(OBO_FORMAT_OPTIONS) --output $@ {% endif -%} -{% if "ttl" in project.export_formats -%} +{% if "ttl" in project.export_format_names -%} {# FIXME: Why are re-annotating? The release.owl product should have been annotated already. See https://github.com/INCATools/ontology-development-kit/issues/1366. -#} @@ -1774,7 +1775,7 @@ $(ONT).ttl: $(ONT).owl convert --check false --format ttl --output $@ {% endif -%} -{% if "json" in project.export_formats -%} +{% if "json" in project.export_format_names -%} $(ONT).json: $(ONT).owl $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ $(ANNOTATE_ONTOLOGY_VERSION) \ @@ -1787,20 +1788,20 @@ $(ONT).json: $(ONT).owl {% else -%} {% set release = "$(ONT)-" ~ r -%} {% endif -%} -{% if "obo" in project.export_formats -%} +{% if "obo" in project.export_format_names -%} {{ release }}.obo: {{ release }}.owl $(ROBOT) convert --input $< --check false \ --format obo $(OBO_FORMAT_OPTIONS) --output $@ {% endif -%} -{% if "ttl" in project.export_formats -%} +{% if "ttl" in project.export_format_names -%} {{ release }}.ttl: {{ release }}.owl $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ $(ANNOTATE_ONTOLOGY_VERSION) \ convert --check false --format ttl --output $@ {% endif -%} -{% if "json" in project.export_formats -%} +{% if "json" in project.export_format_names -%} {{ release }}.json: {{ release }}.owl $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ $(ANNOTATE_ONTOLOGY_VERSION) \ @@ -1809,7 +1810,7 @@ $(ONT).json: $(ONT).owl {% endif -%} {% endfor -%} -{% if 'db' in project.export_formats -%} +{% if 'db' in project.export_format_names -%} {% if project.use_context -%} CONTEXT_FILE_CSV = $(TMPDIR)/context.csv @@ -1826,25 +1827,17 @@ $(CONTEXT_FILE_CSV): $(CONTEXT_FILE) | $(TMPDIR) {% endif -%} -{% for format in project.export_formats -%} -{% if project.gzip_main -%} -$(ONT).{{ format }}.gz: $(ONT).{{ format }} +{% for spec in project.export_formats -%} +{% for comp in spec.compressions -%} +{% if comp == "gz" -%} +%.{{ spec.format }}.gz: %.{{ spec.format }} gzip -c $< > $@ -{% endif -%} +{% endif -%} +{% endfor -%} {% endfor -%} -{% if 'owl' not in project.export_formats -%} -{% if project.gzip_main -%} -$(ONT).owl.gz: $(ONT).owl - gzip -c $< > $@ - -{% endif -%} -$(ONT).owl: $(ONT)-{{ project.primary_release }}.owl - cp $< $@ - -{% endif -%} -{% endif %}{# !if project.export_formats|length > 1 or project.export_formats[0] != "owl" -#} +{% endif %}{# !if project.export_format_names|length > 1 or project.export_format_names[0] != "owl" -#} {# #} # ---------------------------------------- diff --git a/tests/configs/test-export-spec.yaml b/tests/configs/test-export-spec.yaml new file mode 100644 index 0000000..5079445 --- /dev/null +++ b/tests/configs/test-export-spec.yaml @@ -0,0 +1,17 @@ +id: triffo +title: Test new-style export formats +github_org: cmungall +report_fail_on: ERROR +repo: triffo +import_group: + products: + - id: ro + - id: bfo + - id: pato +export_formats: + - format: owl + compressions: [none, gz] + - format: obo + compressions: [gz] + - format: json + compressions: [gz]