Allow per-format compression

pyproject.toml

@@ -14,7 +14,7 @@ classifiers = [
     "Intended Audience :: Science/Research",
     "Topic :: Scientific/Engineering",
 ]
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "click",
     "dacite",

src/incatools/odk/config.py

@@ -10,9 +10,23 @@
 from typing import Any, Dict, List, Optional, TextIO, Tuple
 
 import yaml
-from dacite import from_dict
+from dacite import Config, from_dict
 
-from .model import ImportGroup, ImportProduct, OntologyProject
+from .model import (
+    Compression,
+    ExportFormat,
+    ImportGroup,
+    ImportProduct,
+    OntologyProject,
+)
+
+
+def enum_representer(dumper, data):
+    return yaml.ScalarNode(tag="tag:yaml.org,2002:str", value=str(data))
+
+
+yaml.add_representer(ExportFormat, enum_representer)
+yaml.add_representer(Compression, enum_representer)
 
 
 class ConfigurationError(Exception):
@@ -109,7 +123,13 @@ def load_config(
         project = OntologyProject()
     else:
         obj, config_hash = load_config_dict(config_file)
-        project = from_dict(data_class=OntologyProject, data=obj)
+        project = from_dict(
+            data_class=OntologyProject,
+            data=obj,
+            config=Config(
+                type_hooks={ExportFormat: ExportFormat, Compression: Compression}
+            ),
+        )
         project.config_hash = config_hash
     if title:
         project.title = title
@@ -224,6 +244,27 @@ def update_config_dict(obj: Dict[str, Any]) -> None:
             if use_base:
                 imp["use_variant"] = "base"
 
+    # Old-style export_formats
+    orig_export_formats = obj.get("export_formats")
+    export_formats_converted = False
+    if orig_export_formats is not None:
+        if (
+            isinstance(orig_export_formats, list)
+            and len(orig_export_formats) > 0
+            and isinstance(orig_export_formats[0], str)
+        ):
+            # Old-style format, converting to new-style
+            obj["export_formats"] = [{"format": fmt} for fmt in orig_export_formats]
+            export_formats_converted = True
+
+    # gzip_main: Converted to Gzip compression enabled on all output
+    # formats, but only if export_formats was in old-style.
+    gzip_main = obj.pop("gzip_main", False)
+    if gzip_main and export_formats_converted:
+        for spec in obj["export_formats"]:
+            if spec["format"] != "db":
+                spec["compressions"] = ["none", "gz"]
+
 
 def pop_key(obj: Dict[str, Any], path: str) -> Optional[str]:
     """Gets the value of a key in a nested dictionary structure.

src/incatools/odk/model.py

@@ -9,6 +9,7 @@
 
 import logging
 from dataclasses import dataclass, field
+from enum import StrEnum
 from typing import Any, Dict, List, Optional
 
 from dataclasses_json import dataclass_json
@@ -21,6 +22,59 @@
 Url = str
 
 
+class Compression(StrEnum):
+    """Available compression formats for release artefacts."""
+
+    UNCOMPRESSED = "none"
+    GZIP = "gz"
+
+
+class ExportFormat(StrEnum):
+    """Available export formats for release artefacts."""
+
+    OWL = "owl"
+    """RDF/XML OWL Format."""
+
+    OBO = "obo"
+    """OBO Flat File Format."""
+
+    JSON = "json"
+    """OBOGraph-Json Format."""
+
+    TTL = "ttl"
+    """RDF/Turtle Format."""
+
+    SEMSQL = "db"
+    """SemSQL Format."""
+
+
+@dataclass_json
+@dataclass
+class ExportSpecification(JsonSchemaMixin):
+    """Describes how a release artefact is to be exported.
+
+    This is merely a container for (1) an Export Format and (2) a list
+    of Compression values.
+    """
+
+    format: ExportFormat = ExportFormat.OWL
+    """The export format."""
+
+    compressions: List[Compression] = field(default_factory=list)
+    """The compression(s) to use with the export format."""
+
+    def __post_init__(self):
+        if not self.compressions:
+            # Set default compressions according to the format; for now,
+            # all formats except SemSQL default to no-compression (for
+            # backwards compatibility). This will likely change in the
+            # future to enable compression by default for all formats.
+            if self.format == ExportFormat.SEMSQL:
+                self.compressions = [Compression.GZIP]
+            else:
+                self.compressions = [Compression.UNCOMPRESSED]
+
+
 @dataclass_json
 @dataclass
 class CommandSettings(JsonSchemaMixin):
@@ -1110,9 +1164,6 @@ class OntologyProject(JsonSchemaMixin):
     create_obo_metadata: bool = True
     """Generates metadata files for the OBO Foundry."""
 
-    gzip_main: bool = False
-    """Produces gzipped versions of the main artefacts in all formats."""
-
     release_artefacts: List[str] = field(default_factory=lambda: ["full", "base"])
     """The types of release artefacts to produce.
 
@@ -1135,14 +1186,13 @@ class OntologyProject(JsonSchemaMixin):
     release_materialize_object_properties: Optional[List[str]] = None
     """The object properties to materialise at release time."""
 
-    export_formats: List[str] = field(default_factory=lambda: ["owl", "obo"])
-    """The formats the release artefacts should be exported to.
-
-    Allowed values: owl, obo, json, ttl, db.
-
-    Note that here, ``owl`` means RDF/XML, ``json`` means OBOGraph-Json,
-    and ``db`` means SemSQL.
-    """
+    export_formats: List[ExportSpecification] = field(
+        default_factory=lambda: [
+            ExportSpecification(format=ExportFormat.OWL),
+            ExportSpecification(format=ExportFormat.OBO),
+        ]
+    )
+    """The formats the release artefacts should be exported to."""
 
     namespaces: Optional[List[str]] = None
     """The namespaces that are considered at home in this ontology.
@@ -1256,5 +1306,8 @@ def derive_fields(self) -> None:
             self.public_release = "github"
 
         # Exporting to OWL is mandatory, even if not explicitly listed
-        if "owl" not in self.export_formats:
-            self.export_formats.append("owl")
+        if not [f for f in self.export_formats if f.format == ExportFormat.OWL]:
+            self.export_formats.append(ExportSpecification(format=ExportFormat.OWL))
+
+        # Ready-to-use list of export format names, for convenience
+        self.export_format_names = [f.format for f in self.export_formats]

src/incatools/odk/templates/_dynamic_files.jinja2

@@ -302,20 +302,20 @@ homepage: {{ project.repo_url }}
 products:
   - id: {{ project.id }}.owl
     name: "{{ project.title }} main release in OWL format"
-{%- for format in project.export_formats %}
-{%-   if format != "owl" %}
-  - id: {{ project.id }}.{{ format }}
-    name: "{{ project.title }} main release in {{ format|upper }} format"
+{%- for spec in project.export_formats %}
+{%-   if spec.format != "owl" %}
+  - id: {{ project.id }}.{{ spec.format }}
+    name: "{{ project.title }} main release in {{ spec.format|upper }} format"
 {%-   endif %}
 {%- endfor %}
 {%- for artefact in project.release_artefacts %}
 {%-   if artefact != project.primary_release %}
   - id: {{ project.id }}/{{ project.id }}-{{ artefact }}.owl
     name: "{{ project.title }} {{ artefact|upper }} release in OWL format"
-{%-   for format in project.export_formats %}
-{%-     if format != "owl" %}
-  - id: {{ project.id }}/{{ project.id }}-{{ artefact }}.{{ format }}
-    name: "{{ project.title }} {{ artefact|upper }} release in {{ format|upper}} format"
+{%-   for spec in project.export_formats %}
+{%-     if spec.format != "owl" %}
+  - id: {{ project.id }}/{{ project.id }}-{{ artefact }}.{{ spec.format }}
+    name: "{{ project.title }} {{ artefact|upper }} release in {{ spec.format|upper}} format"
 {%-     endif %}
 {%-   endfor %}
 {%-   endif %}
@@ -349,9 +349,9 @@ base_url: /obo/{{ project.id }}
 products:
 {%- if project.github_org %}
 - {{ project.id }}.owl: https://github.com/{{ project.github_org }}/{{ project.repo }}/releases/latest/download/{{ project.id }}.owl
-{%-   for format in project.export_formats %}
-{%-     if format != "owl" %}
-- {{ project.id }}.{{ format }}: https://github.com/{{ project.github_org }}/{{ project.repo }}/releases/latest/download/{{ project.id }}.{{ format }}
+{%-   for spec in project.export_formats %}
+{%-     if spec.format != "owl" %}
+- {{ project.id }}.{{ spec.format }}: https://github.com/{{ project.github_org }}/{{ project.repo }}/releases/latest/download/{{ project.id }}.{{ spec.format }}
 {%-     endif %}
 {%-   endfor %}
 {%- else %}

src/incatools/odk/templates/src/ontology/Makefile.jinja2

@@ -40,7 +40,7 @@ ONTBASE                   = {{ project.uribase }}/{% if project.uribase_suffix i
                                                        {{ project.id -}}
                                                   {% endif %}
 EDIT_FORMAT               = {{ project.edit_format|default('owl') }}
-FORMATS                   ={% for format in project.export_formats|sort %} {{ format }}{% endfor %}
+FORMATS                   ={% for format in project.export_format_names|sort %} {{ format }}{% endfor %}
 RELEASE_ARTEFACTS         ={% for release in project.release_artefacts %} {# -#}
                            {%   if release.startswith("custom") -%}
                                   {{ release | replace("custom-", "") -}}
@@ -1322,18 +1322,18 @@ $(SUBSETDIR)/%.owl: $(ONT).owl | $(SUBSETDIR) all_robot_plugins
 	         $(ANNOTATE_ONTOLOGY_VERSION) --output $@
 .PRECIOUS: $(SUBSETDIR)/%.owl
 
-{%   if 'obo' in project.export_formats -%}
+{%   if 'obo' in project.export_format_names -%}
 $(SUBSETDIR)/%.obo: $(SUBSETDIR)/%.owl
 	$(ROBOT) convert --input $< --check false --format obo \
 	                 $(OBO_FORMAT_OPTIONS) --output $@
 
 {%   endif -%}
-{%   if 'ttl' in project.export_formats -%}
+{%   if 'ttl' in project.export_format_names -%}
 $(SUBSETDIR)/%.ttl: $(SUBSETDIR)/%.owl
 	$(ROBOT) convert --input $< --check false --format ttl --output $@
 
 {%   endif -%}
-{%   if 'json' in project.export_formats -%}
+{%   if 'json' in project.export_format_names -%}
 $(SUBSETDIR)/%.json: $(SUBSETDIR)/%.owl
 	$(ROBOT) convert --input $< --check false --format json --output $@
 
@@ -1477,10 +1477,11 @@ ROBOT_RELEASE_IMPORT_MODE_BASE = $(ROBOT) remove --input $< --select imports --t
                                  {% endif %}
 
 MAIN_PRODUCTS = $(sort $(foreach a, $(RELEASE_ARTEFACTS), $(a)) $(ONT))
-MAIN_FILES    = $(foreach p, $(MAIN_PRODUCTS), $(foreach f, $(FORMATS), $(p).$(f))){# -#}
-                {% if project.gzip_main %} \
-                $(foreach f, $(FORMATS), $(ONT).$(f).gz){# -#}
-                {% endif %}
+MAIN_FILES    = $(foreach p, $(MAIN_PRODUCTS), {% for spec in project.export_formats -%}
+                                               {%   for comp in spec.compressions -%}
+                                               $(p).{{ spec.format }}{% if comp != "none" %}.{{ comp }}{% endif %} {# -#}
+                                               {%   endfor -%}
+                                               {% endfor -%})
 
 .PHONY: all_main
 all_main: $(MAIN_FILES){# -#}
@@ -1749,7 +1750,7 @@ $(ONT)-international.owl: $(ONT).owl $(TRANSLATIONS_OWL)
 {%   endif -%}
 {% endfor -%}
 
-{% if project.export_formats|length > 1 or project.export_formats[0] != "owl" -%}
+{% if project.export_format_names|length > 1 or project.export_format_names[0] != "owl" -%}
 # Release artefacts: export formats
 # ----------------------------------------
 
@@ -1758,13 +1759,13 @@ $(ONT)-international.owl: $(ONT).owl $(TRANSLATIONS_OWL)
    variants ($(ONT)-base, $(ONT)-full, etc.), so that we do not have
    to duplicate all the code below (once for the primary artefact and
    one when we iterate over the variants. -#}
-{%   if "obo" in project.export_formats -%}
+{%   if "obo" in project.export_format_names -%}
 $(ONT).obo: $(ONT).owl
 	$(ROBOT) convert --input $< --check false \
 	                 --format obo $(OBO_FORMAT_OPTIONS) --output $@
 
 {%   endif -%}
-{%   if "ttl" in project.export_formats -%}
+{%   if "ttl" in project.export_format_names -%}
 {# FIXME: Why are re-annotating? The release.owl product should have
    been annotated already.
    See https://github.com/INCATools/ontology-development-kit/issues/1366. -#}
@@ -1774,7 +1775,7 @@ $(ONT).ttl: $(ONT).owl
 	         convert --check false --format ttl --output $@
 
 {%   endif -%}
-{%   if "json" in project.export_formats -%}
+{%   if "json" in project.export_format_names -%}
 $(ONT).json: $(ONT).owl
 	$(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \
 	         $(ANNOTATE_ONTOLOGY_VERSION) \
@@ -1787,20 +1788,20 @@ $(ONT).json: $(ONT).owl
 {%     else -%}
 {%       set release = "$(ONT)-" ~ r -%}
 {%     endif -%}
-{%     if "obo" in project.export_formats -%}
+{%     if "obo" in project.export_format_names -%}
 {{ release }}.obo: {{ release }}.owl
 	$(ROBOT) convert --input $< --check false \
 	                 --format obo $(OBO_FORMAT_OPTIONS) --output $@
 
 {%     endif -%}
-{%     if "ttl" in project.export_formats -%}
+{%     if "ttl" in project.export_format_names -%}
 {{ release }}.ttl: {{ release }}.owl
 	$(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \
 	         $(ANNOTATE_ONTOLOGY_VERSION) \
 	         convert --check false --format ttl --output $@
 
 {%     endif -%}
-{%     if "json" in project.export_formats -%}
+{%     if "json" in project.export_format_names -%}
 {{ release }}.json: {{ release }}.owl
 	$(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \
 	         $(ANNOTATE_ONTOLOGY_VERSION) \
@@ -1809,7 +1810,7 @@ $(ONT).json: $(ONT).owl
 {%     endif -%}
 {%   endfor -%}
 
-{%   if 'db' in project.export_formats -%}
+{%   if 'db' in project.export_format_names -%}
 
 {%     if project.use_context -%}
 CONTEXT_FILE_CSV = $(TMPDIR)/context.csv
@@ -1826,25 +1827,17 @@ $(CONTEXT_FILE_CSV): $(CONTEXT_FILE) | $(TMPDIR)
 
 {%   endif -%}
 
-{%   for format in project.export_formats -%}
-{%     if project.gzip_main -%}
-$(ONT).{{ format }}.gz: $(ONT).{{ format }}
+{%   for spec in project.export_formats -%}
+{%     for comp in spec.compressions -%}
+{%       if comp == "gz" -%}
+%.{{ spec.format }}.gz: %.{{ spec.format }}
 	gzip -c $< > $@
 
-{%     endif -%}
+{%       endif -%}
+{%     endfor -%}
 {%   endfor -%}
-{%   if 'owl' not in project.export_formats -%}
-{%     if project.gzip_main -%}
-$(ONT).owl.gz: $(ONT).owl
-	gzip -c $< > $@
-
-{%     endif -%}
-$(ONT).owl: $(ONT)-{{ project.primary_release }}.owl
-	cp $< $@
-
-{%   endif -%}
 
-{% endif %}{# !if project.export_formats|length > 1 or project.export_formats[0] != "owl" -#}
+{% endif %}{# !if project.export_format_names|length > 1 or project.export_format_names[0] != "owl" -#}
 
 {# #}
 # ----------------------------------------

tests/configs/test-export-spec.yaml

@@ -0,0 +1,17 @@
+id: triffo
+title: Test new-style export formats
+github_org: cmungall
+report_fail_on: ERROR
+repo: triffo
+import_group:
+  products:
+    - id: ro
+    - id: bfo
+    - id: pato
+export_formats:
+  - format: owl
+    compressions: [none, gz]
+  - format: obo
+    compressions: [gz]
+  - format: json
+    compressions: [gz]