diff --git a/pyproject.toml b/pyproject.toml index e16b6cf9..3535d54b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "openpyxl >=3.0.7, !=3.1.1", "GDX2py >=2.2.0", "ijson >=3.1.4", - "chardet >=4.0.0", + "chardet >=7", "PyMySQL[rsa] >=1.0.2", "psycopg2-binary", "pyarrow >= 19.0", diff --git a/spinedb_api/exception.py b/spinedb_api/exception.py index ec04c09c..9855de9c 100644 --- a/spinedb_api/exception.py +++ b/spinedb_api/exception.py @@ -77,6 +77,11 @@ def __init__(self, msg, rank=None, key=None): self.rank = rank self.key = key + def __eq__(self, other): + if not isinstance(other, InvalidMappingComponent): + return NotImplemented + return self.msg == other.msg and self.rank == other.rank and self.key == other.key + class ReaderError(SpineDBAPIError): """Failure in import reader.""" diff --git a/spinedb_api/import_mapping/generator.py b/spinedb_api/import_mapping/generator.py index cd6dc8ff..1abe6702 100644 --- a/spinedb_api/import_mapping/generator.py +++ b/spinedb_api/import_mapping/generator.py @@ -16,13 +16,15 @@ """ from collections.abc import Callable from copy import deepcopy -from operator import itemgetter +from itertools import dropwhile from typing import Any, Optional from ..exception import ParameterValueFormatError from ..helpers import string_to_bool +from ..import_functions import UnparseCallable from ..mapping import Position, is_pivoted from ..parameter_value import ( Array, + IndexedValue, Map, TimePattern, TimeSeriesVariableResolution, @@ -30,7 +32,16 @@ from_database, split_value_and_type, ) -from .import_mapping import ImportMapping, check_validity +from .import_mapping import ( + ArrayValueRecord, + ImportMapping, + MapValueRecord, + SemiMappedData, + TimePatternValueRecord, + TimeSeriesValueRecord, + ValueRecord, + check_validity, +) from .import_mapping_compat import import_mapping_from_dict _NO_VALUE = object() @@ -178,6 +189,7 @@ def get_mapped_data( _make_entities(mapped_data) _make_entity_metadata(mapped_data) _make_entity_alternatives(mapped_data, errors) + _make_parameter_definitions(mapped_data, unparse_value) _make_parameter_values(mapped_data, unparse_value) _make_parameter_value_metadata(mapped_data) return mapped_data, errors @@ -295,23 +307,34 @@ def _unpivot_rows( return unpivoted_rows, pivoted_pos, non_pivoted_pos, unpivoted_column_pos -def _make_entity_classes(mapped_data): - rows = mapped_data.get("entity_classes") - if rows is None: +def _make_entity_classes(mapped_data: dict) -> None: + try: + rows = mapped_data.pop("entity_classes") + except KeyError: return - rows = [(class_name, tuple(dimension_names)) for class_name, dimension_names in rows.items()] - rows.sort(key=itemgetter(1)) - mapped_data["entity_classes"] = final_rows = [] - for class_name, dimension_names in rows: - row = (class_name, tuple(dimension_names)) if dimension_names else (class_name,) - final_rows.append(row) + final_rows = [] + for name, record in rows.items(): + item = [name, record.dimensions] + if record.description: + item.append(record.description) + final_rows.append(item) + if final_rows: + mapped_data["entity_classes"] = final_rows def _make_entities(mapped_data): - rows = mapped_data.get("entities") - if rows is None: + try: + rows = mapped_data.pop("entities") + except KeyError: return - mapped_data["entities"] = list(rows) + final_rows = [] + for (class_name, name), record in rows.items(): + item = [class_name, name if not record.elements else record.elements] + if record.description: + item.append(record.description) + final_rows.append(item) + if final_rows: + mapped_data["entities"] = final_rows def _make_entity_alternatives(mapped_data, errors): @@ -332,35 +355,59 @@ def _make_entity_alternatives(mapped_data, errors): mapped_data["entity_alternatives"] = rows +def _make_parameter_definitions(mapped_data: SemiMappedData, unparse_value: UnparseCallable) -> None: + key = "parameter_definitions" + try: + rows = mapped_data.pop(key) + except KeyError: + return + final_rows = [] + for (entity_class_name, parameter_name), record in rows.items(): + definition_data = [entity_class_name, parameter_name] + default_value = record.default_value + if isinstance(default_value, ValueRecord): + if default_value.has_value(): + default_value = unparse_value(_make_value(default_value)) + else: + default_value = None + elif isinstance(default_value, str): + try: + default_value = from_database(*split_value_and_type(default_value)) + except ParameterValueFormatError: + pass + reversed_extras = [record.description, record.value_list_name, default_value] + definition_data += reversed(list(dropwhile(lambda x: x is None, reversed_extras))) + final_rows.append(definition_data) + if final_rows: + mapped_data[key] = final_rows + + def _make_parameter_values(mapped_data, unparse_value): - value_pos = 3 key = "parameter_values" - rows = mapped_data.get(key) - if rows is not None: - valued_rows = [] - for row in rows: - raw_value = _make_value(row, value_pos) - if raw_value is _NO_VALUE: - continue - value = unparse_value(raw_value) - if value is not None: - row[value_pos] = value - valued_rows.append(row) - mapped_data[key] = valued_rows - value_pos = 0 - key = "parameter_definitions" - rows = mapped_data.get(key) - if rows is not None: - full_rows = [] - for entity_definition, extras in rows.items(): - if extras: - value = unparse_value(_make_value(extras, value_pos)) - if value is not None: - extras[value_pos] = value - full_rows.append(entity_definition + tuple(extras)) + try: + rows = mapped_data.pop(key) + except KeyError: + return + final_rows = [] + for (entity_class_name, entity_byname, parameter_name, alternative_name), value in rows.items(): + if isinstance(value, ValueRecord): + if value.has_value(): + value = unparse_value(_make_value(value)) else: - full_rows.append(entity_definition) - mapped_data[key] = full_rows + value = None + elif isinstance(value, str): + try: + value = from_database(*split_value_and_type(value)) + except ParameterValueFormatError: + pass + if value is None: + continue + value_data = [entity_class_name, entity_byname, parameter_name, value] + if alternative_name is not None: + value_data.append(alternative_name) + final_rows.append(value_data) + if final_rows: + mapped_data[key] = final_rows def _make_parameter_value_metadata(mapped_data): @@ -377,42 +424,28 @@ def _make_entity_metadata(mapped_data): mapped_data["entity_metadata"] = list(rows) -def _make_value(row, value_pos): - try: - value = row[value_pos] - except IndexError: - return None - if isinstance(value, dict): - if "data" not in value: - return _NO_VALUE - return _parameter_value_from_dict(value) - if isinstance(value, str): - try: - return from_database(*split_value_and_type(value)) - except ParameterValueFormatError: - pass - return value - - -def _parameter_value_from_dict(d): - mapped_index_names = d.get("index_names", {0: ""}) - index_names = (max(mapped_index_names) + 1) * [""] - for i, name in mapped_index_names.items(): - index_names[i] = name - if d["type"] == "map": - map_ = _table_to_map(d["data"], compress=d.get("compress", False)) - if index_names != [""]: - _apply_index_names(map_, index_names) - return map_ - if d["type"] == "time_pattern": - return TimePattern(*zip(*d["data"]), index_name=index_names[0]) - if d["type"] == "time_series": - options = d.get("options", {}) - ignore_year = options.get("ignore_year", False) - repeat = options.get("repeat", False) - return TimeSeriesVariableResolution(*zip(*d["data"]), ignore_year, repeat, index_name=index_names[0]) - if d["type"] == "array": - return Array(d["data"], index_name=index_names[0]) +def _make_value(record: ValueRecord) -> IndexedValue: + match record: + case ArrayValueRecord(): + index_name = record.index_names[0] if record.index_names else "" + return Array(record.values, index_name=index_name) + case TimePatternValueRecord(): + index_name = record.index_names[0] if record.index_names else "" + indexes = [i[0] for i in record.indexes] + return TimePattern(indexes, record.values, index_name) + case TimeSeriesValueRecord(): + index_name = record.index_names[0] if record.index_names else "" + indexes = [i[0] for i in record.indexes] + return TimeSeriesVariableResolution(indexes, record.values, record.ignore_year, record.repeat, index_name) + case MapValueRecord(): + map_value = _table_to_map( + ([*indexes, values] for indexes, values in zip(record.indexes, record.values)), record.compress + ) + if record.index_names: + _apply_index_names(map_value, record.index_names) + return map_value + case _: + raise RuntimeError(f"logic error: unknown record type '{type(record).__name__}'") def _table_to_map(table, compress=False): @@ -456,7 +489,7 @@ def _apply_index_names(map_, index_names): """ name = index_names[0] if name: - map_.index_name = index_names[0] + map_.index_name = name if len(index_names) == 1: return for v in map_.values: @@ -464,16 +497,14 @@ def _apply_index_names(map_, index_names): _apply_index_names(v, index_names[1:]) -def _ensure_mapping_name_consistency(mappings, mapping_names): +def _ensure_mapping_name_consistency(mappings: list[ImportMapping], mapping_names: list[str]) -> None: """Makes sure that there are as many mapping names as actual mappings. Args: - mappings (list(ImportMapping)): list of mappings - mapping_names (list(str)): list of mapping names + mappings: list of mappings + mapping_names: list of mapping names """ n_mappings = len(mappings) n_mapping_names = len(mapping_names) - if n_mapping_names > n_mappings: - mapping_names = mapping_names[:n_mappings] - elif n_mapping_names < n_mappings: + if n_mapping_names < n_mappings: mapping_names += [""] * (n_mappings - n_mapping_names) diff --git a/spinedb_api/import_mapping/import_mapping.py b/spinedb_api/import_mapping/import_mapping.py index 8c62c406..bd58f3e6 100644 --- a/spinedb_api/import_mapping/import_mapping.py +++ b/spinedb_api/import_mapping/import_mapping.py @@ -10,33 +10,33 @@ # this program. If not, see . ###################################################################################################################### """Contains import mappings for database items such as entities, entity classes and parameter values.""" +from __future__ import annotations from collections.abc import Iterable +from dataclasses import dataclass, field from enum import Enum, auto, unique -from typing import Any, ClassVar +from typing import Any, ClassVar, Generic, Type, TypeAlias, TypeVar from spinedb_api.exception import InvalidMapping, InvalidMappingComponent from spinedb_api.mapping import Mapping, Position, is_pivoted, parse_fixed_position_value, unflatten @unique class ImportKey(Enum): - DIMENSION_COUNT = auto() ENTITY_CLASS_NAME = auto() ENTITY_NAME = auto() + ELEMENT_NAMES = auto() GROUP_NAME = auto() MEMBER_NAME = auto() METADATA_NAME = auto() METADATA_VALUE = auto() PARAMETER_NAME = auto() - PARAMETER_DEFINITION = auto() - PARAMETER_DEFINITION_EXTRAS = auto() - PARAMETER_DEFAULT_VALUES = auto() + PARAMETER_DEFAULT_VALUE_RECORD = auto() PARAMETER_DEFAULT_VALUE_INDEXES = auto() - PARAMETER_VALUES = auto() + PARAMETER_DEFAULT_VALUE_INDEX_NAMES = auto() + PARAMETER_VALUE_RECORD = auto() PARAMETER_VALUE_INDEXES = auto() + PARAMETER_VALUE_INDEX_NAMES = auto() PARAMETER_VALUE_METADATA_NAME = auto() PARAMETER_VALUE_METADATA_VALUE = auto() - DIMENSION_NAMES = auto() - ELEMENT_NAMES = auto() ALTERNATIVE_NAME = auto() SCENARIO_NAME = auto() SCENARIO_ALTERNATIVE = auto() @@ -54,13 +54,10 @@ def __str__(self): self.METADATA_NAME: "Metadata names", self.METADATA_VALUE: "Metadata values", self.PARAMETER_NAME.value: "Parameter names", - self.PARAMETER_DEFINITION.value: "Parameter names", self.PARAMETER_DEFAULT_VALUE_INDEXES.value: "Parameter indexes", self.PARAMETER_VALUE_INDEXES.value: "Parameter indexes", self.PARAMETER_VALUE_METADATA_NAME.value: "Metadata names", self.PARAMETER_VALUE_METADATA_VALUE.value: "Metadata values", - self.DIMENSION_NAMES.value: "Dimension names", - self.ELEMENT_NAMES.value: "Element names", self.PARAMETER_VALUE_LIST_NAME.value: "Parameter value lists", self.SCENARIO_NAME.value: "Scenario names", self.SCENARIO_ALTERNATIVE.value: "Alternative names", @@ -72,28 +69,42 @@ def __str__(self): return super().__str__() -class KeyFix(Exception): - """Opposite of KeyError""" - +State: TypeAlias = dict[ImportKey, Any] +SemiMappedData: TypeAlias = dict[str, Any] -def check_validity(root_mapping): - class _DummySourceRow: - def __getitem__(self, key): - return "true" +def check_validity(root_mapping: ImportMapping) -> list[InvalidMappingComponent]: errors = [] for rank, mapping in enumerate(root_mapping.flatten()): - if mapping.position != Position.fixed: - continue - try: - parse_fixed_position_value(mapping.value) - except InvalidMapping as error: - errors.append(InvalidMappingComponent(str(error), rank)) - source_row = _DummySourceRow() - root_mapping.import_row(source_row, {}, {}, errors) + if mapping.position == Position.fixed: + try: + parse_fixed_position_value(mapping.value) + except InvalidMapping as error: + errors.append(InvalidMappingComponent(str(error), rank)) + elif mapping.position != Position.hidden or mapping.value is not None: + try: + mapping.check_validity() + except InvalidMappingComponent as error: + errors.append(error) + errors += _check_dependent_pairs(root_mapping) return errors +def _check_dependent_pairs(root_mapping: ImportMapping) -> list[InvalidMappingComponent]: + flattened = root_mapping.flatten() + try: + definition_mapping = next(m for m in flattened if isinstance(m, ParameterDefinitionMapping)) + value_list_mapping = next(m for m in flattened if isinstance(m, ParameterValueListMapping)) + except StopIteration: + return [] + if (value_list_mapping.position is not Position.hidden or definition_mapping.value is not None) and ( + definition_mapping.position == Position.hidden and definition_mapping.value is None + ): + value_list_rank = next(n for n, m in enumerate(flattened) if isinstance(m, ParameterValueListMapping)) + return [InvalidMappingComponent("value list requires a parameter name", value_list_rank)] + return [] + + class ImportMapping(Mapping): """Base class for import mappings.""" @@ -180,6 +191,9 @@ def check_for_invalid_column_refs(self, header, table_name): return msg return "" + def check_validity(self) -> None: + return + def polish(self, table_name, source_header, mapping_name, column_count=0, for_preview=False): """Polishes the mapping before an import operation. 'Expands' transient ``position`` and ``value`` attributes into their final value. @@ -282,34 +296,20 @@ def filter_accepts_row(self, source_row): self.child is None or self.child.filter_accepts_row(source_row) ) - def import_row(self, source_row, state, mapped_data, errors=None): + def import_row(self, source_row, state, mapped_data): if self.has_filter() and not self.filter_accepts_row(source_row): return - if errors is None: - errors = [] if not (self.position == Position.hidden and self.value is None): source_data = self._data(source_row) if source_data is None: if not self.ignorable or self.child is None: self._skip_row(state) return - self.child.import_row(source_row, state, mapped_data, errors=errors) + self.child.import_row(source_row, state, mapped_data) return - try: - self._import_row(source_data, state, mapped_data) - except KeyError as err: - for key in err.args: - msg = f"Required key '{key}' is invalid" - error = InvalidMappingComponent(msg, self.rank, key) - errors.append(error) - except KeyFix as fix: - indexes = set() - for key in fix.args: - indexes |= {k for k, err in enumerate(errors) if err.key == key} - for k in sorted(indexes, reverse=True): - errors.pop(k) + self._import_row(source_data, state, mapped_data) if self.child is not None: - self.child.import_row(source_row, state, mapped_data, errors=errors) + self.child.import_row(source_row, state, mapped_data) def _data(self, source_row): # pylint: disable=arguments-renamed if source_row is None: @@ -405,46 +405,135 @@ def reconstruct(cls, position, value, skip_columns, read_start_row, filter_re, m mapping = cls(position, value, skip_columns, read_start_row, filter_re, compress, options) return mapping + def _make_value_record(self, value_type: str) -> ValueRecord: + match value_type: + case "array": + return ArrayValueRecord() + case "map": + return MapValueRecord(compress=self.compress) + case "time_series": + return TimeSeriesValueRecord( + ignore_year=self.options.get("ignore_year", False), repeat=self.options.get("repeat", False) + ) + case "time_pattern": + return TimePatternValueRecord() + case _: + raise InvalidMapping(f"unknown value type '{value_type}'") -class EntityClassMapping(ImportMapping): - """Maps entity classes. - Can be used as the topmost mapping. - """ +@dataclass +class EntityClassRecord: + dimensions: list[str] = field(default_factory=list) + description: str | None = None + + +class EntityClassMapping(ImportMapping): + """Maps entity classes.""" MAP_TYPE = "EntityClass" def _import_row(self, source_data, state, mapped_data): - dim_count = len([m for m in self.flatten() if isinstance(m, DimensionMapping)]) - state[ImportKey.DIMENSION_COUNT] = dim_count entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] = str(source_data) - dimension_names = state[ImportKey.DIMENSION_NAMES] = [] entity_classes = mapped_data.setdefault("entity_classes", {}) - entity_classes[entity_class_name] = dimension_names - if dim_count: - raise KeyError(ImportKey.DIMENSION_NAMES) + entity_classes[entity_class_name] = EntityClassRecord() -class EntityMapping(ImportMapping): - """Maps entities. +def _require_parent(mapping: ImportMapping, parent_type: Type[ImportMapping]) -> None: + parent = mapping.parent + while parent is not None: + if isinstance(parent, parent_type): + if parent.position == Position.hidden and parent.value is None: + raise InvalidMappingComponent( + f"{mapping.MAP_TYPE} requires {parent_type.MAP_TYPE} with position or constant value", mapping.rank + ) + return + parent = parent.parent + raise InvalidMappingComponent(f"{mapping.MAP_TYPE} requires {parent_type.MAP_TYPE} as parent", mapping.rank) - Cannot be used as the topmost mapping; one of the parents must be :class:`EntityClassMapping`. - """ - MAP_TYPE = "Entity" +def _require_one_of_parents(mapping: ImportMapping, parent_types: tuple[Type[ImportMapping], ...]) -> None: + parent = mapping.parent + while parent is not None: + if isinstance(parent, parent_types) and (parent.position != Position.hidden or parent.value is not None): + return + parent = parent.parent + display_types = " or ".join(m.MAP_TYPE for m in parent_types) + raise InvalidMappingComponent(f"{mapping.MAP_TYPE} requires {display_types} as parent", mapping.rank) + + +def _require_enough_parents(mapping: ImportMapping, parent_type: Type[ImportMapping]) -> None: + n_same_parent_type = 1 + parent = mapping.parent + while parent is not None: + if isinstance(parent, type(mapping)): + n_same_parent_type += 1 + parent = parent.parent + n_required_parent_type = 0 + parent = mapping.parent + while parent is not None: + if isinstance(parent, parent_type): + n_required_parent_type += 1 + if n_required_parent_type == n_same_parent_type: + return + parent = parent.parent + raise InvalidMappingComponent( + f"the number of {mapping.MAP_TYPE} and {parent_type.MAP_TYPE} mappings do not match", mapping.rank + ) + - def import_row(self, source_row, state, mapped_data, errors=None): - state[ImportKey.ELEMENT_NAMES] = () - super().import_row(source_row, state, mapped_data, errors=errors) +class EntityClassDescriptionMapping(ImportMapping): + """Maps entity class descriptions.""" + + MAP_TYPE = "EntityClassDescription" + ignorable = True + + def _import_row(self, source_data, state, mapped_data): + description = str(source_data) + if description: + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + mapped_data["entity_classes"][entity_class_name].description = description + + def check_validity(self) -> None: + _require_parent(self, EntityClassMapping) + + +@dataclass +class EntityRecord: + elements: list[str] = field(default_factory=list) + description: str | None = None + + +class EntityMapping(ImportMapping): + """Maps entities.""" + + MAP_TYPE = "Entity" def _import_row(self, source_data, state, mapped_data): - if state[ImportKey.DIMENSION_COUNT]: + if self.position == Position.hidden and isinstance(self._child, ElementMapping): return entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] entity_name = state[ImportKey.ENTITY_NAME] = str(source_data) - if isinstance(self.child, EntityGroupMapping): - raise KeyError(ImportKey.MEMBER_NAME) - mapped_data.setdefault("entities", {})[entity_class_name, entity_name] = None + mapped_data.setdefault("entities", {})[entity_class_name, entity_name] = EntityRecord() + + def check_validity(self) -> None: + _require_parent(self, EntityClassMapping) + + +class EntityDescriptionMapping(ImportMapping): + """Maps entity descriptions.""" + + MAP_TYPE = "EntityDescription" + ignorable = True + + def _import_row(self, source_data, state, mapped_data): + description = str(source_data) + if description: + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + entity_name = state[ImportKey.ENTITY_NAME] + mapped_data["entities"][entity_class_name, entity_name].description = description + + def check_validity(self) -> None: + _require_one_of_parents(self, (EntityMapping, ElementMapping)) class EntityMetadataNameMapping(ImportMapping): @@ -458,118 +547,121 @@ def _import_row(self, source_data, state, mapped_data): class EntityMetadataValueMapping(ImportMapping): - """Maps entity metadata names. - - Cannot be used as the topmost mapping; must have :class:`EntityClassMapping`, :class:`EntityMapping` and :class:`EntityMetadataValueMapping` as parent. - """ + """Maps entity metadata names.""" MAP_TYPE = "EntityMetadataValue" ignorable = True def _import_row(self, source_data, state, mapped_data): entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] - if state[ImportKey.DIMENSION_COUNT]: - entity_byname = state[ImportKey.ELEMENT_NAMES] - else: - entity_byname = (state[ImportKey.ENTITY_NAME],) + entity_byname = _byname_from_mapped_data(entity_class_name, state, mapped_data) metadata_name = state[ImportKey.ENTITY_METADATA_NAME] metadata_value = state[ImportKey.ENTITY_METADATA_VALUE] = source_data mapped_data.setdefault("entity_metadata", {})[ entity_class_name, entity_byname, metadata_name, metadata_value ] = None + def check_validity(self) -> None: + _require_parent(self, EntityMetadataNameMapping) -class EntityGroupMapping(ImportEntitiesMixin, ImportMapping): - """Maps entity groups. - Cannot be used as the topmost mapping; must have :class:`EntityClassMapping` and :class:`EntityMapping` as parents. - """ +class EntityGroupMapping(ImportEntitiesMixin, ImportMapping): + """Maps entity groups.""" MAP_TYPE = "EntityGroup" def _import_row(self, source_data, state, mapped_data): entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] - group_name = state.get(ImportKey.ENTITY_NAME) - if group_name is None: - raise KeyError(ImportKey.GROUP_NAME) + group_name = state[ImportKey.ENTITY_NAME] member_name = str(source_data) mapped_data.setdefault("entity_groups", set()).add((entity_class_name, group_name, member_name)) if self.import_entities: - entities = mapped_data.setdefault("entities", {}) - entities[entity_class_name, group_name] = None - entities[entity_class_name, member_name] = None - raise KeyFix(ImportKey.MEMBER_NAME) + mapped_data["entities"][entity_class_name, member_name] = EntityRecord() + else: + try: + del mapped_data["entities"][entity_class_name, group_name] + except KeyError: + pass + def check_validity(self) -> None: + _require_parent(self, EntityMapping) -class EntityAlternativeActivityMapping(ImportMapping): - """Maps activity flags for entity alternative. - Cannot be used as the topmost mapping; must have :class:`EntityMapping` or :class:`ElementMapping`, - and :class:`AlternativeMapping` as parents. - """ +class EntityAlternativeActivityMapping(ImportMapping): + """Maps activity flags for entity alternative.""" MAP_TYPE = "EntityAlternativeActivity" ignorable = True def _import_row(self, source_data, state, mapped_data): - if source_data is None or source_data == "": + if source_data == "": return entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] - if state[ImportKey.DIMENSION_COUNT]: - entity_byname = state[ImportKey.ELEMENT_NAMES] - else: - entity_byname = (state[ImportKey.ENTITY_NAME],) + entity_byname = _byname_from_mapped_data(entity_class_name, state, mapped_data) alternative_name = state[ImportKey.ALTERNATIVE_NAME] mapped_data.setdefault("entity_alternatives", {})[ entity_class_name, entity_byname, alternative_name, source_data ] = None + def check_validity(self) -> None: + _require_parent(self, EntityMapping) + _require_parent(self, AlternativeMapping) -class DimensionMapping(ImportMapping): - """Maps dimensions. - Cannot be used as the topmost mapping; one of the parents must be :class:`EntityClassMapping`. - """ +class DimensionMapping(ImportMapping): + """Maps dimensions.""" MAP_TYPE = "Dimension" def _import_row(self, source_data, state, mapped_data): - if ImportKey.ENTITY_CLASS_NAME not in state: - raise KeyError(ImportKey.ENTITY_CLASS_NAME) - dimension_names = state[ImportKey.DIMENSION_NAMES] - if len(dimension_names) == state[ImportKey.DIMENSION_COUNT]: - return dimension_name = str(source_data) - dimension_names.append(dimension_name) - if len(dimension_names) == state[ImportKey.DIMENSION_COUNT]: - raise KeyFix(ImportKey.DIMENSION_NAMES) + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + mapped_data["entity_classes"][entity_class_name].dimensions.append(dimension_name) + def check_validity(self) -> None: + _require_parent(self, EntityClassMapping) -class ElementMapping(ImportEntitiesMixin, ImportMapping): - """Maps elements. - Cannot be used as the topmost mapping; must have :class:`EntityClassMapping` and :class:`EntityMapping` - as parents. - """ +class ElementMapping(ImportEntitiesMixin, ImportMapping): + """Maps elements.""" MAP_TYPE = "Element" def _import_row(self, source_data, state, mapped_data): - entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] - dimension_names = state[ImportKey.DIMENSION_NAMES] - if len(dimension_names) != state[ImportKey.DIMENSION_COUNT]: - raise KeyError(ImportKey.DIMENSION_NAMES) element_name = str(source_data) - element_names = state[ImportKey.ELEMENT_NAMES] = state[ImportKey.ELEMENT_NAMES] + (element_name,) + if isinstance(self._child, ElementMapping): + element_names = state.setdefault(ImportKey.ELEMENT_NAMES, []) + element_names.append(element_name) + return + element_names = state.pop(ImportKey.ELEMENT_NAMES, []) + element_names.append(element_name) + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + if ImportKey.ENTITY_NAME in state: + entity_name = state[ImportKey.ENTITY_NAME] + try: + record = mapped_data["entities"][entity_class_name, entity_name] + except KeyError: + pass + else: + if all(name == existing_name for name, existing_name in zip(element_names, record.elements)): + return + del state[ImportKey.ENTITY_NAME] + record = EntityRecord(element_names) + byname = tuple(record.elements) + mapped_entities = mapped_data.setdefault("entities", {}) + mapped_entities[entity_class_name, byname] = record + state[ImportKey.ENTITY_NAME] = byname if self.import_entities: - k = len(element_names) - 1 - dimension_name = dimension_names[k] - mapped_data.setdefault("entity_classes", {}).update({dimension_name: ()}) - mapped_data.setdefault("entities", {})[dimension_name, element_name] = None - if len(element_names) == state[ImportKey.DIMENSION_COUNT]: - mapped_data.setdefault("entities", {})[entity_class_name, tuple(element_names)] = None - raise KeyFix(ImportKey.ELEMENT_NAMES) - raise KeyError(ImportKey.ELEMENT_NAMES) + mapped_classes = mapped_data["entity_classes"] + class_record = mapped_classes[entity_class_name] + for element_name, dimension_name in zip(element_names, class_record.dimensions): + if dimension_name not in mapped_classes: + mapped_classes[dimension_name] = EntityClassRecord() + if (dimension_name, element_name) not in mapped_entities: + mapped_entities[dimension_name, element_name] = EntityRecord() + + def check_validity(self) -> None: + _require_enough_parents(self, DimensionMapping) class MetadataNameMapping(ImportMapping): @@ -582,10 +674,7 @@ def _import_row(self, source_data, state, mapped_data): class MetadataValueMapping(ImportMapping): - """Maps metadata values. - - Cannot be used as the topmost mapping; must have a metadata name mapping as one of parents. - """ + """Maps metadata values.""" MAP_TYPE = "MetadataValue" @@ -594,30 +683,88 @@ def _import_row(self, source_data, state, mapped_data): metadata_value = state[ImportKey.METADATA_VALUE] = str(source_data) mapped_data.setdefault("metadata", []).append((metadata_name, metadata_value)) + def check_validity(self) -> None: + _require_parent(self, MetadataNameMapping) -class ParameterDefinitionMapping(ImportMapping): - """Maps parameter definitions. - Cannot be used as the topmost mapping; must have an entity class mapping as one of parents. - """ +T = TypeVar("T") + + +@dataclass +class ValueRecord(Generic[T]): + index_names: list[str] = field(default_factory=list) + indexes: list[list] = field(default_factory=list) + values: list[T] = field(default_factory=list) + + def has_value(self) -> bool: + return bool(self.values) + + +@dataclass +class ArrayValueRecord(ValueRecord[Any]): + pass + + +@dataclass +class TimePatternValueRecord(ValueRecord[float]): + pass + + +@dataclass +class MapValueRecord(ValueRecord[Any]): + compress: bool = False + + +@dataclass +class TimeSeriesValueRecord(ValueRecord[float]): + ignore_year: bool = False + repeat: bool = False + indexes: list = field(default_factory=list) + + +@dataclass +class ParameterDefinitionRecord: + value_list_name: str | None = None + default_value: ValueRecord | None = None + description: str | None = None + + +class ParameterDefinitionMapping(ImportMapping): + """Maps parameter definitions.""" MAP_TYPE = "ParameterDefinition" def _import_row(self, source_data, state, mapped_data): entity_class_name = state.get(ImportKey.ENTITY_CLASS_NAME) parameter_name = state[ImportKey.PARAMETER_NAME] = str(source_data) - definition_extras = state[ImportKey.PARAMETER_DEFINITION_EXTRAS] = [] - parameter_definition_key = state[ImportKey.PARAMETER_DEFINITION] = entity_class_name, parameter_name - default_values = state.get(ImportKey.PARAMETER_DEFAULT_VALUES) - if default_values is None or parameter_definition_key not in default_values: - mapped_data.setdefault("parameter_definitions", {})[parameter_definition_key] = definition_extras + parameter_definition_key = entity_class_name, parameter_name + definitions = mapped_data.setdefault("parameter_definitions", {}) + if parameter_definition_key not in definitions: + definitions[parameter_definition_key] = ParameterDefinitionRecord() + def check_validity(self) -> None: + _require_parent(self, EntityClassMapping) -class ParameterTypeMapping(ImportMapping): - """Maps parameter types. - Cannot be used as the topmost mapping; must have a parameter definition mapping as one of parents. - """ +class ParameterDefinitionDescriptionMapping(ImportMapping): + """Maps parameter definition descriptions.""" + + MAP_TYPE = "ParameterDefinitionDescription" + ignorable = True + + def _import_row(self, source_data, state, mapped_data): + description = str(source_data) + if description: + entity_class_name = state.get(ImportKey.ENTITY_CLASS_NAME) + parameter_name = state[ImportKey.PARAMETER_NAME] + mapped_data["parameter_definitions"][entity_class_name, parameter_name].description = description + + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) + + +class ParameterTypeMapping(ImportMapping): + """Maps parameter types.""" MAP_TYPE = "ParameterType" @@ -629,12 +776,12 @@ def _import_row(self, source_data, state, mapped_data): parameter = state[ImportKey.PARAMETER_NAME] mapped_data.setdefault("parameter_types", []).append((entity_class, parameter, parameter_type)) + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) -class ParameterDefaultValueMapping(ImportMapping): - """Maps scalar (non-indexed) default values - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping` as parent. - """ +class ParameterDefaultValueMapping(ImportMapping): + """Maps scalar (non-indexed) default values.""" MAP_TYPE = "ParameterDefaultValue" @@ -642,89 +789,62 @@ def _import_row(self, source_data, state, mapped_data): default_value = source_data if default_value == "": return - parameter_definition_extras = state[ImportKey.PARAMETER_DEFINITION_EXTRAS] - parameter_definition_extras.append(default_value) - value_list_name = state.get(ImportKey.PARAMETER_VALUE_LIST_NAME) - if value_list_name is not None: - parameter_definition_extras.append(value_list_name) + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + parameter_name = state[ImportKey.PARAMETER_NAME] + mapped_data["parameter_definitions"][entity_class_name, parameter_name].default_value = default_value + + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) class ParameterDefaultValueTypeMapping(IndexedValueMixin, ImportMapping): + """Maps indexed default values.""" + MAP_TYPE = "ParameterDefaultValueType" def _import_row(self, source_data, state, mapped_data): - parameter_definition = state.get(ImportKey.PARAMETER_DEFINITION) - if parameter_definition is None: - # Don't catch errors here, this one's invisible - return - default_values = state.setdefault(ImportKey.PARAMETER_DEFAULT_VALUES, {}) - if parameter_definition in default_values: + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + parameter_name = state[ImportKey.PARAMETER_NAME] + key = (entity_class_name, parameter_name) + definition_record = mapped_data["parameter_definitions"][key] + if definition_record.default_value is not None: + state[ImportKey.PARAMETER_DEFAULT_VALUE_RECORD] = definition_record.default_value return - value_type = str(source_data) - default_value = default_values[parameter_definition] = {"type": value_type} - if self.compress and value_type == "map": - default_value["compress"] = self.compress - if self.options and value_type == "time_series": - default_value["options"] = self.options - parameter_definition_extras = state[ImportKey.PARAMETER_DEFINITION_EXTRAS] - parameter_definition_extras.append(default_value) - value_list_name = state.get(ImportKey.PARAMETER_VALUE_LIST_NAME) - if value_list_name is not None: - parameter_definition_extras.append(value_list_name) - + record = self._make_value_record(source_data) + definition_record.default_value = record + state[ImportKey.PARAMETER_DEFAULT_VALUE_RECORD] = record -class IndexNameMappingBase(ImportMapping): - """Base class for index name mappings.""" + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) - _STATE_KEY = NotImplemented - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._id = None +class DefaultValueIndexNameMapping(ImportMapping): + """Maps default value index names.""" - def _value_key(self, state): - raise NotImplementedError() + MAP_TYPE = "DefaultValueIndexName" def _import_row(self, source_data, state, mapped_data): - values = state[self._STATE_KEY] - value = values[self._value_key(state)] - if self._id is None: - self._id = 0 - current = self - while True: - if current.parent is None: - break - current = current.parent - if isinstance(current, type(self)): - self._id += 1 - value.setdefault("index_names", {})[self._id] = source_data - - -class DefaultValueIndexNameMapping(IndexNameMappingBase): - """Maps default value index names. - - Cannot be used as the topmost mapping; must have a :class:`ParameterDefaultValueTypeMapping` as parent. - """ - - MAP_TYPE = "DefaultValueIndexName" - _STATE_KEY = ImportKey.PARAMETER_DEFAULT_VALUES + if ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES in state: + i = len(state[ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES]) + state.setdefault(ImportKey.PARAMETER_DEFAULT_VALUE_INDEX_NAMES, {})[i] = str(source_data) + else: + state[ImportKey.PARAMETER_DEFAULT_VALUE_INDEX_NAMES] = {0: str(source_data)} - def _value_key(self, state): - return _default_value_key(state) + def check_validity(self) -> None: + _require_parent(self, ParameterDefaultValueTypeMapping) class ParameterDefaultValueIndexMapping(ImportMapping): - """Maps default value indexes. - - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping` as parent. - """ + """Maps default value indexes.""" MAP_TYPE = "ParameterDefaultValueIndex" def _import_row(self, source_data, state, mapped_data): - _ = state[ImportKey.PARAMETER_NAME] - index = source_data - state.setdefault(ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES, []).append(index) + state.setdefault(ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES, []).append(source_data) + + def check_validity(self) -> None: + _require_parent(self, ParameterDefaultValueTypeMapping) + _require_enough_parents(self, DefaultValueIndexNameMapping) class ExpandedParameterDefaultValueMapping(ImportMapping): @@ -732,69 +852,90 @@ class ExpandedParameterDefaultValueMapping(ImportMapping): Whenever this mapping is a child of :class:`ParameterDefaultValueIndexMapping`, it maps individual values of indexed parameters. - - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping` as parent. """ MAP_TYPE = "ExpandedDefaultValue" def _import_row(self, source_data, state, mapped_data): - values = state.setdefault(ImportKey.PARAMETER_DEFAULT_VALUES, {}) - value = values[_default_value_key(state)] - val = source_data - data = value.setdefault("data", []) - if value["type"] == "array": - data.append(val) - return - indexes = state.pop(ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES) - data.append(indexes + [val]) + record = state[ImportKey.PARAMETER_DEFAULT_VALUE_RECORD] + record.values.append(source_data) + try: + record.indexes.append(state.pop(ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES)) + except KeyError: + pass + try: + index_names = state.pop(ImportKey.PARAMETER_DEFAULT_VALUE_INDEX_NAMES) + except KeyError: + pass + else: + if record.indexes: + n_indexes = len(record.indexes[-1]) + if n_indexes == len(index_names): + record.index_names = list(index_names.values()) + else: + name_list = [] + for i in range(n_indexes): + name_list.append(index_names.get(i)) + record.index_names = name_list + else: + # Arrays + record.index_names = [index_names[0]] def _skip_row(self, state): - state.pop(ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES, None) + try: + del state[ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES] + except KeyError: + pass + def check_validity(self) -> None: + _require_parent(self, ParameterDefaultValueTypeMapping) -class ParameterValueMapping(ImportMapping): - """Maps scalar (non-indexed) parameter values. - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping`, an entity mapping and - an :class:`AlternativeMapping` as parents. - """ +class ParameterValueMapping(ImportMapping): + """Maps scalar (non-indexed) parameter values.""" MAP_TYPE = "ParameterValue" def _import_row(self, source_data, state, mapped_data): - value = source_data - if value == "": + if source_data == "": return - entity_class_name, entity_byname, parameter_name, alternative_name = _parameter_value_key(state) - parameter_value = [entity_class_name, entity_byname, parameter_name, value] - if alternative_name is not None: - parameter_value.append(alternative_name) - mapped_data.setdefault("parameter_values", []).append(parameter_value) + entity_class_name = state.get(ImportKey.ENTITY_CLASS_NAME) + entity_name = state[ImportKey.ENTITY_NAME] + entity_byname = entity_name if isinstance(entity_name, tuple) else (entity_name,) + parameter_name = state[ImportKey.PARAMETER_NAME] + alternative_name = state.get(ImportKey.ALTERNATIVE_NAME) + mapped_data.setdefault("parameter_values", {})[ + entity_class_name, entity_byname, parameter_name, alternative_name + ] = source_data + + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) + _require_one_of_parents(self, (EntityMapping, ElementMapping)) class ParameterValueTypeMapping(IndexedValueMixin, ImportMapping): + """Maps indexed parameter values.""" + MAP_TYPE = "ParameterValueType" def _import_row(self, source_data, state, mapped_data): - if ImportKey.PARAMETER_NAME not in state: - # Don't catch errors here, this one's invisible - return - key = _parameter_value_key(state) - values = state.setdefault(ImportKey.PARAMETER_VALUES, {}) - if key in values: + entity_class_name = state.get(ImportKey.ENTITY_CLASS_NAME) + entity_name = state[ImportKey.ENTITY_NAME] + entity_byname = entity_name if isinstance(entity_name, tuple) else (entity_name,) + parameter_name = state[ImportKey.PARAMETER_NAME] + alternative_name = state.get(ImportKey.ALTERNATIVE_NAME) + key = entity_class_name, entity_byname, parameter_name, alternative_name + mapped_values = mapped_data.setdefault("parameter_values", {}) + if key in mapped_values: + state[ImportKey.PARAMETER_VALUE_RECORD] = mapped_values[key] return - entity_class_name, entity_byname, parameter_name, alternative_name = key - value_type = str(source_data) - value = values[key] = {"type": value_type} # See import_mapping.generator._parameter_value_from_dict() - if self.compress and value_type == "map": - value["compress"] = self.compress - if self.options and value_type == "time_series": - value["options"] = self.options - parameter_value = [entity_class_name, entity_byname, parameter_name, value] - if alternative_name is not None: - parameter_value.append(alternative_name) - mapped_data.setdefault("parameter_values", []).append(parameter_value) + record = self._make_value_record(source_data) + mapped_values[key] = record + state[ImportKey.PARAMETER_VALUE_RECORD] = record + + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) + _require_one_of_parents(self, (EntityMapping, ElementMapping)) class ParameterValueMetadataNameMapping(ImportMapping): @@ -808,21 +949,14 @@ def _import_row(self, source_data, state, mapped_data): class ParameterValueMetadataValueMapping(ImportMapping): - """Maps parameter value metadata values. - - Cannot be used as the topmost mapping; must have a :class:`ParameterValueMapping` or - a :class:`ParameterValueTypeMapping` and :class:`ParameterValueMetadataName` as parents. - """ + """Maps parameter value metadata values.""" MAP_TYPE = "ParameterValueMetadataValue" ignorable = True def _import_row(self, source_data, state, mapped_data): entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] - if state[ImportKey.DIMENSION_COUNT]: - entity_byname = state[ImportKey.ELEMENT_NAMES] - else: - entity_byname = (state[ImportKey.ENTITY_NAME],) + entity_byname = _byname_from_mapped_data(entity_class_name, state, mapped_data) parameter_name = state[ImportKey.PARAMETER_NAME] alternative_name = state[ImportKey.ALTERNATIVE_NAME] metadata_name = state[ImportKey.PARAMETER_VALUE_METADATA_NAME] @@ -831,33 +965,37 @@ def _import_row(self, source_data, state, mapped_data): entity_class_name, entity_byname, parameter_name, metadata_name, metadata_value, alternative_name ] = None + def check_validity(self) -> None: + _require_parent(self, ParameterValueMetadataNameMapping) -class ParameterValueIndexMapping(ImportMapping): - """Maps parameter value indexes. - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping`, an entity mapping and - an :class:`ParameterValueTypeMapping` as parents. - """ +class ParameterValueIndexMapping(ImportMapping): + """Maps parameter value indexes.""" MAP_TYPE = "ParameterValueIndex" def _import_row(self, source_data, state, mapped_data): - _ = state[ImportKey.PARAMETER_NAME] - index = source_data - state.setdefault(ImportKey.PARAMETER_VALUE_INDEXES, []).append(index) + state.setdefault(ImportKey.PARAMETER_VALUE_INDEXES, []).append(source_data) + def check_validity(self) -> None: + _require_parent(self, ParameterValueTypeMapping) + _require_enough_parents(self, IndexNameMapping) -class IndexNameMapping(IndexNameMappingBase): - """Maps index names for indexed parameter values. - Cannot be used as the topmost mapping; must have an :class:`ParameterValueTypeMapping` as a parent. - """ +class IndexNameMapping(ImportMapping): + """Maps index names for indexed parameter values.""" MAP_TYPE = "IndexName" - _STATE_KEY = ImportKey.PARAMETER_VALUES - def _value_key(self, state): - return _parameter_value_key(state) + def _import_row(self, source_data, state, mapped_data): + if ImportKey.PARAMETER_VALUE_INDEXES in state: + i = len(state[ImportKey.PARAMETER_VALUE_INDEXES]) + state.setdefault(ImportKey.PARAMETER_VALUE_INDEX_NAMES, {})[i] = str(source_data) + else: + state[ImportKey.PARAMETER_VALUE_INDEX_NAMES] = {0: str(source_data)} + + def check_validity(self) -> None: + _require_parent(self, ParameterValueTypeMapping) class ExpandedParameterValueMapping(ImportMapping): @@ -865,49 +1003,63 @@ class ExpandedParameterValueMapping(ImportMapping): Whenever this mapping is a child of :class:`ParameterValueIndexMapping`, it maps individual values of indexed parameters. - - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping`, an entity mapping and - an :class:`ParameterValueTypeMapping` as parents. """ MAP_TYPE = "ExpandedValue" def _import_row(self, source_data, state, mapped_data): - values = state.setdefault(ImportKey.PARAMETER_VALUES, {}) - value = values[_parameter_value_key(state)] - data = value.setdefault("data", []) - if value["type"] == "array": - data.append(source_data) - return - indexes = state.pop(ImportKey.PARAMETER_VALUE_INDEXES) - data.append(indexes + [source_data]) + record = state[ImportKey.PARAMETER_VALUE_RECORD] + record.values.append(source_data) + try: + record.indexes.append(state.pop(ImportKey.PARAMETER_VALUE_INDEXES)) + except KeyError: + pass + try: + index_names = state.pop(ImportKey.PARAMETER_VALUE_INDEX_NAMES) + except KeyError: + pass + else: + if record.indexes: + n_indexes = len(record.indexes[-1]) + if n_indexes == len(index_names): + record.index_names = list(index_names.values()) + else: + name_list = [] + for i in range(n_indexes): + name_list.append(index_names.get(i)) + record.index_names = name_list + else: + # Arrays + record.index_names = [index_names[0]] def _skip_row(self, state): - state.pop(ImportKey.PARAMETER_VALUE_INDEXES, None) + try: + del state[ImportKey.PARAMETER_VALUE_INDEXES] + except KeyError: + pass + def check_validity(self) -> None: + _require_parent(self, ParameterValueTypeMapping) -class ParameterValueListMapping(ImportMapping): - """Maps parameter value list names. - Can be used as the topmost mapping; in case the mapping has a :class:`ParameterDefinitionMapping` as parent, - yields value list name for that parameter definition. - """ +class ParameterValueListMapping(ImportMapping): + """Maps parameter value list names.""" MAP_TYPE = "ParameterValueList" def _import_row(self, source_data, state, mapped_data): - if self.parent is not None: - # Trigger a KeyError in case there's no parameter definition, so check_validity() registers the issue - _ = state[ImportKey.PARAMETER_DEFINITION] - state[ImportKey.PARAMETER_VALUE_LIST_NAME] = str(source_data) + value_list_name = str(source_data) + if not value_list_name: + return + state[ImportKey.PARAMETER_VALUE_LIST_NAME] = value_list_name + if ImportKey.PARAMETER_NAME in state: + parameter_name = state[ImportKey.PARAMETER_NAME] + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + mapped_data["parameter_definitions"][entity_class_name, parameter_name].value_list_name = value_list_name class ParameterValueListValueMapping(ImportMapping): - """Maps parameter value list values. - - Cannot be used as the topmost mapping; must have a :class:`ParameterValueListMapping` as parent. - - """ + """Maps parameter value list values.""" MAP_TYPE = "ParameterValueListValue" @@ -918,12 +1070,12 @@ def _import_row(self, source_data, state, mapped_data): value_list_name = state[ImportKey.PARAMETER_VALUE_LIST_NAME] mapped_data.setdefault("parameter_value_lists", []).append([value_list_name, list_value]) + def check_validity(self) -> None: + _require_parent(self, ParameterValueListMapping) -class AlternativeMapping(ImportMapping): - """Maps alternatives. - Can be used as the topmost mapping. - """ +class AlternativeMapping(ImportMapping): + """Maps alternatives.""" MAP_TYPE = "Alternative" @@ -932,26 +1084,37 @@ def _import_row(self, source_data, state, mapped_data): mapped_data.setdefault("alternatives", set()).add(alternative) -class ScenarioMapping(ImportMapping): - """Maps scenarios. +class AlternativeDescriptionMapping(ImportMapping): + """Maps alternative descriptions.""" + + MAP_TYPE = "AlternativeDescription" + ignorable = True + + def _import_row(self, source_data, state, mapped_data): + description = str(source_data) + if description: + alternative = state[ImportKey.ALTERNATIVE_NAME] + alternative_data = mapped_data["alternatives"] + alternative_data.discard(alternative) + alternative_data.add((alternative, description)) + + def check_validity(self) -> None: + _require_parent(self, AlternativeMapping) - Can be used as the topmost mapping. - """ + +class ScenarioMapping(ImportMapping): + """Maps scenarios.""" MAP_TYPE = "Scenario" def _import_row(self, source_data, state, mapped_data): scenario = str(source_data) state[ImportKey.SCENARIO_NAME] = scenario - if self._child is None: - mapped_data.setdefault("scenarios", set()).add((scenario,)) + mapped_data.setdefault("scenarios", set()).add((scenario,)) class ScenarioAlternativeMapping(ImportMapping): - """Maps scenario alternatives. - - Cannot be used as the topmost mapping; must have a :class:`ScenarioMapping` as parent. - """ + """Maps scenario alternatives.""" MAP_TYPE = "ScenarioAlternative" @@ -963,12 +1126,12 @@ def _import_row(self, source_data, state, mapped_data): scen_alt = state[ImportKey.SCENARIO_ALTERNATIVE] = [scenario, alternative] mapped_data.setdefault("scenario_alternatives", []).append(scen_alt) + def check_validity(self) -> None: + _require_parent(self, ScenarioMapping) -class ScenarioBeforeAlternativeMapping(ImportMapping): - """Maps scenario 'before' alternatives. - Cannot be used as the topmost mapping; must have a :class:`ScenarioAlternativeMapping` as parent. - """ +class ScenarioBeforeAlternativeMapping(ImportMapping): + """Maps scenario 'before' alternatives.""" MAP_TYPE = "ScenarioBeforeAlternative" @@ -977,6 +1140,27 @@ def _import_row(self, source_data, state, mapped_data): alternative = str(source_data) scen_alt.append(alternative) + def check_validity(self) -> None: + _require_parent(self, ScenarioAlternativeMapping) + + +class ScenarioDescriptionMapping(ImportMapping): + """Maps scenario descriptions.""" + + MAP_TYPE = "ScenarioDescription" + ignorable: ClassVar[bool] = True + + def _import_row(self, source_data, state, mapped_data): + description = str(source_data) + if description: + scenario = state[ImportKey.SCENARIO_NAME] + scenario_data = mapped_data["scenarios"] + scenario_data.discard((scenario,)) + scenario_data.add((scenario, description)) + + def check_validity(self) -> None: + _require_parent(self, ScenarioMapping) + def default_import_mapping(map_type: str) -> ImportMapping: """Creates default mappings for given map type. @@ -1001,42 +1185,46 @@ def default_import_mapping(map_type: str) -> ImportMapping: return make_root_mapping() -def _default_entity_class_mapping(): +def _default_entity_class_mapping() -> EntityClassMapping: """Creates default entity class mappings. Returns: - EntityClassMapping: root mapping + root mapping """ root_mapping = EntityClassMapping(Position.hidden) - root_mapping.child = EntityMapping(Position.hidden) + description_mapping = root_mapping.child = EntityClassDescriptionMapping(Position.hidden) + entity_mapping = description_mapping.child = EntityMapping(Position.hidden) + entity_mapping.child = EntityDescriptionMapping(Position.hidden) return root_mapping -def _default_alternative_mapping(): +def _default_alternative_mapping() -> AlternativeMapping: """Creates default alternative mappings. Returns: - AlternativeMapping: root mapping + root mapping """ root_mapping = AlternativeMapping(Position.hidden) + root_mapping.child = AlternativeDescriptionMapping(Position.hidden) return root_mapping -def _default_scenario_mapping(): +def _default_scenario_mapping() -> ScenarioMapping: """Creates default scenario mappings. Returns: - ScenarioMapping: root mapping + root mapping """ root_mapping = ScenarioMapping(Position.hidden) + root_mapping.child = ScenarioDescriptionMapping(Position.hidden) return root_mapping -def _default_scenario_alternative_mapping(): +def _default_scenario_alternative_mapping() -> ScenarioMapping: """Creates default scenario alternative mappings. Returns: - ScenarioAlternativeMapping: root mapping + root mapping """ root_mapping = ScenarioMapping(Position.hidden) root_mapping.child = ScenarioAlternativeMapping(Position.hidden) @@ -1108,7 +1296,9 @@ def from_dict(serialized): klass.MAP_TYPE: klass for klass in ( EntityClassMapping, + EntityClassDescriptionMapping, EntityMapping, + EntityDescriptionMapping, EntityMetadataNameMapping, EntityMetadataValueMapping, EntityGroupMapping, @@ -1116,6 +1306,7 @@ def from_dict(serialized): ElementMapping, EntityAlternativeActivityMapping, ParameterDefinitionMapping, + ParameterDefinitionDescriptionMapping, ParameterTypeMapping, ParameterDefaultValueMapping, ParameterDefaultValueTypeMapping, @@ -1134,9 +1325,11 @@ def from_dict(serialized): MetadataNameMapping, MetadataValueMapping, AlternativeMapping, + AlternativeDescriptionMapping, ScenarioMapping, ScenarioAlternativeMapping, ScenarioBeforeAlternativeMapping, + ScenarioDescriptionMapping, ) } legacy_mappings = { @@ -1182,35 +1375,35 @@ def from_dict(serialized): return unflatten(flattened) -def _parameter_value_key(state): +def _parameter_value_key(state: State, mapped_data: SemiMappedData) -> tuple[str, tuple[str, ...], str, str]: """Creates parameter value's key from current state. Args: - state (dict): import state + state: import state Returns: - tuple of str: class name, entity byname, parameter name, and alternative name + class name, entity byname, parameter name, and alternative name """ entity_class_name = state.get(ImportKey.ENTITY_CLASS_NAME) - if state.get(ImportKey.DIMENSION_COUNT): - element_names = state[ImportKey.ELEMENT_NAMES] - if len(element_names) != state[ImportKey.DIMENSION_COUNT]: - raise KeyError(ImportKey.ELEMENT_NAMES) - entity_byname = element_names - else: - entity_byname = state[ImportKey.ENTITY_NAME] + entity_byname = _byname_from_mapped_data(entity_class_name, state, mapped_data) parameter_name = state[ImportKey.PARAMETER_NAME] alternative_name = state.get(ImportKey.ALTERNATIVE_NAME) return entity_class_name, entity_byname, parameter_name, alternative_name -def _default_value_key(state): +def _default_value_key(state: State) -> tuple[str, str]: """Creates parameter default value's key from current state. Args: - state (dict): import state + state: import state Returns: - tuple of str: class name and parameter name + class name and parameter name """ return state[ImportKey.ENTITY_CLASS_NAME], state[ImportKey.PARAMETER_NAME] + + +def _byname_from_mapped_data(entity_class_name: str, state: State, mapped_data: SemiMappedData) -> tuple[str, ...]: + entity_name = state[ImportKey.ENTITY_NAME] + entity_record = mapped_data["entities"][entity_class_name, entity_name] + return tuple(entity_record.elements) if entity_record.elements else (entity_name,) diff --git a/spinedb_api/import_mapping/import_mapping_compat.py b/spinedb_api/import_mapping/import_mapping_compat.py index c8afaaed..a629e7e1 100644 --- a/spinedb_api/import_mapping/import_mapping_compat.py +++ b/spinedb_api/import_mapping/import_mapping_compat.py @@ -20,21 +20,18 @@ EntityClassMapping, EntityGroupMapping, EntityMapping, - EntityMetadataNameMapping, - EntityMetadataValueMapping, ExpandedParameterDefaultValueMapping, ExpandedParameterValueMapping, IndexNameMapping, ParameterDefaultValueIndexMapping, ParameterDefaultValueMapping, ParameterDefaultValueTypeMapping, + ParameterDefinitionDescriptionMapping, ParameterDefinitionMapping, ParameterValueIndexMapping, ParameterValueListMapping, ParameterValueListValueMapping, ParameterValueMapping, - ParameterValueMetadataNameMapping, - ParameterValueMetadataValueMapping, ParameterValueTypeMapping, Position, ScenarioAlternativeMapping, @@ -132,7 +129,6 @@ def _scenario_alternative_mapping_from_dict(map_dict): def _object_class_mapping_from_dict(map_dict): name = map_dict.get("name") entities = map_dict.get("objects", map_dict.get("object")) - object_metadata = map_dict.get("object_metadata", None) parameters = map_dict.get("parameters") skip_columns = map_dict.get("skip_columns", []) read_start_row = map_dict.get("read_start_row", 0) @@ -196,7 +192,8 @@ def parameter_mapping_from_dict(map_dict): if map_type == "ParameterDefinition": default_value_dict = map_dict.get("default_value") value_list_name = map_dict.get("parameter_value_list_name") - param_def_mapping.child = value_list_mapping = ParameterValueListMapping(*_pos_and_val(value_list_name)) + description = param_def_mapping.child = ParameterDefinitionDescriptionMapping(Position.hidden) + description.child = value_list_mapping = ParameterValueListMapping(*_pos_and_val(value_list_name)) value_list_mapping.child = parameter_default_value_mapping_from_dict(default_value_dict) return param_def_mapping alternative_name = map_dict.get("alternative_name") diff --git a/spinedb_api/spine_io/importers/csv_reader.py b/spinedb_api/spine_io/importers/csv_reader.py index de2986b2..90d570e1 100644 --- a/spinedb_api/spine_io/importers/csv_reader.py +++ b/spinedb_api/spine_io/importers/csv_reader.py @@ -10,7 +10,7 @@ # this program. If not, see . ###################################################################################################################### -""" Contains CSVReader class and helper functions. """ +"""Contains CSVReader class and helper functions.""" import csv from itertools import islice @@ -44,6 +44,7 @@ class CSVReader(Reader): def __init__(self, settings): super().__init__(settings) self._filename = None + self._detector = chardet.UniversalDetector(max_bytes=1024) def connect_to_source(self, source, **extras): """saves filepath @@ -67,7 +68,13 @@ def get_tables_and_properties(self): options = {"skip": 0} # try to find options for file with open(self._filename, "rb") as input_file: - sniff_result = chardet.detect(input_file.read(1024)) + for line in input_file: + self._detector.feed(line) + if self._detector.done: + break + self._detector.close() + sniff_result = self._detector.result + self._detector.reset() sniffed_encoding = sniff_result["encoding"] if sniffed_encoding is not None: sniffed_encoding = sniffed_encoding.lower() diff --git a/tests/import_mapping/test_generator.py b/tests/import_mapping/test_generator.py index ab88c118..09c24bd5 100644 --- a/tests/import_mapping/test_generator.py +++ b/tests/import_mapping/test_generator.py @@ -14,7 +14,9 @@ import unittest from spinedb_api import Array, DateTime, Duration, Map from spinedb_api.import_mapping.generator import get_mapped_data +from spinedb_api.import_mapping.import_mapping import EntityClassMapping, default_import_mapping from spinedb_api.import_mapping.type_conversion import value_to_convert_spec +from spinedb_api.mapping import to_dict, unflatten class TestGetMappedData(unittest.TestCase): @@ -67,10 +69,10 @@ def test_returns_appropriate_error_if_last_row_is_empty(self): mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Object",)], - "parameter_values": [["Object", "data", "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], - "parameter_definitions": [("Object", "Parameter")], - "entities": [("Object", "data")], + "entity_classes": [["Object", []]], + "parameter_values": [["Object", ("data",), "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], + "parameter_definitions": [["Object", "Parameter"]], + "entities": [["Object", "data"]], }, ) @@ -99,10 +101,10 @@ def test_convert_functions_get_expanded_over_last_defined_column_in_pivoted_data mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Object",)], - "parameter_values": [["Object", "data", "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], - "parameter_definitions": [("Object", "Parameter")], - "entities": [("Object", "data")], + "entity_classes": [["Object", []]], + "parameter_values": [["Object", ("data",), "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], + "parameter_definitions": [["Object", "Parameter"]], + "entities": [["Object", "data"]], }, ) @@ -130,10 +132,10 @@ def test_read_start_row_skips_rows_in_pivoted_data(self): self.assertEqual( mapped_data, { - "entity_classes": [("klass",)], - "parameter_values": [["klass", "kloss", "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], - "parameter_definitions": [("klass", "Parameter_2")], - "entities": [("klass", "kloss")], + "entity_classes": [["klass", []]], + "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], + "parameter_definitions": [["klass", "Parameter_2"]], + "entities": [["klass", "kloss"]], }, ) @@ -184,10 +186,9 @@ def test_map_without_values_is_ignored_and_not_interpreted_as_null(self): mapped_data, { "alternatives": {"base"}, - "entity_classes": [("o",)], - "parameter_definitions": [("o", "parameter_name")], - "parameter_values": [], - "entities": [("o", "o1")], + "entity_classes": [["o", []]], + "parameter_definitions": [["o", "parameter_name"]], + "entities": [["o", "o1"]], }, ) @@ -220,17 +221,17 @@ def test_import_object_works_with_multiple_relationship_object_imports(self): mapped_data, { "alternatives": {"base"}, - "entity_classes": [("o",), ("q",), ("o_to_q", ("o", "q"))], + "entity_classes": [["o_to_q", ["o", "q"]], ["o", []], ["q", []]], "entities": [ - ("o", "o1"), - ("q", "q1"), - ("o_to_q", ("o1", "q1")), - ("o", "o2"), - ("q", "q2"), - ("o_to_q", ("o2", "q2")), - ("o_to_q", ("o1", "q2")), + ["o_to_q", ["o1", "q1"]], + ["o", "o1"], + ["q", "q1"], + ["o_to_q", ["o2", "q2"]], + ["o", "o2"], + ["q", "q2"], + ["o_to_q", ["o1", "q2"]], ], - "parameter_definitions": [("o_to_q", "param")], + "parameter_definitions": [["o_to_q", "param"]], "parameter_values": [ ["o_to_q", ("o1", "q1"), "param", Map(["t1", "t2"], [11, 22], index_name="time"), "base"], ["o_to_q", ("o2", "q2"), "param", Map(["t1", "t2"], [33, 44], index_name="time"), "base"], @@ -264,10 +265,10 @@ def test_default_convert_function_in_column_convert_functions(self): self.assertEqual( mapped_data, { - "entity_classes": [("klass",)], - "parameter_values": [["klass", "kloss", "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], - "parameter_definitions": [("klass", "Parameter_2")], - "entities": [("klass", "kloss")], + "entity_classes": [["klass", []]], + "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], + "parameter_definitions": [["klass", "Parameter_2"]], + "entities": [["klass", "kloss"]], }, ) @@ -292,10 +293,10 @@ def test_identity_function_is_used_as_convert_function_when_no_convert_functions self.assertEqual( mapped_data, { - "entity_classes": [("klass",)], - "parameter_values": [["klass", "kloss", "Parameter_2", Map(["T1", "T2"], ["2.3", "23.0"])]], - "parameter_definitions": [("klass", "Parameter_2")], - "entities": [("klass", "kloss")], + "entity_classes": [["klass", []]], + "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], ["2.3", "23.0"])]], + "parameter_definitions": [["klass", "Parameter_2"]], + "entities": [["klass", "kloss"]], }, ) @@ -322,10 +323,10 @@ def test_last_convert_function_gets_used_as_default_convert_function_when_no_def self.assertEqual( mapped_data, { - "entity_classes": [("klass",)], - "parameter_values": [["klass", "kloss", "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], - "parameter_definitions": [("klass", "Parameter_2")], - "entities": [("klass", "kloss")], + "entity_classes": [["klass", []]], + "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], + "parameter_definitions": [["klass", "Parameter_2"]], + "entities": [["klass", "kloss"]], }, ) @@ -355,13 +356,13 @@ def test_array_parameters_get_imported_correctly_when_objects_are_in_header(self mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("class",)], + "entity_classes": [["class", []]], "parameter_values": [ - ["class", "object_1", "param", Array([-1.1, 1.1]), "Base"], - ["class", "object_2", "param", Array([2.3, -2.3]), "Base"], + ["class", ("object_1",), "param", Array([-1.1, 1.1]), "Base"], + ["class", ("object_2",), "param", Array([2.3, -2.3]), "Base"], ], - "parameter_definitions": [("class", "param")], - "entities": [("class", "object_1"), ("class", "object_2")], + "parameter_definitions": [["class", "param"]], + "entities": [["class", "object_1"], ["class", "object_2"]], }, ) @@ -391,13 +392,13 @@ def test_arrays_get_imported_correctly_when_objects_are_in_header_and_alternativ mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Gadget",)], + "entity_classes": [["Gadget", []]], "parameter_values": [ - ["Gadget", "object_1", "data", Array([-1.1, 1.1]), "Base"], - ["Gadget", "object_2", "data", Array([2.3, -2.3]), "Base"], + ["Gadget", ("object_1",), "data", Array([-1.1, 1.1]), "Base"], + ["Gadget", ("object_2",), "data", Array([2.3, -2.3]), "Base"], ], - "parameter_definitions": [("Gadget", "data")], - "entities": [("Gadget", "object_1"), ("Gadget", "object_2")], + "parameter_definitions": [["Gadget", "data"]], + "entities": [["Gadget", "object_1"], ["Gadget", "object_2"]], }, ) @@ -426,15 +427,15 @@ def test_header_position_is_ignored_in_last_mapping_if_other_mappings_are_in_hea mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Data",)], + "entity_classes": [["Data", []]], "parameter_values": [ - ["Data", "d1", "parameter1", 1.1, "Base"], - ["Data", "d1", "parameter2", -2.3, "Base"], - ["Data", "d2", "parameter1", -1.1, "Base"], - ["Data", "d2", "parameter2", 2.3, "Base"], + ["Data", ("d1",), "parameter1", 1.1, "Base"], + ["Data", ("d1",), "parameter2", -2.3, "Base"], + ["Data", ("d2",), "parameter1", -1.1, "Base"], + ["Data", ("d2",), "parameter2", 2.3, "Base"], ], - "parameter_definitions": [("Data", "parameter1"), ("Data", "parameter2")], - "entities": [("Data", "d1"), ("Data", "d2")], + "parameter_definitions": [["Data", "parameter1"], ["Data", "parameter2"]], + "entities": [["Data", "d1"], ["Data", "d2"]], }, ) @@ -496,13 +497,13 @@ def test_importing_multidimensional_class_when_there_is_an_extra_column(self): { "alternatives": {"Base"}, "entities": [ - ("unit", "Dyson sphere"), - ("node", "Gamma Ceti"), - ("node", "Ring world"), - ("unit__node__node", ("Dyson sphere", "Gamma Ceti", "Ring world")), + ["unit__node__node", ["Dyson sphere", "Gamma Ceti", "Ring world"]], + ["unit", "Dyson sphere"], + ["node", "Gamma Ceti"], + ["node", "Ring world"], ], - "entity_classes": [("unit",), ("node",), ("unit__node__node", ("unit", "node", "node"))], - "parameter_definitions": [("unit__node__node", "flow")], + "entity_classes": [["unit__node__node", ["unit", "node", "node"]], ["unit", []], ["node", []]], + "parameter_definitions": [["unit__node__node", "flow"]], "parameter_values": [ ["unit__node__node", ("Dyson sphere", "Gamma Ceti", "Ring world"), "flow", 23.3, "Base"] ], @@ -532,10 +533,10 @@ def test_importing_empty_rows_does_unnecessarily_not_repeat_mapped_data(self): self.assertEqual( mapped_data, { - "entities": [("Generator", "MyHydroGenerator")], - "entity_classes": [("Generator",)], - "parameter_definitions": [("Generator", "Type")], - "parameter_values": [["Generator", "MyHydroGenerator", "Type", "Hydro"]], + "entities": [["Generator", "MyHydroGenerator"]], + "entity_classes": [["Generator", []]], + "parameter_definitions": [["Generator", "Type"]], + "parameter_values": [["Generator", ("MyHydroGenerator",), "Type", "Hydro"]], }, ) @@ -581,25 +582,25 @@ def test_pivoted_mapping_has_position_outside_source_bounds(self): mapped_data, { "entities": [ - ("connection", "A1"), - ("node", "B1"), - ("node", "C1"), - ("connection__node__node", ("A1", "B1", "C1")), - ("connection", "A2"), - ("node", "B2"), - ("node", "C2"), - ("connection__node__node", ("A2", "B2", "C2")), - ("connection", "A3"), - ("node", "B3"), - ("node", "C3"), - ("connection__node__node", ("A3", "B3", "C3")), + ["connection__node__node", ["A1", "B1", "C1"]], + ["connection", "A1"], + ["node", "B1"], + ["node", "C1"], + ["connection__node__node", ["A2", "B2", "C2"]], + ["connection", "A2"], + ["node", "B2"], + ["node", "C2"], + ["connection__node__node", ["A3", "B3", "C3"]], + ["connection", "A3"], + ["node", "B3"], + ["node", "C3"], ], "entity_classes": [ - ("connection",), - ("node",), - ("connection__node__node", ("connection", "node", "node")), + ["connection__node__node", ["connection", "node", "node"]], + ["connection", []], + ["node", []], ], - "parameter_definitions": [("connection__node__node", "flow_t")], + "parameter_definitions": [["connection__node__node", "flow_t"]], "parameter_values": [ [ "connection__node__node", @@ -675,15 +676,15 @@ def test_import_datetime_values(self): mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), - ("Object", "o2"), + ["Object", "o1"], + ["Object", "o2"], ], - "parameter_definitions": [("Object", "t")], + "parameter_definitions": [["Object", "t"]], "parameter_values": [ - ["Object", "o1", "t", DateTime("2024-06-24T09:00:00"), "Base"], - ["Object", "o2", "t", DateTime("2024-06-24T00:00:00"), "Base"], + ["Object", ("o1",), "t", DateTime("2024-06-24T09:00:00"), "Base"], + ["Object", ("o2",), "t", DateTime("2024-06-24T00:00:00"), "Base"], ], }, ) @@ -710,15 +711,15 @@ def test_import_durations(self): mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), - ("Object", "o2"), + ["Object", "o1"], + ["Object", "o2"], ], - "parameter_definitions": [("Object", "t")], + "parameter_definitions": [["Object", "t"]], "parameter_values": [ - ["Object", "o1", "t", Duration("23D"), "Base"], - ["Object", "o2", "t", Duration("19D"), "Base"], + ["Object", ("o1",), "t", Duration("23D"), "Base"], + ["Object", ("o2",), "t", Duration("19D"), "Base"], ], }, ) @@ -784,9 +785,9 @@ def test_import_entity_alternatives_with_activity_string(self): mapped_data, { "alternatives": {"Base", "alt1", "alt2"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), + ["Object", "o1"], ], "entity_alternatives": [("Object", ("o1",), "Base", True), ("Object", ("o1",), "alt1", False)], }, @@ -811,9 +812,9 @@ def test_import_entity_alternatives_with_activity_boolean(self): mapped_data, { "alternatives": {"Base", "alt1", "alt2"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), + ["Object", "o1"], ], "entity_alternatives": [("Object", ("o1",), "Base", True), ("Object", ("o1",), "alt1", False)], }, @@ -838,9 +839,9 @@ def test_import_entity_alternatives_with_activity_integer(self): mapped_data, { "alternatives": {"Base", "alt1", "alt2"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), + ["Object", "o1"], ], "entity_alternatives": [("Object", ("o1",), "Base", True), ("Object", ("o1",), "alt1", False)], }, @@ -871,9 +872,9 @@ def test_import_entity_alternatives_errors_gracefully_when_activity_cannot_be_co mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), + ["Object", "o1"], ], "entity_alternatives": [], }, @@ -902,13 +903,13 @@ def test_import_entity_alternatives_with_multidimensional_entities(self): mapped_data, { "alternatives": {"Base", "alt1"}, - "entity_classes": [("Widget",), ("Gadget",), ("Widget__Gadget", ("Widget", "Gadget"))], + "entity_classes": [["Widget__Gadget", ["Widget", "Gadget"]], ["Widget", []], ["Gadget", []]], "entities": [ - ("Widget", "o1"), - ("Gadget", "p1"), - ("Widget__Gadget", ("o1", "p1")), - ("Gadget", "p2"), - ("Widget__Gadget", ("o1", "p2")), + ["Widget__Gadget", ["o1", "p1"]], + ["Widget", "o1"], + ["Gadget", "p1"], + ["Widget__Gadget", ["o1", "p2"]], + ["Gadget", "p2"], ], "entity_alternatives": [ ("Widget__Gadget", ("o1", "p1"), "Base", True), @@ -943,8 +944,8 @@ def test_import_parameter_types(self): self.assertEqual( mapped_data, { - "entity_classes": [("Widget",), ("Gadget",), ("Object",)], - "parameter_definitions": [("Widget", "x"), ("Gadget", "p"), ("Gadget", "q"), ("Object", "w")], + "entity_classes": [["Widget", []], ["Gadget", []], ["Object", []]], + "parameter_definitions": [["Widget", "x"], ["Gadget", "p"], ["Gadget", "q"], ["Object", "w"]], "parameter_types": [ ("Widget", "x", "float"), ("Widget", "x", "bool"), @@ -974,7 +975,10 @@ def test_skip_first_row_when_importing_pivoted_data(self): self.assertEqual(errors, []) self.assertEqual( mapped_data, - {"scenario_alternatives": [["Scenario1", "Base"], ["Scenario1", "fixed_prices"]]}, + { + "scenarios": {("Scenario1",)}, + "scenario_alternatives": [["Scenario1", "Base"], ["Scenario1", "fixed_prices"]], + }, ) def test_leaf_mapping_with_position_on_row_is_still_considered_as_pivoted(self): @@ -999,13 +1003,14 @@ def test_leaf_mapping_with_position_on_row_is_still_considered_as_pivoted(self): self.assertEqual( mapped_data, { + "scenarios": {("Scenario1",), ("Scenario2",)}, "scenario_alternatives": [ ["Scenario1", "Base"], ["Scenario1", "alt1"], ["Scenario2", "Base"], ["Scenario2", "alt1"], ["Scenario2", "alt2"], - ] + ], }, ) @@ -1031,9 +1036,9 @@ def test_column_header_position_while_leaf_is_hidden(self): mapped_data, { "entity_classes": [ - ("Widget",), + ["Widget", []], ], - "entities": [("Widget", "gadget")], + "entities": [["Widget", "gadget"]], }, ) @@ -1067,13 +1072,13 @@ def test_missing_entity_alternative_does_not_prevent_importing_of_values(self): mapped_data, { "alternatives": {"Succeed", "Fail"}, - "entities": [("unit", "Wind_plant")], + "entities": [["unit", "Wind_plant"]], "entity_alternatives": [("unit", ("Wind_plant",), "Succeed", True)], - "entity_classes": [("unit",)], - "parameter_definitions": [("unit", "existing")], + "entity_classes": [["unit", []]], + "parameter_definitions": [["unit", "existing"]], "parameter_values": [ - ["unit", "Wind_plant", "existing", 150.0, "Fail"], - ["unit", "Wind_plant", "existing", 200.0, "Succeed"], + ["unit", ("Wind_plant",), "existing", 150.0, "Fail"], + ["unit", ("Wind_plant",), "existing", 200.0, "Succeed"], ], }, ) @@ -1128,8 +1133,8 @@ def test_import_entity_metadata(self): self.assertEqual( mapped_data, { - "entities": [("cat", "Garfield"), ("cat", "Tom")], - "entity_classes": [("cat",)], + "entities": [["cat", "Garfield"], ["cat", "Tom"]], + "entity_classes": [["cat", []]], "entity_metadata": [ ("cat", ("Garfield",), "Created", "1976"), ("cat", ("Garfield",), "Keywords", "laziness, gluttony"), @@ -1164,9 +1169,9 @@ def test_import_parameter_value_metadata(self): mapped_data, { "alternatives": {"Base"}, - "entities": [("cat", "Garfield"), ("cat", "Tom")], - "entity_classes": [("cat",)], - "parameter_definitions": [("cat", "weight")], + "entities": [["cat", "Garfield"], ["cat", "Tom"]], + "entity_classes": [["cat", []]], + "parameter_definitions": [["cat", "weight"]], "parameter_value_metadata": [ ("cat", ("Garfield",), "weight", "Tools", "Harrison-Stetson 1.0", "Base"), ("cat", ("Garfield",), "weight", "Licences", "Public domain", "Base"), @@ -1175,3 +1180,204 @@ def test_import_parameter_value_metadata(self): ], }, ) + + def test_import_alternatives_with_descriptions(self): + header = ["Alternative", "Description"] + data_source = iter( + [ + ["alt1", "First alternative."], + ["alt2", ""], + ["alt3", None], + ["duplicate", "Overridden description."], + ["duplicate", "Overriding description."], + ] + ) + flattened = default_import_mapping("Alternative").flatten() + flattened[0].position = 0 + flattened[1].position = 1 + mapped_data, errors = get_mapped_data(data_source, [to_dict(unflatten(flattened))], header) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "alternatives": { + ("alt1", "First alternative."), + "alt2", + "alt3", + ("duplicate", "Overridden description."), + ("duplicate", "Overriding description."), + }, + }, + ) + + def test_import_scenarios_with_descriptions(self): + header = ["Scenario", "Description"] + data_source = iter( + [ + ["scen1", "First scenario."], + ["scen2", None], + ["scen3", ""], + ["duplicate", "Possible description no. 1."], + ["duplicate", "Possible description no. 2."], + ] + ) + flattened = default_import_mapping("Scenario").flatten() + flattened[0].position = 0 + flattened[1].position = 1 + mapped_data, errors = get_mapped_data(data_source, [to_dict(unflatten(flattened))], header) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "scenarios": { + ("scen1", "First scenario."), + ("scen2",), + ("scen3",), + ("duplicate", "Possible description no. 1."), + ("duplicate", "Possible description no. 2."), + }, + }, + ) + + def test_import_entity_classes_with_description(self): + header = ["Class", "Description", "Entity"] + data_source = iter( + [ + ["unit", "Unit of production.", "coal_plant"], + ["node", "Nodes of processing.", "southern_hemisphere"], + ["node", "Nodes of processing.", "northern_hemisphere"], + ["model", None, "all_year_round"], + ["direction", "", "up"], + ["direction", "", "down"], + ] + ) + flattened = default_import_mapping("EntityClass").flatten() + flattened[0].position = 0 + flattened[1].position = 1 + flattened[2].position = 2 + mapped_data, errors = get_mapped_data(data_source, [to_dict(unflatten(flattened))], header) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "entity_classes": [ + ["unit", [], "Unit of production."], + ["node", [], "Nodes of processing."], + ["model", []], + ["direction", []], + ], + "entities": [ + ["unit", "coal_plant"], + ["node", "southern_hemisphere"], + ["node", "northern_hemisphere"], + ["model", "all_year_round"], + ["direction", "up"], + ["direction", "down"], + ], + }, + ) + + def test_import_entities_with_description(self): + header = ["Class", "Entity", "Description"] + data_source = iter( + [ + ["unit", "coal_plant", "Where coal is sacrificed to please the gods of Power."], + ["direction", "up", None], + ["direction", "down", ""], + ] + ) + flattened = default_import_mapping("EntityClass").flatten() + flattened[0].position = 0 + flattened[2].position = 1 + flattened[3].position = 2 + mapped_data, errors = get_mapped_data(data_source, [to_dict(unflatten(flattened))], header) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "entity_classes": [ + ["unit", []], + ["direction", []], + ], + "entities": [ + ["unit", "coal_plant", "Where coal is sacrificed to please the gods of Power."], + ["direction", "up"], + ["direction", "down"], + ], + }, + ) + + def test_import_multidimensional_entities_with_descriptions(self): + header = ["Widget", "Gadget", "Description"] + data_source = iter( + [ + ["check_box", "mobile_phone", "A cozy relationship."], + ] + ) + mappings = [ + [ + {"map_type": "EntityClass", "position": "hidden", "value": "Widget__Gadget"}, + {"map_type": "Dimension", "position": "hidden", "value": "Widget"}, + {"map_type": "Dimension", "position": "hidden", "value": "Gadget"}, + {"map_type": "EntityClassDescription", "position": "hidden"}, + {"map_type": "Entity", "position": "hidden", "value": "relationship"}, + {"map_type": "Element", "position": 0, "import_objects": True}, + {"map_type": "Element", "position": 1, "import_objects": True}, + {"map_type": "EntityDescription", "position": 2}, + ] + ] + mapped_data, errors = get_mapped_data(data_source, mappings, header) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "entity_classes": [ + ["Widget__Gadget", ["Widget", "Gadget"]], + ["Widget", []], + ["Gadget", []], + ], + "entities": [ + ["Widget__Gadget", ["check_box", "mobile_phone"], "A cozy relationship."], + ["Widget", "check_box"], + ["Gadget", "mobile_phone"], + ], + }, + ) + + def test_import_different_relationships_in_same_table(self): + data_source = iter( + [ + ["Widget__Gadget", "Widget", "Gadget", "tableview", "watch", "A cozy relationship 1."], + ["Widget__Gadget", "Widget", "Gadget", "checkbox", "watch", "A cozy relationship 2."], + ["Widget__Gadget", "Widget", "Gadget", "checkbox", "clock", "A cozy relationship 3."], + ["Gadget__Widget", "Gadget", "Widget", "watch", "checkbox", "A cozy relationship 4."], + ["Gadget__Widget", "Gadget", "Widget", "clock", "tableview", "A cozy relationship 5."], + ] + ) + mappings = [ + [ + {"map_type": "EntityClass", "position": 0}, + {"map_type": "Dimension", "position": 1}, + {"map_type": "Dimension", "position": 2}, + {"map_type": "EntityClassDescription", "position": "hidden"}, + {"map_type": "Entity", "position": "hidden"}, + {"map_type": "Element", "position": 3}, + {"map_type": "Element", "position": 4}, + {"map_type": "EntityDescription", "position": 5}, + ] + ] + mapped_data, errors = get_mapped_data(data_source, mappings) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "entity_classes": [["Widget__Gadget", ["Widget", "Gadget"]], ["Gadget__Widget", ["Gadget", "Widget"]]], + "entities": [ + ["Widget__Gadget", ["tableview", "watch"], "A cozy relationship 1."], + ["Widget__Gadget", ["checkbox", "watch"], "A cozy relationship 2."], + ["Widget__Gadget", ["checkbox", "clock"], "A cozy relationship 3."], + ["Gadget__Widget", ["watch", "checkbox"], "A cozy relationship 4."], + ["Gadget__Widget", ["clock", "tableview"], "A cozy relationship 5."], + ], + }, + ) diff --git a/tests/import_mapping/test_import_mapping.py b/tests/import_mapping/test_import_mapping.py index 0bea2a4c..26c61675 100644 --- a/tests/import_mapping/test_import_mapping.py +++ b/tests/import_mapping/test_import_mapping.py @@ -13,7 +13,7 @@ """Unit tests for import Mappings.""" import unittest from unittest.mock import Mock -from spinedb_api.exception import InvalidMapping +from spinedb_api.exception import InvalidMapping, InvalidMappingComponent from spinedb_api.import_mapping.generator import get_mapped_data from spinedb_api.import_mapping.import_mapping import ( AlternativeMapping, @@ -26,6 +26,7 @@ IndexNameMapping, ParameterDefaultValueIndexMapping, ParameterDefaultValueTypeMapping, + ParameterDefinitionDescriptionMapping, ParameterDefinitionMapping, ParameterValueIndexMapping, ParameterValueMapping, @@ -60,9 +61,9 @@ def test_convert_functions_float(self): param_def_mapping.flatten()[-1].position = 1 mapped_data, _ = get_mapped_data(data, [mapping], column_convert_fns=column_convert_fns) expected = { - "entity_classes": [("a",)], - "entities": [("a", "obj")], - "parameter_definitions": [("a", "param", 1.2)], + "entity_classes": [["a", []]], + "entities": [["a", "obj"]], + "parameter_definitions": [["a", "param", 1.2]], } self.assertEqual(mapped_data, expected) @@ -79,9 +80,9 @@ def test_convert_functions_str(self): param_def_mapping.flatten()[-1].position = 1 mapped_data, _ = get_mapped_data(data, [mapping], column_convert_fns=column_convert_fns) expected = { - "entity_classes": [("a",)], - "entities": [("a", "obj")], - "parameter_definitions": [("a", "param", "1111.2222")], + "entity_classes": [["a", []]], + "entities": [["a", "obj"]], + "parameter_definitions": [["a", "param", "1111.2222"]], } self.assertEqual(mapped_data, expected) @@ -98,9 +99,9 @@ def test_convert_functions_bool(self): param_def_mapping.flatten()[-1].position = 1 mapped_data, _ = get_mapped_data(data, [mapping], column_convert_fns=column_convert_fns) expected = { - "entity_classes": [("a",)], - "entities": [("a", "obj")], - "parameter_definitions": [("a", "param", False)], + "entity_classes": [["a", []]], + "entities": [["a", "obj"]], + "parameter_definitions": [["a", "param", False]], } self.assertEqual(mapped_data, expected) @@ -549,14 +550,16 @@ def test_valid_object_default_value_mapping_not_missing_parameter_definition(sel issues = check_validity(cls_mapping) self.assertFalse(issues) - def test_invalid_object_value_list_mapping_missing_parameter_definition(self): + def test_value_list_mapping_missing_parameter_definition_is_ok(self): cls_mapping = import_mapping_from_dict({"map_type": "ObjectClass"}) cls_mapping.flatten()[-1].child = parameter_mapping_from_dict({"map_type": "ParameterDefinition"}) value_list_mapping = cls_mapping.flatten()[-2] cls_mapping.position = 0 value_list_mapping.position = 1 issues = check_validity(cls_mapping) - self.assertTrue(issues) + self.assertEqual( + issues, [InvalidMappingComponent("value list requires a parameter name", value_list_mapping.rank)] + ) def test_valid_object_value_list_mapping_not_missing_parameter_definition(self): cls_mapping = import_mapping_from_dict({"map_type": "ObjectClass"}) @@ -656,7 +659,9 @@ def test_invalid_relationship_value_list_mapping_missing_parameter_definition(se cls_mapping.position = 0 value_list_mapping.position = 1 issues = check_validity(cls_mapping) - self.assertTrue(issues) + self.assertEqual( + issues, [InvalidMappingComponent("value list requires a parameter name", value_list_mapping.rank)] + ) def test_valid_relationship_value_list_mapping_not_missing_parameter_definition(self): cls_mapping = import_mapping_from_dict({"map_type": "RelationshipClass"}) @@ -737,8 +742,11 @@ def test_invalid_single_value_mapping_missing_parameter_definition(self): self.assertTrue(issues) def test_valid_array_mapping(self): - value_mapping = parameter_value_mapping_from_dict({"value_type": "array"}) - issues = check_validity(value_mapping) + root_mapping = default_import_mapping("EntityClass") + root_mapping.position = 0 + definition_mapping = root_mapping.tail_mapping().child = parameter_mapping_from_dict({"value_type": "array"}) + definition_mapping.position = 3 + issues = check_validity(root_mapping) self.assertFalse(issues) def test_invalid_array_mapping_missing_parameter_definition(self): @@ -748,8 +756,13 @@ def test_invalid_array_mapping_missing_parameter_definition(self): self.assertTrue(issues) def test_valid_time_series_mapping(self): - value_mapping = parameter_value_mapping_from_dict({"value_type": "time_series"}) - issues = check_validity(value_mapping) + root_mapping = default_import_mapping("EntityClass") + root_mapping.position = 0 + definition_mapping = root_mapping.tail_mapping().child = parameter_mapping_from_dict( + {"value_type": "time_series"} + ) + definition_mapping.position = 3 + issues = check_validity(root_mapping) self.assertFalse(issues) def test_invalid_time_series_mapping_missing_parameter_definition(self): @@ -782,7 +795,7 @@ def test_read_iterator_with_row_with_all_Nones(self): [None, None, None, None], ["oc2", "obj2", "parameter_name2", 2], ] - expected = {"entity_classes": [("oc2",)]} + expected = {"entity_classes": [["oc2", []]]} data = iter(input_data) data_header = next(data) @@ -795,7 +808,7 @@ def test_read_iterator_with_row_with_all_Nones(self): def test_read_iterator_with_None(self): input_data = [["object_class", "object", "parameter", "value"], None, ["oc2", "obj2", "parameter_name2", 2]] - expected = {"entity_classes": [("oc2",)]} + expected = {"entity_classes": [["oc2", []]]} data = iter(input_data) data_header = next(data) @@ -813,10 +826,10 @@ def test_read_flat_file(self): ["oc2", "obj2", "parameter_name2", 2], ] expected = { - "entity_classes": [("oc1",), ("oc2",)], - "entities": [("oc1", "obj1"), ("oc2", "obj2")], - "parameter_definitions": [("oc1", "parameter_name1"), ("oc2", "parameter_name2")], - "parameter_values": [["oc1", "obj1", "parameter_name1", 1], ["oc2", "obj2", "parameter_name2", 2]], + "entity_classes": [["oc1", []], ["oc2", []]], + "entities": [["oc1", "obj1"], ["oc2", "obj2"]], + "parameter_definitions": [["oc1", "parameter_name1"], ["oc2", "parameter_name2"]], + "parameter_values": [["oc1", ("obj1",), "parameter_name1", 1], ["oc2", ("obj2",), "parameter_name2", 2]], } data = iter(input_data) @@ -840,10 +853,10 @@ def test_read_flat_file_array(self): ["oc1", "obj1", "parameter_name1", 2], ] expected = { - "entity_classes": [("oc1",)], - "entities": [("oc1", "obj1")], - "parameter_definitions": [("oc1", "parameter_name1")], - "parameter_values": [["oc1", "obj1", "parameter_name1", Array([1, 2])]], + "entity_classes": [["oc1", []]], + "entities": [["oc1", "obj1"]], + "parameter_definitions": [["oc1", "parameter_name1"]], + "parameter_values": [["oc1", ("obj1",), "parameter_name1", Array([1, 2])]], } data = iter(input_data) @@ -867,10 +880,10 @@ def test_read_flat_file_array_with_ed(self): ["oc1", "obj1", "parameter_name1", 2, 1], ] expected = { - "entity_classes": [("oc1",)], - "entities": [("oc1", "obj1")], - "parameter_definitions": [("oc1", "parameter_name1")], - "parameter_values": [["oc1", "obj1", "parameter_name1", Array([1, 2])]], + "entity_classes": [["oc1", []]], + "entities": [["oc1", "obj1"]], + "parameter_definitions": [["oc1", "parameter_name1"]], + "parameter_values": [["oc1", ("obj1",), "parameter_name1", Array([1, 2])]], } data = iter(input_data) @@ -895,7 +908,7 @@ def test_read_flat_file_array_with_ed(self): def test_read_flat_file_with_column_name_reference(self): input_data = [["object", "parameter", "value"], ["obj1", "parameter_name1", 1], ["obj2", "parameter_name2", 2]] - expected = {"entity_classes": [("object",)], "entities": [("object", "obj1"), ("object", "obj2")]} + expected = {"entity_classes": [["object", []]], "entities": [["object", "obj1"], ["object", "obj2"]]} data = iter(input_data) data_header = next(data) @@ -909,8 +922,8 @@ def test_read_flat_file_with_column_name_reference(self): def test_read_object_class_from_header_using_string_as_integral_index(self): input_data = [["object_class"], ["obj1"], ["obj2"]] expected = { - "entity_classes": [("object_class",)], - "entities": [("object_class", "obj1"), ("object_class", "obj2")], + "entity_classes": [["object_class", []]], + "entities": [["object_class", "obj1"], ["object_class", "obj2"]], } data = iter(input_data) @@ -925,8 +938,8 @@ def test_read_object_class_from_header_using_string_as_integral_index(self): def test_read_object_class_from_header_using_string_as_column_header_name(self): input_data = [["object_class"], ["obj1"], ["obj2"]] expected = { - "entity_classes": [("object_class",)], - "entities": [("object_class", "obj1"), ("object_class", "obj2")], + "entity_classes": [["object_class", []]], + "entities": [["object_class", "obj1"], ["object_class", "obj2"]], } data = iter(input_data) @@ -944,7 +957,7 @@ def test_read_object_class_from_header_using_string_as_column_header_name(self): def test_read_with_list_of_mappings(self): input_data = [["object", "parameter", "value"], ["obj1", "parameter_name1", 1], ["obj2", "parameter_name2", 2]] - expected = {"entity_classes": [("object",)], "entities": [("object", "obj1"), ("object", "obj2")]} + expected = {"entity_classes": [["object", []]], "entities": [["object", "obj1"], ["object", "obj2"]]} data = iter(input_data) data_header = next(data) @@ -958,14 +971,14 @@ def test_read_with_list_of_mappings(self): def test_read_pivoted_parameters_from_header(self): input_data = [["object", "parameter_name1", "parameter_name2"], ["obj1", 0, 1], ["obj2", 2, 3]] expected = { - "entity_classes": [("object",)], - "entities": [("object", "obj1"), ("object", "obj2")], - "parameter_definitions": [("object", "parameter_name1"), ("object", "parameter_name2")], + "entity_classes": [["object", []]], + "entities": [["object", "obj1"], ["object", "obj2"]], + "parameter_definitions": [["object", "parameter_name1"], ["object", "parameter_name2"]], "parameter_values": [ - ["object", "obj1", "parameter_name1", 0], - ["object", "obj1", "parameter_name2", 1], - ["object", "obj2", "parameter_name1", 2], - ["object", "obj2", "parameter_name2", 3], + ["object", ("obj1",), "parameter_name1", 0], + ["object", ("obj1",), "parameter_name2", 1], + ["object", ("obj2",), "parameter_name1", 2], + ["object", ("obj2",), "parameter_name2", 3], ], } @@ -1004,14 +1017,14 @@ def test_read_empty_pivot(self): def test_read_pivoted_parameters_from_data(self): input_data = [["object", "parameter_name1", "parameter_name2"], ["obj1", 0, 1], ["obj2", 2, 3]] expected = { - "entity_classes": [("object",)], - "entities": [("object", "obj1"), ("object", "obj2")], - "parameter_definitions": [("object", "parameter_name1"), ("object", "parameter_name2")], + "entity_classes": [["object", []]], + "entities": [["object", "obj1"], ["object", "obj2"]], + "parameter_definitions": [["object", "parameter_name1"], ["object", "parameter_name2"]], "parameter_values": [ - ["object", "obj1", "parameter_name1", 0], - ["object", "obj1", "parameter_name2", 1], - ["object", "obj2", "parameter_name1", 2], - ["object", "obj2", "parameter_name2", 3], + ["object", ("obj1",), "parameter_name1", 0], + ["object", ("obj1",), "parameter_name2", 1], + ["object", ("obj2",), "parameter_name1", 2], + ["object", ("obj2",), "parameter_name2", 3], ], } @@ -1038,13 +1051,13 @@ def test_pivoted_value_has_actual_position(self): ["obj2", "T2", 22.0], ] expected = { - "entity_classes": [("timeline",)], - "entities": [("timeline", "obj1"), ("timeline", "obj2")], - "parameter_definitions": [("timeline", "value")], + "entity_classes": [["timeline", []]], + "entities": [["timeline", "obj1"], ["timeline", "obj2"]], + "parameter_definitions": [["timeline", "value"]], "alternatives": {"Base"}, "parameter_values": [ - ["timeline", "obj1", "value", Map(["T1", "T2"], [11.0, 12.0], index_name="timestep"), "Base"], - ["timeline", "obj2", "value", Map(["T1", "T2"], [21.0, 22.0], index_name="timestep"), "Base"], + ["timeline", ("obj1",), "value", Map(["T1", "T2"], [11.0, 12.0], index_name="timestep"), "Base"], + ["timeline", ("obj2",), "value", Map(["T1", "T2"], [21.0, 22.0], index_name="timestep"), "Base"], ], } data = iter(input_data) @@ -1068,13 +1081,13 @@ def test_import_objects_from_pivoted_data_when_they_lack_parameter_values(self): """Pivoted mapping works even when last mapping has valid position in columns.""" input_data = [["object", "is_skilled", "has_powers"], ["obj1", "yes", "no"], ["obj2", None, None]] expected = { - "entity_classes": [("node",)], - "entities": [("node", "obj1"), ("node", "obj2")], - "parameter_definitions": [("node", "is_skilled"), ("node", "has_powers")], + "entity_classes": [["node", []]], + "entities": [["node", "obj1"], ["node", "obj2"]], + "parameter_definitions": [["node", "is_skilled"], ["node", "has_powers"]], "alternatives": {"Base"}, "parameter_values": [ - ["node", "obj1", "is_skilled", "yes", "Base"], - ["node", "obj1", "has_powers", "no", "Base"], + ["node", ("obj1",), "is_skilled", "yes", "Base"], + ["node", ("obj1",), "has_powers", "no", "Base"], ], } data = iter(input_data) @@ -1099,12 +1112,18 @@ def test_import_objects_from_pivoted_data_when_they_lack_map_type_parameter_valu ["obj1", "today", None, "yes"], ] expected = { - "entity_classes": [("node",)], - "entities": [("node", "obj1")], - "parameter_definitions": [("node", "is_skilled"), ("node", "has_powers")], + "entity_classes": [["node", []]], + "entities": [["node", "obj1"]], + "parameter_definitions": [["node", "is_skilled"], ["node", "has_powers"]], "alternatives": {"Base"}, "parameter_values": [ - ["node", "obj1", "has_powers", Map(["yesterday", "today"], ["no", "yes"], index_name="period"), "Base"] + [ + "node", + ("obj1",), + "has_powers", + Map(["yesterday", "today"], ["no", "yes"], index_name="period"), + "Base", + ] ], } data = iter(input_data) @@ -1128,13 +1147,13 @@ def test_read_flat_file_with_extra_value_dimensions(self): input_data = [["object", "time", "parameter_name1"], ["obj1", "2018-01-01", 1], ["obj1", "2018-01-02", 2]] expected = { - "entity_classes": [("object",)], - "entities": [("object", "obj1")], - "parameter_definitions": [("object", "parameter_name1")], + "entity_classes": [["object", []]], + "entities": [["object", "obj1"]], + "parameter_definitions": [["object", "parameter_name1"]], "parameter_values": [ [ "object", - "obj1", + ("obj1",), "parameter_name1", TimeSeriesVariableResolution(["2018-01-01", "2018-01-02"], [1, 2], False, False), ] @@ -1165,9 +1184,9 @@ def test_read_flat_file_with_parameter_definition(self): input_data = [["object", "time", "parameter_name1"], ["obj1", "2018-01-01", 1], ["obj1", "2018-01-02", 2]] expected = { - "entity_classes": [("object",)], - "entities": [("object", "obj1")], - "parameter_definitions": [("object", "parameter_name1")], + "entity_classes": [["object", []]], + "entities": [["object", "obj1"]], + "parameter_definitions": [["object", "parameter_name1"]], } data = iter(input_data) @@ -1192,8 +1211,8 @@ def test_read_flat_file_with_parameter_definition(self): def test_read_1dim_relationships(self): input_data = [["unit", "node"], ["u1", "n1"], ["u1", "n2"]] expected = { - "entity_classes": [("node_group", ("node",))], - "entities": [("node_group", ("n1",)), ("node_group", ("n2",))], + "entity_classes": [["node_group", ["node"]]], + "entities": [["node_group", ["n1"]], ["node_group", ["n2"]]], } data = iter(input_data) @@ -1213,8 +1232,8 @@ def test_read_1dim_relationships(self): def test_read_relationships(self): input_data = [["unit", "node"], ["u1", "n1"], ["u1", "n2"]] expected = { - "entity_classes": [("unit__node", ("unit", "node"))], - "entities": [("unit__node", ("u1", "n1")), ("unit__node", ("u1", "n2"))], + "entity_classes": [["unit__node", ["unit", "node"]]], + "entities": [["unit__node", ["u1", "n1"]], ["unit__node", ["u1", "n2"]]], } data = iter(input_data) @@ -1237,9 +1256,9 @@ def test_read_relationships(self): def test_read_relationships_with_parameters(self): input_data = [["unit", "node", "rel_parameter"], ["u1", "n1", 0], ["u1", "n2", 1]] expected = { - "entity_classes": [("unit__node", ("unit", "node"))], - "entities": [("unit__node", ("u1", "n1")), ("unit__node", ("u1", "n2"))], - "parameter_definitions": [("unit__node", "rel_parameter")], + "entity_classes": [["unit__node", ["unit", "node"]]], + "entities": [["unit__node", ["u1", "n1"]], ["unit__node", ["u1", "n2"]]], + "parameter_definitions": [["unit__node", "rel_parameter"]], "parameter_values": [ ["unit__node", ("u1", "n1"), "rel_parameter", 0], ["unit__node", ("u1", "n2"), "rel_parameter", 1], @@ -1267,15 +1286,15 @@ def test_read_relationships_with_parameters(self): def test_read_relationships_with_parameters2(self): input_data = [["nuts2", "Capacity", "Fueltype"], ["BE23", 268.0, "Bioenergy"], ["DE11", 14.0, "Bioenergy"]] expected = { - "entity_classes": [("nuts2",), ("fueltype",), ("nuts2__fueltype", ("nuts2", "fueltype"))], + "entity_classes": [["nuts2__fueltype", ["nuts2", "fueltype"]], ["nuts2", []], ["fueltype", []]], "entities": [ - ("nuts2", "BE23"), - ("fueltype", "Bioenergy"), - ("nuts2__fueltype", ("BE23", "Bioenergy")), - ("nuts2", "DE11"), - ("nuts2__fueltype", ("DE11", "Bioenergy")), + ["nuts2__fueltype", ["BE23", "Bioenergy"]], + ["nuts2", "BE23"], + ["fueltype", "Bioenergy"], + ["nuts2__fueltype", ["DE11", "Bioenergy"]], + ["nuts2", "DE11"], ], - "parameter_definitions": [("nuts2__fueltype", "capacity")], + "parameter_definitions": [["nuts2__fueltype", "capacity"]], "parameter_values": [ ["nuts2__fueltype", ("BE23", "Bioenergy"), "capacity", 268.0], ["nuts2__fueltype", ("DE11", "Bioenergy"), "capacity", 14.0], @@ -1311,12 +1330,12 @@ def test_read_relationships_with_parameters2(self): def test_read_parameter_header_with_only_one_parameter(self): input_data = [["object", "parameter_name1"], ["obj1", 0], ["obj2", 2]] expected = { - "entity_classes": [("object",)], - "entities": [("object", "obj1"), ("object", "obj2")], - "parameter_definitions": [("object", "parameter_name1")], + "entity_classes": [["object", []]], + "entities": [["object", "obj1"], ["object", "obj2"]], + "parameter_definitions": [["object", "parameter_name1"]], "parameter_values": [ - ["object", "obj1", "parameter_name1", 0], - ["object", "obj2", "parameter_name1", 2], + ["object", ("obj1",), "parameter_name1", 0], + ["object", ("obj2",), "parameter_name1", 2], ], } @@ -1337,12 +1356,12 @@ def test_read_parameter_header_with_only_one_parameter(self): def test_read_pivoted_parameters_from_data_with_skipped_column(self): input_data = [["object", "parameter_name1", "parameter_name2"], ["obj1", 0, 1], ["obj2", 2, 3]] expected = { - "entity_classes": [("object",)], - "entities": [("object", "obj1"), ("object", "obj2")], - "parameter_definitions": [("object", "parameter_name1")], + "entity_classes": [["object", []]], + "entities": [["object", "obj1"], ["object", "obj2"]], + "parameter_definitions": [["object", "parameter_name1"]], "parameter_values": [ - ["object", "obj1", "parameter_name1", 0], - ["object", "obj2", "parameter_name1", 2], + ["object", ("obj1",), "parameter_name1", 0], + ["object", ("obj2",), "parameter_name1", 2], ], } @@ -1363,14 +1382,18 @@ def test_read_pivoted_parameters_from_data_with_skipped_column(self): def test_read_relationships_and_import_objects(self): input_data = [["unit", "node"], ["u1", "n1"], ["u2", "n2"]] expected = { - "entity_classes": [("unit",), ("node",), ("unit__node", ("unit", "node"))], + "entity_classes": [ + ["unit__node", ["unit", "node"]], + ["unit", []], + ["node", []], + ], "entities": [ - ("unit", "u1"), - ("node", "n1"), - ("unit__node", ("u1", "n1")), - ("unit", "u2"), - ("node", "n2"), - ("unit__node", ("u2", "n2")), + ["unit__node", ["u1", "n1"]], + ["unit", "u1"], + ["node", "n1"], + ["unit__node", ["u2", "n2"]], + ["unit", "u2"], + ["node", "n2"], ], } @@ -1394,11 +1417,10 @@ def test_read_relationships_and_import_objects(self): def test_read_relationships_parameter_values_with_extra_dimensions(self): input_data = [["", "a", "b"], ["", "c", "d"], ["", "e", "f"], ["a", 2, 3], ["b", 4, 5]] - expected = { - "entity_classes": [("unit__node", ("unit", "node"))], - "parameter_definitions": [("unit__node", "e"), ("unit__node", "f")], - "entities": [("unit__node", ("a", "c")), ("unit__node", ("b", "d"))], + "entity_classes": [["unit__node", ["unit", "node"]]], + "parameter_definitions": [["unit__node", "e"], ["unit__node", "f"]], + "entities": [["unit__node", ["a", "c"]], ["unit__node", ["b", "d"]]], "parameter_values": [ ["unit__node", ("a", "c"), "e", Map(["a", "b"], [2, 4])], ["unit__node", ("b", "d"), "f", Map(["a", "b"], [3, 5])], @@ -1433,10 +1455,10 @@ def test_read_data_with_read_start_row(self): ["oc2", "obj2", "parameter_name2", 2], ] expected = { - "entity_classes": [("oc1",), ("oc2",)], - "entities": [("oc1", "obj1"), ("oc2", "obj2")], - "parameter_definitions": [("oc1", "parameter_name1"), ("oc2", "parameter_name2")], - "parameter_values": [["oc1", "obj1", "parameter_name1", 1], ["oc2", "obj2", "parameter_name2", 2]], + "entity_classes": [["oc1", []], ["oc2", []]], + "entities": [["oc1", "obj1"], ["oc2", "obj2"]], + "parameter_definitions": [["oc1", "parameter_name1"], ["oc2", "parameter_name2"]], + "parameter_values": [["oc1", ("obj1",), "parameter_name1", 1], ["oc2", ("obj2",), "parameter_name2", 2]], } data = iter(input_data) @@ -1462,13 +1484,13 @@ def test_read_data_with_two_mappings_with_different_read_start_row(self): ["oc1_obj2", "oc2_obj2", 2, 4], ] expected = { - "entity_classes": [("oc1",), ("oc2",)], - "entities": [("oc1", "oc1_obj1"), ("oc1", "oc1_obj2"), ("oc2", "oc2_obj2")], - "parameter_definitions": [("oc1", "parameter_class1"), ("oc2", "parameter_class2")], + "entity_classes": [["oc1", []], ["oc2", []]], + "entities": [["oc1", "oc1_obj1"], ["oc1", "oc1_obj2"], ["oc2", "oc2_obj2"]], + "parameter_definitions": [["oc1", "parameter_class1"], ["oc2", "parameter_class2"]], "parameter_values": [ - ["oc1", "oc1_obj1", "parameter_class1", 1], - ["oc1", "oc1_obj2", "parameter_class1", 2], - ["oc2", "oc2_obj2", "parameter_class2", 4], + ["oc1", ("oc1_obj1",), "parameter_class1", 1], + ["oc1", ("oc1_obj2",), "parameter_class1", 2], + ["oc2", ("oc2_obj2",), "parameter_class2", 4], ], } @@ -1505,8 +1527,8 @@ def test_read_object_class_with_table_name_as_class_name(self): } out, errors = get_mapped_data(data, [mapping], data_header, "class name") expected = { - "entity_classes": [("class name",)], - "entities": [("class name", "object 1"), ("class name", "object 2")], + "entity_classes": [["class name", []]], + "entities": [["class name", "object 1"], ["class name", "object 2"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1530,10 +1552,10 @@ def test_read_flat_map_from_columns(self): out, errors = get_mapped_data(data, [mapping], data_header) expected_map = Map(["key1", "key2"], [-2, -1]) expected = { - "entity_classes": [("object_class",)], - "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "entity_classes": [["object_class", []]], + "entities": [["object_class", "object"]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1557,10 +1579,10 @@ def test_read_nested_map_from_columns(self): out, errors = get_mapped_data(data, [mapping], data_header) expected_map = Map(["key11", "key21"], [Map(["key12"], [-2]), Map(["key22"], [-1])]) expected = { - "entity_classes": [("object_class",)], - "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "entity_classes": [["object_class", []]], + "entities": [["object_class", "object"]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1601,10 +1623,10 @@ def test_read_uneven_nested_map_from_columns(self): ], ) expected = { - "entity_classes": [("object_class",)], - "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "entity_classes": [["object_class", []]], + "entities": [["object_class", "object"]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1643,10 +1665,10 @@ def test_read_nested_map_with_compression(self): ], ) expected = { - "entity_classes": [("object_class",)], - "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "entity_classes": [["object_class", []]], + "entities": [["object_class", "object"]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1697,12 +1719,14 @@ def test_read_scenario_alternative(self): "before_alternative_name": 2, } out, errors = get_mapped_data(data, [mapping], data_header) - expected = {} - expected["scenario_alternatives"] = [ - ["scenario_A", "alternative1", "second_alternative"], - ["scenario_A", "second_alternative", "last_one"], - ["scenario_B", "last_one", ""], - ] + expected = { + "scenarios": {("scenario_A",), ("scenario_B",)}, + "scenario_alternatives": [ + ["scenario_A", "alternative1", "second_alternative"], + ["scenario_A", "second_alternative", "last_one"], + ["scenario_B", "last_one", ""], + ], + } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1711,12 +1735,14 @@ def test_pivoted_scenario_alternative(self): data = iter(input_data) mappings = [{"map_type": "Scenario", "position": -1}, {"map_type": "ScenarioAlternative", "position": "hidden"}] out, errors = get_mapped_data(data, [mappings]) - expected = {} - expected["scenario_alternatives"] = [ - ["scenario_A", "first_alternative"], - ["scenario_A", "second_alternative"], - ["scenario_B", "Base"], - ] + expected = { + "scenarios": {("scenario_A",), ("scenario_B",)}, + "scenario_alternatives": [ + ["scenario_A", "first_alternative"], + ["scenario_A", "second_alternative"], + ["scenario_B", "Base"], + ], + } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1791,12 +1817,13 @@ def test_read_object_group_without_parameters(self): data_header = next(data) mapping = {"map_type": "ObjectGroup", "name": 0, "groups": 1, "members": 2} out, errors = get_mapped_data(data, [mapping], data_header) - expected = {} - expected["entity_classes"] = [("class_A",)] - expected["entity_groups"] = { - ("class_A", "group1", "object1"), - ("class_A", "group1", "object2"), - ("class_A", "group2", "object3"), + expected = { + "entity_classes": [["class_A", []]], + "entity_groups": { + ("class_A", "group1", "object1"), + ("class_A", "group1", "object2"), + ("class_A", "group2", "object3"), + }, } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1812,20 +1839,21 @@ def test_read_object_group_and_import_objects(self): data_header = next(data) mapping = {"map_type": "ObjectGroup", "name": 0, "groups": 1, "members": 2, "import_objects": True} out, errors = get_mapped_data(data, [mapping], data_header) - expected = {} - expected["entity_groups"] = { - ("class_A", "group1", "object1"), - ("class_A", "group1", "object2"), - ("class_A", "group2", "object3"), - } - expected["entity_classes"] = [("class_A",)] - expected["entities"] = [ - ("class_A", "group1"), - ("class_A", "object1"), - ("class_A", "object2"), - ("class_A", "group2"), - ("class_A", "object3"), - ] + expected = { + "entity_groups": { + ("class_A", "group1", "object1"), + ("class_A", "group1", "object2"), + ("class_A", "group2", "object3"), + }, + "entity_classes": [["class_A", []]], + "entities": [ + ["class_A", "group1"], + ["class_A", "object1"], + ["class_A", "object2"], + ["class_A", "group2"], + ["class_A", "object3"], + ], + } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1849,13 +1877,14 @@ def test_read_parameter_definition_with_default_values_and_value_lists(self): }, } out, errors = get_mapped_data(data, [mapping], data_header) - expected = {} - expected["entity_classes"] = [("class_A",), ("class_B",)] - expected["parameter_definitions"] = [ - ("class_A", "param1", 23.0, "listA"), - ("class_A", "param2", 42.0, "listB"), - ("class_B", "param3", 5.0, "listA"), - ] + expected = { + "entity_classes": [["class_A", []], ["class_B", []]], + "parameter_definitions": [ + ["class_A", "param1", 23.0, "listA"], + ["class_A", "param2", 42.0, "listB"], + ["class_B", "param3", 5.0, "listA"], + ], + } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1874,8 +1903,8 @@ def test_map_as_default_parameter_value(self): out, errors = get_mapped_data(data, [mapping]) expected_map = Map(["key1", "key2", "key3"], [-2.3, 5.5, 3.2]) expected = { - "entity_classes": [("object_class",)], - "parameter_definitions": [("object_class", "parameter", expected_map)], + "entity_classes": [["object_class", []]], + "parameter_definitions": [["object_class", "parameter", expected_map]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1896,8 +1925,8 @@ def test_read_parameter_definition_with_nested_map_as_default_value(self): out, errors = get_mapped_data(data, [mapping], data_header) expected_map = Map(["key11", "key21"], [Map(["key12"], [-2]), Map(["key22"], [-1])]) expected = { - "entity_classes": [("object_class",)], - "parameter_definitions": [("object_class", "parameter", expected_map)], + "entity_classes": [["object_class", []]], + "parameter_definitions": [["object_class", "parameter", expected_map]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1926,10 +1955,10 @@ def test_read_map_index_names_from_columns(self): index_name="Index 1", ) expected = { - "entity_classes": [("object_class",)], - "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "entity_classes": [["object_class", []]], + "entities": [["object_class", "object"]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1958,10 +1987,10 @@ def test_missing_map_index_name(self): index_name="", ) expected = { - "entity_classes": [("object_class",)], - "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "entity_classes": [["object_class", []]], + "entities": [["object_class", "object"]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1989,8 +2018,8 @@ def test_read_default_value_index_names_from_columns(self): index_name="Index 1", ) expected = { - "entity_classes": [("object_class",)], - "parameter_definitions": [("object_class", "parameter", expected_map)], + "entity_classes": [["object_class", []]], + "parameter_definitions": [["object_class", "parameter", expected_map]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -2000,7 +2029,7 @@ def test_filter_regular_expression_in_root_mapping(self): data = iter(input_data) mapping_root = unflatten([EntityClassMapping(0, filter_re="B"), EntityMapping(1)]) out, errors = get_mapped_data(data, [mapping_root]) - expected = {"entity_classes": [("B",)], "entities": [("B", "r")]} + expected = {"entity_classes": [["B", []]], "entities": [["B", "r"]]} self.assertFalse(errors) self.assertEqual(out, expected) @@ -2009,7 +2038,7 @@ def test_filter_regular_expression_in_child_mapping(self): data = iter(input_data) mapping_root = unflatten([EntityClassMapping(0), EntityMapping(1, filter_re="q|r")]) out, errors = get_mapped_data(data, [mapping_root]) - expected = {"entity_classes": [("A",), ("B",)], "entities": [("A", "q"), ("B", "r")]} + expected = {"entity_classes": [["A", []], ["B", []]], "entities": [["A", "q"], ["B", "r"]]} self.assertFalse(errors) self.assertEqual(out, expected) @@ -2018,7 +2047,7 @@ def test_filter_regular_expression_in_child_mapping_filters_parent_mappings_too( data = iter(input_data) mapping_root = unflatten([EntityClassMapping(0), EntityMapping(1, filter_re="q")]) out, errors = get_mapped_data(data, [mapping_root]) - expected = {"entity_classes": [("A",)], "entities": [("A", "q")]} + expected = {"entity_classes": [["A", []]], "entities": [["A", "q"]]} self.assertFalse(errors) self.assertEqual(out, expected) @@ -2037,13 +2066,13 @@ def test_arrays_get_imported_to_correct_alternatives(self): ) out, errors = get_mapped_data(data, [mapping_root]) expected = { - "entity_classes": [("class",)], - "entities": [("class", "y")], - "parameter_definitions": [("class", "parameter")], + "entity_classes": [["class", []]], + "entities": [["class", "y"]], + "parameter_definitions": [["class", "parameter"]], "alternatives": {"Base", "alternative"}, "parameter_values": [ - ["class", "y", "parameter", Array(["p1"]), "Base"], - ["class", "y", "parameter", Array(["p1"]), "alternative"], + ["class", ("y",), "parameter", Array(["p1"]), "Base"], + ["class", ("y",), "parameter", Array(["p1"]), "alternative"], ], } self.assertFalse(errors) @@ -2114,3 +2143,39 @@ def test_mappings_are_hidden(self): root = default_import_mapping(map_type) flattened = root.flatten() self.assertTrue(all(m.position == Position.hidden for m in flattened)) + + +class TestParameterDefinitionDescriptionMapping: + def test_imports_correctly(self): + data_source = iter( + [ + ["Gadget", "weight", "Weight of a non-widget."], + ] + ) + flattened = [EntityClassMapping(0), ParameterDefinitionMapping(1), ParameterDefinitionDescriptionMapping(2)] + root_mapping = unflatten(flattened) + mapped_data, errors = get_mapped_data(data_source, [root_mapping]) + assert errors == [] + assert mapped_data == { + "entity_classes": [ + ["Gadget", []], + ], + "parameter_definitions": [["Gadget", "weight", None, None, "Weight of a non-widget."]], + } + + def test_empty_description_is_skipped(self): + data_source = iter( + [ + ["Gadget", "weight", ""], + ] + ) + flattened = [EntityClassMapping(0), ParameterDefinitionMapping(1), ParameterDefinitionDescriptionMapping(2)] + root_mapping = unflatten(flattened) + mapped_data, errors = get_mapped_data(data_source, [root_mapping]) + assert errors == [] + assert mapped_data == { + "entity_classes": [ + ["Gadget", []], + ], + "parameter_definitions": [["Gadget", "weight"]], + } diff --git a/tests/spine_io/importers/test_csv_reader.py b/tests/spine_io/importers/test_csv_reader.py index 79d22137..12b9628b 100644 --- a/tests/spine_io/importers/test_csv_reader.py +++ b/tests/spine_io/importers/test_csv_reader.py @@ -45,7 +45,7 @@ def test_get_tables_and_properties(self): self.assertEqual(len(tables), 1) self.assertTrue("data" in tables) options = tables["data"].options - self.assertEqual(options["encoding"], "ascii") + self.assertEqual(options["encoding"], "utf-8") self.assertEqual(options["delimiter"], ",") self.assertEqual(options["quotechar"], '"') self.assertEqual(options["skip"], 0) diff --git a/tests/spine_io/importers/test_datapackage_reader.py b/tests/spine_io/importers/test_datapackage_reader.py index 33ebd6eb..b23aefb9 100644 --- a/tests/spine_io/importers/test_datapackage_reader.py +++ b/tests/spine_io/importers/test_datapackage_reader.py @@ -13,6 +13,7 @@ import csv from pathlib import Path import pickle +import sys from tempfile import TemporaryDirectory import unittest from frictionless import Package, Resource @@ -53,25 +54,6 @@ def test_header_off_does_not_append_numbers_to_duplicate_cells(self): self.assertIsNone(header) self.assertEqual(list(data_iterator), data) - def test_wrong_datapackage_encoding_raises_reader_error(self): - broken_text = b"Slagn\xe4s" - # Fool the datapackage sniffing algorithm by hiding the broken line behind a large number of UTF-8 lines. - data = 1000 * [b"normal_text\n"] + [broken_text] - with TemporaryDirectory() as temp_dir: - csv_file_path = Path(temp_dir, "test_data.csv") - with open(csv_file_path, "wb") as csv_file: - for row in data: - csv_file.write(row) - package = Package(basepath=temp_dir) - package.add_resource(Resource(path=str(csv_file_path.relative_to(temp_dir)))) - package_path = Path(temp_dir, "datapackage.json") - package.to_json(package_path) - reader = DatapackageReader(None) - reader.connect_to_source(str(package_path)) - data_iterator, header = reader.get_data_iterator("test_data", {"has_header": False}) - self.assertIsNone(header) - self.assertRaises(ReaderError, list, data_iterator) - def test_get_table_cell(self): data = [["11", "12", "13"], ["21", "22", "23"]] with check_datapackage(data) as package_path: diff --git a/tests/spine_io/importers/test_reader.py b/tests/spine_io/importers/test_reader.py index 99f03ce1..d2243a8b 100644 --- a/tests/spine_io/importers/test_reader.py +++ b/tests/spine_io/importers/test_reader.py @@ -58,7 +58,7 @@ def test_get_mapped_data(self): table_row_convert_specs, ) self.assertEqual(errors, []) - self.assertEqual(mapped_data, {"entity_classes": [("A",)], "entities": [("A", "b")]}) + self.assertEqual(mapped_data, {"entity_classes": [["A", []]], "entities": [["A", "b"]]}) def test_resolve_values_for_fixed_position_mappings(self): reader = Reader(None) @@ -126,7 +126,3 @@ def raise_exception(*args): ) self.assertEqual(errors, ["this is expected"]) self.assertEqual(mapped_data, {}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/spine_io/test_excel_integration.py b/tests/spine_io/test_excel_integration.py index 713a1e0e..63d4002a 100644 --- a/tests/spine_io/test_excel_integration.py +++ b/tests/spine_io/test_excel_integration.py @@ -10,7 +10,7 @@ # this program. If not, see . ###################################################################################################################### -""" Integration tests for Excel import and export. """ +"""Integration tests for Excel import and export.""" import json from pathlib import PurePath @@ -101,10 +101,10 @@ def test_map(self): def _check_parameter_value(self, val): input_data = { - "entity_classes": {("dog",)}, + "entity_classes": {("dog", ())}, "entities": {("dog", "pluto")}, "parameter_definitions": [("dog", "bone")], - "parameter_values": [("dog", "pluto", "bone", val)], + "parameter_values": [("dog", ("pluto",), "bone", val)], } with DatabaseMapping("sqlite://", create=True) as db_map: self._assert_imports(import_data(db_map, **input_data)) @@ -133,7 +133,3 @@ def indexed_values(value, k=1, prefix=()): yield from indexed_values(new_value, k=k + 1, prefix=(*prefix, str(index))) except AttributeError: yield str(prefix), value - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_filtered_database_mapping.py b/tests/test_filtered_database_mapping.py index d0d96a88..c396e761 100644 --- a/tests/test_filtered_database_mapping.py +++ b/tests/test_filtered_database_mapping.py @@ -157,7 +157,7 @@ def test_rename_entity_to_something_that_has_been_filtered_out(tmp_path): bigglesworth = db_map.entity(name="Bigglesworth", entity_class_name="cat") bigglesworth.update(name="Tom") with pytest.raises( - SpineDBAPIError, match="^there's already a entity with \{'entity_class_name': 'cat', 'name': 'Tom'\}$" + SpineDBAPIError, match="^there's already a entity with \\{'entity_class_name': 'cat', 'name': 'Tom'\\}$" ): db_map.commit_session("Rename Bigglesworth.") assert bigglesworth.mapped_item.status == Status.to_update