diff --git a/MANIFEST.in b/MANIFEST.in index 9092bc91..f152ebda 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ recursive-include * nomad_plugin.yaml graft src/nomad_simulation_parsers/example_uploads +recursive-include src/nomad_simulation_parsers/parsers/h5md/schemas *.yaml diff --git a/pyproject.toml b/pyproject.toml index 42ca5700..a55ed985 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ md = [ "nomad-simulations[md]@git+https://github.com/nomad-coe/nomad-simulations.git@develop", "MDAnalysis>=2.4.0", ] -dev = ["ruff", "pytest", "structlog"] +dev = ["ruff", "pytest", "structlog", "h5py", "pyyaml"] [tool.ruff] # Exclude a variety of commonly ignored directories. @@ -131,6 +131,9 @@ line-ending = "auto" [tool.setuptools] package-dir = { "" = "src" } +[tool.setuptools.package-data] +"nomad_simulation_parsers.parsers.h5md.schemas" = ["*.yaml"] + [tool.setuptools.packages.find] where = ["src"] diff --git a/src/nomad_simulation_parsers/parsers/gromacs/parser.py b/src/nomad_simulation_parsers/parsers/gromacs/parser.py index 7730e165..7ca031ff 100644 --- a/src/nomad_simulation_parsers/parsers/gromacs/parser.py +++ b/src/nomad_simulation_parsers/parsers/gromacs/parser.py @@ -309,9 +309,7 @@ def get_thermostat_type(self, tcoupl: str) -> str | None: ) @staticmethod - def _get_grpopts_scalar( - input_params: dict[str, Any], key: str - ) -> Any: + def _get_grpopts_scalar(input_params: dict[str, Any], key: str) -> Any: """Read a scalar from grpopts[key], falling back to input_params[key]. Keys are stored with hyphens in grpopts and with underscores at the diff --git a/src/nomad_simulation_parsers/parsers/h5md/schema.py b/src/nomad_simulation_parsers/parsers/h5md/schema.py new file mode 100644 index 00000000..c8e8b0b4 --- /dev/null +++ b/src/nomad_simulation_parsers/parsers/h5md/schema.py @@ -0,0 +1,338 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from importlib import resources +from pathlib import Path +from typing import Any + +import numpy as np + +try: + import h5py +except ImportError: + h5py = None + +try: + import yaml +except ImportError: + yaml = None + + +DEFAULT_SCHEMA = 'h5md-nomad.schema.yaml' + + +@dataclass +class ValidationIssue: + path: str + message: str + + +@dataclass +class ValidationResult: + issues: list[ValidationIssue] = field(default_factory=list) + + @property + def is_valid(self) -> bool: + return not self.issues + + def add(self, path: str, message: str) -> None: + self.issues.append(ValidationIssue(path=path, message=message)) + + +def load_schema(path: str | Path | None = None) -> dict[str, Any]: + """Load a H5MD schema DSL document from YAML.""" + if yaml is None: + raise RuntimeError('PyYAML is required to load H5MD schema files.') + + if path is None: + schema_text = ( + resources.files('nomad_simulation_parsers.parsers.h5md.schemas') + .joinpath(DEFAULT_SCHEMA) + .read_text() + ) + return yaml.safe_load(schema_text) + + with Path(path).open() as schema_file: + return yaml.safe_load(schema_file) + + +def validate_hdf5_file( + path: str | Path, schema: dict[str, Any] | None = None +) -> ValidationResult: + """Validate an HDF5 file against a H5MD schema DSL document.""" + if h5py is None: + raise RuntimeError('h5py is required to validate HDF5 files.') + + if schema is None: + schema = load_schema() + with h5py.File(path, 'r') as h5_file: + return validate_hdf5(h5_file, schema) + + +def validate_hdf5(h5_object: Any, schema: dict[str, Any]) -> ValidationResult: + """Validate an open h5py file or group against a H5MD schema DSL document.""" + if h5py is None: + raise RuntimeError('h5py is required to validate HDF5 files.') + + result = ValidationResult() + dimensions: dict[str, int] = {} + _validate_node(h5_object, schema['root'], '/', result, dimensions) + return result + + +def _validate_node( + h5_object: Any, + node_schema: dict[str, Any], + path: str, + result: ValidationResult, + dimensions: dict[str, int], +) -> None: + if 'any_of' in node_schema: + _validate_any_of(h5_object, node_schema, path, result, dimensions) + return + + node_type = node_schema.get('type', 'group') + if node_type == 'group': + _validate_group(h5_object, node_schema, path, result, dimensions) + elif node_type == 'dataset': + _validate_dataset(h5_object, node_schema, path, result, dimensions) + else: + result.add(path, f'Unsupported node type {node_type!r}.') + + +def _validate_any_of( + h5_object: Any, + node_schema: dict[str, Any], + path: str, + result: ValidationResult, + dimensions: dict[str, int], +) -> None: + issues_by_alternative: list[list[ValidationIssue]] = [] + for alternative in node_schema['any_of']: + alternative_result = ValidationResult() + alternative_dimensions = dimensions.copy() + _validate_node( + h5_object, + alternative, + path, + alternative_result, + alternative_dimensions, + ) + if alternative_result.is_valid: + dimensions.clear() + dimensions.update(alternative_dimensions) + return + issues_by_alternative.append(alternative_result.issues) + + result.add(path, 'Does not match any allowed schema alternative.') + if issues_by_alternative and issues_by_alternative[0]: + result.issues.extend(issues_by_alternative[0]) + + +def _validate_group( + group: Any, + group_schema: dict[str, Any], + path: str, + result: ValidationResult, + dimensions: dict[str, int], +) -> None: + if not isinstance(group, h5py.Group): + result.add(path, 'Expected HDF5 group.') + return + + _validate_attributes( + group.attrs, + group_schema.get('attributes', {}), + path, + result, + dimensions, + ) + + for name, child_schema in group_schema.get('children', {}).items(): + child_path = _join_path(path, name) + required = child_schema.get('required', True) + child_type = child_schema.get('type', 'group') + + if child_type in {'soft_link', 'external_link'}: + _validate_link(group, name, child_schema, child_path, result) + continue + + if name not in group: + if required: + result.add(child_path, 'Missing required child.') + continue + + _validate_node(group[name], child_schema, child_path, result, dimensions) + + +def _validate_dataset( + dataset: Any, + dataset_schema: dict[str, Any], + path: str, + result: ValidationResult, + dimensions: dict[str, int], +) -> None: + if not isinstance(dataset, h5py.Dataset): + result.add(path, 'Expected HDF5 dataset.') + return + + _validate_dtype(dataset.dtype, dataset_schema.get('dtype'), path, result) + _validate_shape( + dataset.shape, dataset_schema.get('shape'), path, result, dimensions + ) + _validate_attributes( + dataset.attrs, + dataset_schema.get('attributes', {}), + path, + result, + dimensions, + ) + + +def _validate_link( + group: Any, + name: str, + link_schema: dict[str, Any], + path: str, + result: ValidationResult, +) -> None: + required = link_schema.get('required', True) + link = group.get(name, getlink=True) + if link is None: + if required: + result.add(path, 'Missing required link.') + return + + link_type = link_schema.get('type') + expected_link_type = { + 'soft_link': h5py.SoftLink, + 'external_link': h5py.ExternalLink, + }[link_type] + if not isinstance(link, expected_link_type): + result.add(path, f'Expected {link_type}.') + + +def _validate_attributes( + attributes: Any, + attribute_schemas: dict[str, Any], + owner_path: str, + result: ValidationResult, + dimensions: dict[str, int], +) -> None: + for name, attribute_schema in attribute_schemas.items(): + if owner_path == '/': + attribute_path = f'{owner_path}@{name}' + else: + attribute_path = f'{owner_path}/@{name}' + required = attribute_schema.get('required', True) + if name not in attributes: + if required: + result.add(attribute_path, 'Missing required attribute.') + continue + + value = attributes[name] + _validate_dtype( + np.asarray(value).dtype, + attribute_schema.get('dtype'), + attribute_path, + result, + ) + _validate_shape( + np.shape(value), + attribute_schema.get('shape'), + attribute_path, + result, + dimensions, + ) + + +def _validate_dtype( + actual_dtype: Any, + expected_dtype: str | list[str] | None, + path: str, + result: ValidationResult, +) -> None: + if expected_dtype is None or expected_dtype == 'any': + return + + expected_dtypes = ( + [expected_dtype] if isinstance(expected_dtype, str) else expected_dtype + ) + if any( + _matches_dtype(actual_dtype, dtype, path, result, report=False) + for dtype in expected_dtypes + ): + return + + expected = ', '.join(expected_dtypes) + result.add(path, f'Expected dtype {expected}, got {actual_dtype}.') + + +def _matches_dtype( + actual_dtype: Any, + expected_dtype: str, + path: str, + result: ValidationResult, + report: bool = True, +) -> bool: + kind = np.dtype(actual_dtype).kind + expected_kinds = { + 'bool': {'b'}, + 'boolean': {'b'}, + 'float': {'f'}, + 'integer': {'i', 'u'}, + 'numeric': {'f', 'i', 'u'}, + 'string': {'O', 'S', 'U'}, + } + + if expected_dtype not in expected_kinds: + if report: + result.add(path, f'Unsupported dtype constraint {expected_dtype!r}.') + return False + + matches = kind in expected_kinds[expected_dtype] + if not matches and report: + result.add(path, f'Expected dtype {expected_dtype}, got {actual_dtype}.') + return matches + + +def _validate_shape( + actual_shape: tuple[int, ...], + expected_shape: list[Any] | None, + path: str, + result: ValidationResult, + dimensions: dict[str, int], +) -> None: + if expected_shape is None: + return + + expected_shape = list(expected_shape) + if len(actual_shape) != len(expected_shape): + result.add( + path, + f'Expected rank {len(expected_shape)}, got rank {len(actual_shape)}.', + ) + return + + for index, (actual, expected) in enumerate(zip(actual_shape, expected_shape)): + if expected in {'*', 'any'}: + continue + if isinstance(expected, int): + if actual != expected: + result.add(path, f'Expected shape[{index}]={expected}, got {actual}.') + continue + if isinstance(expected, str): + previous = dimensions.setdefault(expected, actual) + if previous != actual: + result.add( + path, + f'Expected dimension {expected!r}={previous}, got {actual}.', + ) + continue + result.add(path, f'Unsupported shape constraint {expected!r}.') + + +def _join_path(parent: str, child: str) -> str: + if parent == '/': + return f'/{child}' + return f'{parent}/{child}' diff --git a/src/nomad_simulation_parsers/parsers/h5md/schemas/__init__.py b/src/nomad_simulation_parsers/parsers/h5md/schemas/__init__.py new file mode 100644 index 00000000..cfc779fe --- /dev/null +++ b/src/nomad_simulation_parsers/parsers/h5md/schemas/__init__.py @@ -0,0 +1 @@ +"""Packaged H5MD schema DSL files.""" diff --git a/src/nomad_simulation_parsers/parsers/h5md/schemas/h5md-nomad.schema.yaml b/src/nomad_simulation_parsers/parsers/h5md/schemas/h5md-nomad.schema.yaml new file mode 100644 index 00000000..f958eb7d --- /dev/null +++ b/src/nomad_simulation_parsers/parsers/h5md/schemas/h5md-nomad.schema.yaml @@ -0,0 +1,206 @@ +schema: h5md-nomad +schema_version: 0.1.0 +description: > + First machine-readable slice of the H5MD-NOMAD HDF5 layout. This focuses on + the structural header and the core particles/all trajectory groups used by + the current parser/export discussion. It is intentionally an H5MD-NOMAD + profile, not a complete strict H5MD schema. +limitations: + - Fixed-step storage with scalar step/time increments and offsets is not yet modeled. + - Hard-link equality between shared step/time datasets is not yet enforced. + - Observables, connectivity, and parameters are only represented as optional root groups. + +root: + type: group + children: + h5md: + type: group + required: true + attributes: + version: + dtype: integer + shape: [2] + required: true + children: + author: + type: group + required: true + attributes: + name: + dtype: string + shape: [] + required: true + email: + dtype: string + shape: [] + required: false + creator: + type: group + required: true + attributes: + name: + dtype: string + shape: [] + required: true + version: + dtype: string + shape: [] + required: true + program: + type: group + required: false + attributes: + name: + dtype: string + shape: [] + required: true + version: + dtype: string + shape: [] + required: true + + particles: + type: group + required: false + children: + all: + type: group + required: true + children: + box: + type: group + required: true + attributes: + dimension: + dtype: integer + shape: [] + required: false + boundary: + description: > + H5MD stores boundary as strings such as periodic/none. + H5MD-NOMAD also accepts the documented boolean form where + true means periodic. + dtype: [bool, string] + shape: [spatial_dimension] + required: false + children: + edges: + required: false + any_of: + - type: group + attributes: + unit: + dtype: string + shape: [] + required: false + unit_factor: + dtype: numeric + shape: [] + required: false + children: + step: + type: dataset + dtype: integer + shape: [box_frames] + required: true + time: + type: dataset + dtype: numeric + shape: [box_frames] + required: false + value: + required: true + any_of: + - type: dataset + dtype: numeric + shape: [box_frames, spatial_dimension] + - type: dataset + dtype: numeric + shape: [box_frames, spatial_dimension, spatial_dimension] + - type: dataset + dtype: numeric + shape: [spatial_dimension] + - type: dataset + dtype: numeric + shape: [spatial_dimension, spatial_dimension] + position: + type: group + required: false + attributes: + unit: + dtype: string + shape: [] + required: false + unit_factor: + dtype: numeric + shape: [] + required: false + children: + step: + type: dataset + dtype: integer + shape: [position_frames] + required: true + time: + type: dataset + dtype: numeric + shape: [position_frames] + required: false + value: + type: dataset + dtype: numeric + shape: [position_frames, n_particles, spatial_dimension] + required: true + velocity: + type: group + required: false + attributes: + unit: + dtype: string + shape: [] + required: false + unit_factor: + dtype: numeric + shape: [] + required: false + children: + step: + type: dataset + dtype: integer + shape: [velocity_frames] + required: true + time: + type: dataset + dtype: numeric + shape: [velocity_frames] + required: false + value: + type: dataset + dtype: numeric + shape: [velocity_frames, n_particles, spatial_dimension] + required: true + species_label: + type: dataset + description: H5MD-NOMAD extension for particle species labels. + dtype: string + shape: [n_particles] + required: false + id: + type: group + required: false + children: + value: + type: dataset + dtype: integer + shape: [n_particles] + required: true + + observables: + type: group + required: false + connectivity: + type: group + required: false + parameters: + type: group + required: false diff --git a/tests/parsers/test_h5md_schema.py b/tests/parsers/test_h5md_schema.py new file mode 100644 index 00000000..30fe9475 --- /dev/null +++ b/tests/parsers/test_h5md_schema.py @@ -0,0 +1,149 @@ +from pathlib import Path + +import h5py +import numpy as np + +from nomad_simulation_parsers.parsers.h5md.schema import ( + load_schema, + validate_hdf5_file, +) + + +def _write_minimal_h5md_file(path: Path) -> None: + with h5py.File(path, 'w') as h5_file: + h5md = h5_file.create_group('h5md') + h5md.attrs['version'] = np.array([1, 0], dtype=np.int32) + + author = h5md.create_group('author') + author.attrs['name'] = 'Ada Lovelace' + author.attrs['email'] = 'ada@example.org' + + creator = h5md.create_group('creator') + creator.attrs['name'] = 'h5py' + creator.attrs['version'] = '3.0.0' + + particles = h5_file.create_group('particles') + particles_all = particles.create_group('all') + box = particles_all.create_group('box') + box.attrs['dimension'] = np.int32(3) + box.attrs['boundary'] = np.array([False, False, False]) + + position = particles_all.create_group('position') + position.attrs['unit'] = 'angstrom' + position.create_dataset('step', data=np.array([0, 1], dtype=np.int64)) + position.create_dataset('time', data=np.array([0.0, 0.5])) + position.create_dataset('value', data=np.ones((2, 3, 3))) + particles_all.create_dataset( + 'species_label', + data=np.array([b'H', b'O', b'H']), + ) + + +def test_h5md_schema_validates_minimal_core_file(tmp_path): + mainfile = tmp_path / 'minimal.h5' + _write_minimal_h5md_file(mainfile) + + result = validate_hdf5_file(mainfile, load_schema()) + + assert result.is_valid, [ + f'{issue.path}: {issue.message}' for issue in result.issues + ] + + +def test_h5md_schema_reports_missing_required_header(tmp_path): + mainfile = tmp_path / 'missing_creator.h5' + _write_minimal_h5md_file(mainfile) + with h5py.File(mainfile, 'a') as h5_file: + del h5_file['h5md/creator'] + + result = validate_hdf5_file(mainfile, load_schema()) + + assert not result.is_valid + assert any(issue.path == '/h5md/creator' for issue in result.issues) + + +def test_h5md_schema_checks_attribute_dimensions_against_datasets(tmp_path): + mainfile = tmp_path / 'invalid_boundary_dimension.h5' + _write_minimal_h5md_file(mainfile) + with h5py.File(mainfile, 'a') as h5_file: + box = h5_file['particles/all/box'] + box.attrs['boundary'] = np.array([True, True]) + + result = validate_hdf5_file(mainfile, load_schema()) + + assert not result.is_valid + assert any( + issue.path == '/particles/all/position/value' + and "Expected dimension 'spatial_dimension'=2, got 3." == issue.message + for issue in result.issues + ) + + +def test_h5md_schema_allows_optional_time_dataset(tmp_path): + mainfile = tmp_path / 'without_time.h5' + _write_minimal_h5md_file(mainfile) + with h5py.File(mainfile, 'a') as h5_file: + del h5_file['particles/all/position/time'] + + result = validate_hdf5_file(mainfile, load_schema()) + + assert result.is_valid, [ + f'{issue.path}: {issue.message}' for issue in result.issues + ] + + +def test_h5md_schema_allows_different_element_frame_counts(tmp_path): + mainfile = tmp_path / 'different_frame_counts.h5' + _write_minimal_h5md_file(mainfile) + with h5py.File(mainfile, 'a') as h5_file: + velocity = h5_file['particles/all'].create_group('velocity') + velocity.create_dataset('step', data=np.array([0], dtype=np.int64)) + velocity.create_dataset('value', data=np.ones((1, 3, 3))) + + result = validate_hdf5_file(mainfile, load_schema()) + + assert result.is_valid, [ + f'{issue.path}: {issue.message}' for issue in result.issues + ] + + +def test_h5md_schema_allows_string_boundary_and_static_edges(tmp_path): + mainfile = tmp_path / 'string_boundary_static_edges.h5' + _write_minimal_h5md_file(mainfile) + with h5py.File(mainfile, 'a') as h5_file: + box = h5_file['particles/all/box'] + box.attrs['boundary'] = np.array([b'periodic', b'periodic', b'none']) + box.create_dataset('edges', data=np.eye(3)) + + result = validate_hdf5_file(mainfile, load_schema()) + + assert result.is_valid, [ + f'{issue.path}: {issue.message}' for issue in result.issues + ] + + +def test_h5md_schema_requires_box_for_particles_group(tmp_path): + mainfile = tmp_path / 'missing_box.h5' + _write_minimal_h5md_file(mainfile) + with h5py.File(mainfile, 'a') as h5_file: + del h5_file['particles/all/box'] + + result = validate_hdf5_file(mainfile, load_schema()) + + assert not result.is_valid + assert any(issue.path == '/particles/all/box' for issue in result.issues) + + +def test_h5md_schema_validates_reference_fixture(): + mainfile = ( + Path(__file__).resolve().parents[1] + / 'data' + / 'h5md' + / 'test_traj_openmm_reduced-SOL_5frames_07-10-25.h5' + ) + + result = validate_hdf5_file(mainfile, load_schema()) + + assert result.is_valid, [ + f'{issue.path}: {issue.message}' for issue in result.issues + ]