Initial API straw architecture.

.gitignore

@@ -101,7 +101,7 @@ ipython_config.py
 #   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
 #   This is especially recommended for binary packages to ensure reproducibility, and is more
 #   commonly ignored for libraries.
-#uv.lock
+uv.lock
 
 # poetry
 #   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.

.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+repos:
+  - repo: https://github.com/pycqa/isort
+    rev: 6.0.1
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.17
+    hooks:
+      - id: ruff-check
+      - id: ruff-format
+        args: [--check, --diff]
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: debug-statements
+
+# Replace commitsar with conventional commit checker
+  - repo: https://github.com/compilerla/conventional-pre-commit
+    rev: v2.1.1
+    hooks:
+      - id: conventional-pre-commit
+        stages: [commit-msg]
+        args: [] # default: [feat, fix, ci, chore, docs, style, refactor, perf, test, build, revert]
+
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0
+    hooks:
+    - id: python-check-blanket-noqa
+    - id: python-no-eval

pyproject.toml

@@ -13,15 +13,100 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
 ]
-dependencies = ["numpy"]
-
-[project.optional-dependencies]
-dev = [
-    "pytest"
+dependencies = [
+    "click",
+    "fastapi",
+    "numpy",
+    "pydantic",
+    "pydantic-settings",
+    "sqlalchemy",
+    "sqlalchemy-utils",
+    "loguru",
 ]
 
 [project.urls]
 Homepage = "https://github.com/simonsobs/archivist"
 
 [tool.setuptools_scm]
 write_to = "src/archivist/_version.py"
+
+[dependency-groups]
+dev = [
+    "coverage>=7.10.7",
+    "coveralls>=4.0.1",
+    "isort>=6.1.0",
+    "pre-commit>=4.3.0",
+    "pytest>=8.4.2",
+    "pytest-cov>=7.1.0",
+    "pytest-random-order>=1.2.0",
+    "ruff>=0.15.17",
+]
+
+[project.scripts]
+archivist = "archivist.__main__:main"
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+ignore = ["ANN001",
+          "SIM118",
+          "T201",
+          "D100",
+          "INP001",
+          "D212",
+          "PTH119",
+          "D415",
+          "D103",
+          "D411",
+          "D410",
+          "COM812",
+          "D205",
+          "S101",
+          "D202",
+          "SLF001",
+          "D401",
+          "D400",
+          "FBT001",
+          "ANN202",
+          "PTH123",
+          "FBT002",
+          "ANN201",
+          "F401",
+          "F841"
+]
+
+[tool.ruff.format]
+docstring-code-format = true
+docstring-code-line-length = 80
+
+[tool.ruff.lint.per-file-ignores]
+"**/scripts/*" = [
+  "INP001",
+  "T201",
+]
+"**/tests/**/*" = [
+  "PLC1901",
+  "PLR2004",
+  "PLR6301",
+  "S",
+  "TID252",
+  "T201",
+  "DTZ005",
+  "N802",
+  "ARG005"
+]
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.lint.isort]
+known-first-party = ["archivist"]
+
+[tool.ruff.lint.flake8-pytest-style]
+fixture-parentheses = false
+mark-parentheses = false
+
+[tool.isort]
+line_length = 120
+skip = ["_version.py"]

src/archivist/__init__.py

@@ -1,11 +1,9 @@
 # Copyright (c) 2025-2026 Simons Observatory.
 # Full license can be found in the top level "LICENSE" file.
 
-"""Archive tools for data.
-"""
+"""Archive tools for data."""
 
 from ._version import __version__
-
-from .archive import Archive
-from .registry import RegistrySqlite, RegistryLibrarian
-from .storage import StorageDisk, StorageHPSS
+from .core.archive import Archive
+from .core.registry import RegistryLibrarian, RegistrySqlite
+from .core.storage import StorageDisk, StorageHPSS

src/archivist/__main__.py

@@ -0,0 +1,61 @@
+import sys
+
+import click
+
+import Archivist
+
+
+@click.group()
+@click.option(
+    "--config",
+    "-c",
+    type=click.Path(exists=True),
+    required=False,
+    help="Path to the Archivist configuration file",
+)
+@click.pass_context
+def main(ctx, config):
+    if config is None:
+        click.echo("Archivist requires a configuration file", err=True)
+        click.echo("See readme for more information", err=True)
+        sys.exit(1)
+    ctx.ensure_object(dict)
+    ctx.obj["archivist"] = Archivist(config)
+
+
+@main.command()
+@click.option("--source-path", type=str, help="The source path of the file to archive")
+@click.option(
+    "--dest-path",
+    type=str,
+    help="The destination path where the file should be archived",
+)
+@click.pass_context
+def archive(ctx, source_path, dest_path):
+    """Command to archive a file.
+    For example: `archivist archive "--source-path /path/to/source --dest-path /path/to/dest"`
+    """
+    pass
+
+
+@main.command()
+@click.argument("cmd", nargs=1)
+@click.option("--archive", type=str, help="Name of the job")
+@click.option(
+    "--dest-path",
+    type=str,
+    help="The destination path where the file should be restored",
+)
+@click.pass_context
+def extract(ctx, archive, dest_path):
+    """Command to record a job.
+    For example: `slurmise record "-o 2 -i 3 -m fast"`
+    """
+    pass
+
+
+@main.command()
+@click.pass_context
+def start_server(ctx):
+    """Command to check the status of the archivist."""
+    pass

src/archivist/api.py

@@ -0,0 +1,16 @@
+class Archivist:
+    def __init__(self, config):
+        self.config = config
+        self.archivist = None
+
+    def start_server(self):
+        # Initialize the archivist with the provided configuration
+        self.archivist = self.config.get("archivist", {})
+
+    def archive(self, source_path, dest_path):
+        # Implementation for archiving a file
+        pass
+
+    def extract(self, archive, dest_path):
+        # Implementation for extracting an archive
+        pass

src/archivist/api/__init__.py

@@ -0,0 +1,3 @@
+from fastapi import APIRouter
+
+router = APIRouter(prefix="/api/v1")

src/archivist/api/archive.py

@@ -0,0 +1,44 @@
+import json
+import uuid
+
+from fastapi import Depends, Response
+
+from archivist.api import router
+from archivist.core.models import Archive, ManifestFailedResponse, ManifestRequest, ManifestResponse
+from archivist.queue import ArchiveQueue, get_queue
+from archivist.settings import Settings, get_settings
+
+
+@router.post("/archive", response_model=ManifestResponse | ManifestFailedResponse)
+def archive(
+    manifest_request: ManifestRequest,
+    response: Response,
+    queue: ArchiveQueue = Depends(get_queue),
+    settings: Settings = Depends(get_settings),
+):
+    """
+    Endpoint to archive a file.
+    """
+    # Here you would implement the logic to handle the archiving process
+    # For now, we will just return a dummy response
+
+    total_size = sum(entry.size for entry in manifest_request.store_files)
+    if total_size > settings.maximal_size_bytes:  # Example size limit of 1GB
+        response.status_code = 400
+        return ManifestFailedResponse(error="The total size of the files exceeds the allowed limit.")
+
+    manifest_id = uuid.uuid4()
+
+    archive = Archive(
+        manifest_id=str(manifest_id),
+        manifest=json.dumps(manifest_request),
+        paths=[entry.instance_path for entry in manifest_request.store_files],
+        root=settings.storage_root,  # Example root path
+    )
+
+    # TODO: Add the archive to a queue for processing
+    queue.enqueue_archive(archive)
+
+    return ManifestResponse(
+        manifest_id=str(manifest_id),
+    )

src/archivist/api/extract.py

@@ -0,0 +1,25 @@
+import json
+import uuid
+
+from fastapi import Depends, Response
+
+from archivist.api import router
+from archivist.core.models import Archive, ManifestFailedResponse, ManifestRequest, ManifestResponse
+from archivist.queue import ExtractQueue, get_extract_queue
+from archivist.settings import Settings, get_settings
+
+
+@router.post("/extract", response_model=ManifestResponse | ManifestFailedResponse)
+def extract(
+    manifest_request: ManifestRequest,
+    response: Response,
+    queue: ExtractQueue = Depends(get_extract_queue),
+    settings: Settings = Depends(get_settings),
+):
+    """
+    Endpoint to extract a file.
+    """
+    # Here you would implement the logic to handle the extraction process
+    # For now, we will just return a dummy response
+
+    pass

src/archivist/config.py

@@ -0,0 +1,3 @@
+class ArchivistConfiguration:
+    def __init__(self, config):
+        self.config = config

src/archivist/core/__init__.py

@@ -0,0 +1,3 @@
+from .storage import StorageDisk, StorageHPSS
+
+storage_factory = {"hpss:": StorageHPSS, "posix": StorageDisk}

src/archivist/archive.py → src/archivist/core/archive.py

@@ -2,6 +2,8 @@
 # Full license can be found in the top level "LICENSE" file.
 """Classes for working with archives."""
 
+import subprocess
+
 
 class Archive(object):
     """Class representing a single archive in a storage system.
@@ -20,18 +22,33 @@ class Archive(object):
             for all objects.
 
     """
-    def __init__(self, manifest=None, paths=None, root=None):
-        pass
+
+    def __init__(self, manifest=None, paths=None, root=None, type=None):
+        self._manifest = manifest
+        self._paths = paths
+        self._root = root
+        self._type = type
+        self._checksum = None  # Placeholder for checksum value
 
     @property
     def paths(self):
-        pass
+        return self._paths
 
     @property
     def root(self):
-        pass
+        return self._root
 
     @property
     def checksum(self):
-        pass
-
+        """Return the checksum of the archive."""
+
+    def create_archive(self):
+        """Create the archive based on the manifest or paths."""
+        # Placeholder for archive creation logic
+        if self._type == "posix":
+            # Implement logic for creating a POSIX archive
+            pass
+        elif self._type == "hpss":
+            # Implement logic for creating an HPSS archive
+            # Call subprocess to create the archive using HPSS commands
+            pass