From f9c3144b335902752ef97c3f5781de3be0a241d3 Mon Sep 17 00:00:00 2001 From: Ioannis Paraskevakos Date: Wed, 1 Jul 2026 13:11:18 -0400 Subject: [PATCH 1/2] feat: functional server --- pyproject.toml | 1 + src/archivist/__main__.py | 19 ++++++++++++--- src/archivist/api/__init__.py | 3 +++ src/archivist/api/archive.py | 4 +-- src/archivist/api/health.py | 20 +++++++++++++++ src/archivist/server.py | 46 +++++++++++++++++++++++++++++++++++ src/archivist/settings.py | 18 +++++++++++++- 7 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 src/archivist/api/health.py create mode 100644 src/archivist/server.py diff --git a/pyproject.toml b/pyproject.toml index 4236756..75c7958 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "sqlalchemy", "sqlalchemy-utils", "loguru", + "uvicorn", ] [project.urls] diff --git a/src/archivist/__main__.py b/src/archivist/__main__.py index 0b84386..80264d0 100644 --- a/src/archivist/__main__.py +++ b/src/archivist/__main__.py @@ -1,9 +1,8 @@ +import os import sys import click -import Archivist - @click.group() @click.option( @@ -19,8 +18,9 @@ def main(ctx, config): click.echo("Archivist requires a configuration file", err=True) click.echo("See readme for more information", err=True) sys.exit(1) + + os.environ["ARCHIVIST_CONFIG_PATH"] = config ctx.ensure_object(dict) - ctx.obj["archivist"] = Archivist(config) @main.command() @@ -58,4 +58,15 @@ def extract(ctx, archive, dest_path): @click.pass_context def start_server(ctx): """Command to check the status of the archivist.""" - pass + + import uvicorn + + from .settings import server_settings + + uvicorn.run( + "archivist.server:main", + host=server_settings.host, + port=server_settings.port, + log_level=server_settings.log_level.lower(), + factory=True, + ) diff --git a/src/archivist/api/__init__.py b/src/archivist/api/__init__.py index 91129c5..66a137b 100644 --- a/src/archivist/api/__init__.py +++ b/src/archivist/api/__init__.py @@ -1,3 +1,6 @@ from fastapi import APIRouter router = APIRouter(prefix="/api/v1") +health_router = APIRouter() + +from . import archive, extract, health # noqa: E402,F401 diff --git a/src/archivist/api/archive.py b/src/archivist/api/archive.py index 16d09e4..1b41b0d 100644 --- a/src/archivist/api/archive.py +++ b/src/archivist/api/archive.py @@ -5,7 +5,7 @@ from archivist.api import router from archivist.core.models import Archive, ManifestFailedResponse, ManifestRequest, ManifestResponse -from archivist.queue import ArchiveQueue, get_queue +from archivist.queue import ArchiveQueue, get_archive_queue from archivist.settings import Settings, get_settings @@ -13,7 +13,7 @@ def archive( manifest_request: ManifestRequest, response: Response, - queue: ArchiveQueue = Depends(get_queue), + queue: ArchiveQueue = Depends(get_archive_queue), settings: Settings = Depends(get_settings), ): """ diff --git a/src/archivist/api/health.py b/src/archivist/api/health.py new file mode 100644 index 0000000..bb01714 --- /dev/null +++ b/src/archivist/api/health.py @@ -0,0 +1,20 @@ +from typing import Any + +from fastapi import Depends + +from archivist.api import health_router +from archivist.settings import Settings, get_settings + + +@health_router.get("/health") +def health( + settings: Settings = Depends(get_settings), +) -> dict[str, Any]: + """ + Health check endpoint used by orchestrators/load balancers to confirm + the server is up and responding. + """ + + return_dict = {"name": settings.displayed_site_name, "status": "ok"} + + return return_dict diff --git a/src/archivist/server.py b/src/archivist/server.py new file mode 100644 index 0000000..36d0593 --- /dev/null +++ b/src/archivist/server.py @@ -0,0 +1,46 @@ +""" +The Archivist v2.0 server. +""" + +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from .settings import server_settings + + +@asynccontextmanager +async def slack_post_at_startup_shutdown(app: FastAPI): + """ + Lifespan event that posts to the slack hook once + the FastAPI server starts up and shuts down. + """ + from loguru import logger + + logger.info("Archivist server starting up") + yield + logger.info("Archivist server shutting down") + + +def main() -> FastAPI: + from loguru import logger + + logger.info("Starting Archivist server.") + logger.debug("Creating FastAPI app instance.") + + app = FastAPI( + title=server_settings.displayed_site_name, + description=server_settings.displayed_site_description, + openapi_url="/api/v2/openapi.json" if server_settings.debug else None, + lifespan=slack_post_at_startup_shutdown, + ) + + logger.debug("Adding API router.") + + from .api import health_router + from .api import router as api_router + + app.include_router(api_router) + app.include_router(health_router) + + return app diff --git a/src/archivist/settings.py b/src/archivist/settings.py index fb2eb5f..c425e60 100644 --- a/src/archivist/settings.py +++ b/src/archivist/settings.py @@ -126,7 +126,7 @@ def get_settings() -> "Settings": global _settings try_paths = [ - os.environ.get("LIBRARIAN_CONFIG_PATH", None), + os.environ.get("ARCHIVIST_CONFIG_PATH", None), ] for path in try_paths: @@ -150,3 +150,19 @@ def get_settings() -> "Settings": raise e return _settings + + +def __getattr__(name) -> Settings: + """ + Try to load the settings if they haven't been loaded yet. + """ + + if name == "server_settings": + global _settings + + if _settings is not None: + return _settings + + return get_settings() + + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") From 2e95253dcc25278e6a9752f70f34276dc446125f Mon Sep 17 00:00:00 2001 From: Ioannis Paraskevakos Date: Wed, 1 Jul 2026 15:21:11 -0400 Subject: [PATCH 2/2] feat: server responds to requests --- pyproject.toml | 1 + src/archivist/__main__.py | 81 +++++++++++++++++++++++++++++++++--- src/archivist/api/archive.py | 13 ++++-- 3 files changed, 87 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 75c7958..2e2357e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "sqlalchemy-utils", "loguru", "uvicorn", + "requests>=2.32.5", ] [project.urls] diff --git a/src/archivist/__main__.py b/src/archivist/__main__.py index 80264d0..318ecec 100644 --- a/src/archivist/__main__.py +++ b/src/archivist/__main__.py @@ -1,7 +1,14 @@ import os import sys +from datetime import datetime +from pathlib import Path import click +import requests + +from archivist.core.models import ManifestEntry, ManifestFailedResponse, ManifestRequest, ManifestResponse +from archivist.settings import get_settings +from archivist.utils import _checksum @click.group() @@ -24,18 +31,82 @@ def main(ctx, config): @main.command() -@click.option("--source-path", type=str, help="The source path of the file to archive") +@click.option( + "--source-path", type=click.Path(exists=True), required=True, help="The source path of the file to archive" +) @click.option( "--dest-path", - type=str, + type=click.Path(exists=False), + required=True, help="The destination path where the file should be archived", ) +@click.option( + "--librarian-name", + type=str, + required=False, + help="The name of the librarian archiving the file", + default="librarian", +) @click.pass_context -def archive(ctx, source_path, dest_path): +def archive(ctx, source_path, dest_path, librarian_name): """Command to archive a file. - For example: `archivist archive "--source-path /path/to/source --dest-path /path/to/dest"` + For example: `archivist -c config archive "--source-path /path/to/source --dest-path /path/to/dest"` """ - pass + # Here you would implement the logic to handle the archiving process + # For now, we will just print the source and destination paths + click.echo(f"Archiving file from {source_path} to {dest_path}") + + settings = get_settings() + + source_root = Path(source_path) + dest_root = Path(dest_path) if dest_path is not None else "source_root" + + if source_root.is_file(): + files = [source_root] + walk_root = source_root.parent + else: + files = [Path(dirpath) / filename for dirpath, _, filenames in os.walk(source_root) for filename in filenames] + walk_root = source_root + + uploader = librarian_name + store_files = [] + + for file_path in files: + stat = file_path.stat() + relative_path = file_path.relative_to(walk_root) + + store_files.append( + ManifestEntry( + name=file_path.name, + create_time=datetime.fromtimestamp(stat.st_ctime), + size=stat.st_size, + checksum=_checksum(file_path), + uploader=uploader, + source=str(source_root), + instance_path=str(dest_root / relative_path), + instance_create_time=datetime.now(), + instance_available=True, + outgoing_transfer_id=0, + ) + ) + + manifest_request = ManifestRequest(librarian_name=librarian_name, store_files=store_files) + try: + req = requests.post( + f"http://{settings.host}:{settings.port}/api/v1/archive", json=manifest_request.model_dump(mode="json") + ) + except (TimeoutError, requests.exceptions.ConnectionError) as ex: + raise ex + + click.echo(f"Status code: {req.status_code}") + + if req.status_code == 200: + response = ManifestResponse.model_validate(req.json()) + click.echo(f"Archive succeeded, manifest_id={response.manifest_id}") + else: + response = ManifestFailedResponse.model_validate(req.json()) + click.secho("[ERROR]", fg="red", nl=False, err=True) + click.echo(f" Archive failed: {response.error}", err=True) @main.command() diff --git a/src/archivist/api/archive.py b/src/archivist/api/archive.py index 1b41b0d..9703223 100644 --- a/src/archivist/api/archive.py +++ b/src/archivist/api/archive.py @@ -1,7 +1,7 @@ -import json import uuid from fastapi import Depends, Response +from loguru import logger from archivist.api import router from archivist.core.models import Archive, ManifestFailedResponse, ManifestRequest, ManifestResponse @@ -25,19 +25,26 @@ def archive( total_size = sum(entry.size for entry in manifest_request.store_files) if total_size > settings.maximal_size_bytes: # Example size limit of 1GB response.status_code = 400 + logger.error( + f"Archiving manifest from librarian '{manifest_request.librarian_name}' failed: total size {total_size} exceeds limit of {settings.maximal_size_bytes} bytes" + ) return ManifestFailedResponse(error="The total size of the files exceeds the allowed limit.") manifest_id = uuid.uuid4() + logger.info( + f"Archiving manifest {manifest_id} from librarian '{manifest_request.librarian_name}': {len(manifest_request.store_files)} file(s), {total_size} bytes total" + ) + archive = Archive( manifest_id=str(manifest_id), - manifest=json.dumps(manifest_request), + manifest=manifest_request.model_dump_json(), paths=[entry.instance_path for entry in manifest_request.store_files], root=settings.storage_root, # Example root path ) # TODO: Add the archive to a queue for processing - queue.enqueue_archive(archive) + # queue.enqueue_archive(archive) return ManifestResponse( manifest_id=str(manifest_id),