Add ability to push local (to the server) files

docs/generate_cli_docs.py

@@ -39,7 +39,9 @@ def extract_sub_commands(output: str) -> list[str]:
     sub_commands = []
     for line in output.split("\n"):
         if in_commands:
-            if line:
+            # Lines indented further than the command column are continuations
+            # of the previous command's help text.
+            if line and not line.startswith("   "):
                 sub_commands.append(line.split()[0])
         if line == "Commands:":
             in_commands = True

docs/how-to/push-pull.md

@@ -42,6 +42,55 @@ simdb simulation push SIM_ID --add-watcher
 See [watchers](../explanation/concepts.md#watchers) and the
 `simdb remote watcher` commands in the [CLI reference](../reference/cli.md).
 
+## Push on a shared file system
+
+If your machine and the server can reach the same physical file paths (as on the
+ITER network), sending large datasets over HTTP is slow and redundant. Use
+`push_local` instead:
+
+```bash
+simdb simulation push_local SIM_ID
+```
+
+`push_local` sends only the metadata and the storage paths. The server then
+
+1. validates the metadata against the active schemas,
+2. queues the file copy as a background [Celery task](operate-server/run-celery-workers.md), and
+3. completes the ingestion once the copy finishes.
+
+The command blocks and reports the ingestion state as it changes:
+
+```text
+Waiting for ingestion to complete... QUEUED -> COPYING -> COPIED -> COMPLETED
+Successfully pushed simulation UUID
+```
+
+### Configure partitions
+
+For `push_local` to resolve files on both sides, client and server must agree on
+a set of *partitions*: short logical names mapped to absolute directories. Add a
+`[partition]` section to your client configuration (see
+[Client configuration](../reference/configuration.md#partition)):
+
+```ini
+[partition]
+data = /home/user/my_simdb_data
+work = /work/imas/shared
+sdcc = /
+```
+
+Mapping `sdcc` to the system root makes any path under `/sdcc/projects/...`
+match, so `/sdcc/projects/my_run` becomes `sdcc:sdcc/projects/my_run`. When
+several partitions contain a file the most specific (deepest) path wins, so a
+catch-all mapping like this never shadows the others.
+
+When you run `push_local`, SimDB checks every input and output path against your
+partitions. A path inside a partition is rewritten to a partition-relative URI —
+`/home/user/my_simdb_data/scenarios/run1.txt` becomes
+`data:scenarios/run1.txt`. The server resolves that URI against its own
+`[partition]` configuration, so the two sides may mount the same storage at
+different absolute paths.
+
 ## Pull
 
 Pull copies a simulation's metadata into your local catalogue and downloads its

docs/reference/configuration.md

@@ -90,6 +90,17 @@ One section per configured remote server. Manage these with
 | --- | --- |
 | `file` | Path to the local SQLite catalogue. Defaults to `sim.db` in the user data directory (for example `~/.local/share/simdb/sim.db`). |
 
+### `[partition]`
+
+Maps logical partition names to absolute directories on this machine. Used by
+`simdb simulation push_local` to rewrite file paths into partition-relative URIs
+that the server can resolve (see
+[Push and pull simulations](../how-to/push-pull.md#configure-partitions)).
+
+| Option | Description |
+| --- | --- |
+| `NAME` | Directory that partition `NAME` is mounted at, for example `data = /home/user/my_simdb_data`. |
+
 ### `[development]`
 
 | Option | Description |

pyproject.toml

@@ -39,7 +39,7 @@ dependencies = [
     "distro>=1.8.0",
     "email-validator>=1.1",
     "imas-python>=2.0.1",
-    "netCDF4>=1.5",
+    "netCDF4>=1.7.2",
     "numpy>=1.14",
     "pydantic>=2.10.6",
     "python-dateutil>=2.6",

scripts/test_v13_ingestion.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """Test script for v1.3 simulation ingestion against a running server."""
-from simdb.workers.tasks import _calculate_checksum
+from simdb.checksum import calculate_checksum
 from pathlib import Path
 
 import base64
@@ -37,7 +37,7 @@
 
 
 def generate_simulation_file():
-    checksum = _calculate_checksum(Path("tmp/partition_data/subdir/test_file.txt"))
+    checksum = calculate_checksum(Path("tmp/partition_data/subdir/test_file.txt"))
     return FileData(
         type="FILE",
         uri="data:///subdir/test_file.txt",
@@ -46,7 +46,7 @@ def generate_simulation_file():
     )
 
 def generate_imas_file(relative_path):
-    checksum = _calculate_checksum(Path(f"tmp/partition_data/subdir/{relative_path}"))
+    checksum = calculate_checksum(Path(f"tmp/partition_data/subdir/{relative_path}"))
     return FileData(
         type="IMAS",
         uri=f"data:///subdir/{relative_path}",

src/simdb/checksum.py

@@ -4,6 +4,19 @@
 from simdb.imas.utils import SimDBUrl
 
 
+def calculate_checksum(path: Path) -> str:
+    """Generate a SHA1 checksum from the file at the given path.
+
+    :param path: the path of the file to checksum
+    :return: a string containing the hex representation of the computed SHA1 checksum
+    """
+    sha1 = hashlib.sha1()
+    with path.open("rb") as file:
+        for chunk in iter(lambda: file.read(4096), b""):
+            sha1.update(chunk)
+    return sha1.hexdigest()
+
+
 def sha1_checksum(uri: SimDBUrl) -> str:
     """Generate a SHA1 checksum from the given file.
 
@@ -21,8 +34,4 @@ def sha1_checksum(uri: SimDBUrl) -> str:
     if not path.is_file():
         raise ValueError("File appears to be a directory")
 
-    sha1 = hashlib.sha1()
-    with path.open("rb") as file:
-        for chunk in iter(lambda: file.read(4096), b""):
-            sha1.update(chunk)
-    return sha1.hexdigest()
+    return calculate_checksum(path)