From 7059578eb6e67285e2a734d35c2695357181c671 Mon Sep 17 00:00:00 2001 From: Christian Chwala Date: Tue, 5 May 2026 16:42:51 +0200 Subject: [PATCH 1/5] feat(generate_config): add --ssh-keys-dir option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows SSH keys to live outside the repository root — required when generate_config.py is run from a parent deployment repository with the submodule as --repo-root. Behaviour: - --ssh-keys-dir PATH sets both the directory used by ensure_ssh_keys() for key generation/lookup and the host-side path written into docker-compose.override.yml for the authorized_keys bind-mounts. - The path is written verbatim into the compose file, so a relative path like ../ssh_keys is resolved by Docker Compose relative to the override file (i.e. relative to the submodule directory). - Default (./ssh_keys) is unchanged — no impact on existing workflows. Example usage from a deployment repo: python GMDI_prototype/scripts/generate_config.py \ --users-file users.yml \ --repo-root GMDI_prototype \ --ssh-keys-dir ../ssh_keys --- scripts/generate_config.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/scripts/generate_config.py b/scripts/generate_config.py index a902a8f..5512c9b 100644 --- a/scripts/generate_config.py +++ b/scripts/generate_config.py @@ -4,6 +4,7 @@ Usage: python scripts/generate_config.py [--users-file PATH] [--repo-root PATH] + [--ssh-keys-dir PATH] Run this script after editing users.yml, then commit all changed files. The deployed system never runs this generator; it only reads the outputs. @@ -76,14 +77,14 @@ def _validate(users: list[dict]) -> None: # --------------------------------------------------------------------------- -def _sftp_volumes_entry(user: dict) -> list[str]: +def _sftp_volumes_entry(user: dict, ssh_keys_dir: str) -> list[str]: """Named-volume and host-key-mount entries for one user.""" lines = [] for src in user["sources"]: vol = f"sftp_{user['id']}_{src['id']}" lines.append(f" - {vol}:/home/{user['id']}/uploads/{src['id']}") lines.append( - f" - ./ssh_keys/{user['id']}/authorized_keys" + f" - {ssh_keys_dir}/{user['id']}/authorized_keys" f":/home/{user['id']}/.ssh/keys/authorized_keys:ro" ) return lines @@ -132,7 +133,7 @@ def _parser_service(user: dict, src: dict) -> str: ) -def generate_compose_override(users: list[dict]) -> str: +def generate_compose_override(users: list[dict], ssh_keys_dir: str = "./ssh_keys") -> str: lines = [ "# docker-compose.override.yml", "# AUTO-GENERATED by scripts/generate_config.py — do not edit by hand.", @@ -149,7 +150,7 @@ def generate_compose_override(users: list[dict]) -> str: lines.append(f" command: [{_sftp_command_args(users)}]") lines.append(" volumes:") for user in users: - lines.extend(_sftp_volumes_entry(user)) + lines.extend(_sftp_volumes_entry(user, ssh_keys_dir)) lines.append("") @@ -530,14 +531,34 @@ def main(argv: list[str] | None = None) -> None: default=None, help="Repository root directory (default: directory of this script's parent)", ) + parser.add_argument( + "--ssh-keys-dir", + default=None, + help=( + "Directory that holds per-user SSH key subdirectories " + "(default: /ssh_keys). The path is written verbatim " + "into docker-compose.override.yml as the host side of the " + "authorized_keys bind-mount, so a relative path like ../ssh_keys " + "is resolved by Docker Compose relative to the override file." + ), + ) args = parser.parse_args(argv) script_dir = Path(__file__).resolve().parent repo_root = Path(args.repo_root) if args.repo_root else script_dir.parent users_file = Path(args.users_file) if args.users_file else repo_root / "users.yml" + # ssh_keys_dir for key generation is always an absolute/real path; + # ssh_keys_mount is the string written into the compose file (may be relative). + if args.ssh_keys_dir: + ssh_keys_mount = args.ssh_keys_dir.rstrip("/") + ssh_keys_dir = (repo_root / ssh_keys_mount).resolve() + else: + ssh_keys_mount = "./ssh_keys" + ssh_keys_dir = repo_root / "ssh_keys" print(f"Repository root : {repo_root}") print(f"Users file : {users_file}") + print(f"SSH keys dir : {ssh_keys_dir} (mount path: {ssh_keys_mount})") print() users = load_users(users_file) @@ -546,7 +567,7 @@ def main(argv: list[str] | None = None) -> None: # 1. docker-compose.override.yml override_path = repo_root / "docker-compose.override.yml" - override_path.write_text(generate_compose_override(users)) + override_path.write_text(generate_compose_override(users, ssh_keys_mount)) print(f" [compose] Written {override_path.relative_to(repo_root)}") # 2. sftp_receiver/entrypoint.sh @@ -579,7 +600,6 @@ def main(argv: list[str] | None = None) -> None: print(f" [grafana] Updated {init_grafana_path.relative_to(repo_root)}") # 7. SSH keys - ssh_keys_dir = repo_root / "ssh_keys" ensure_ssh_keys(users, ssh_keys_dir) print() From e7a6919e71dc0e4f2f6b722843a44c89fedc70bc Mon Sep 17 00:00:00 2001 From: Christian Chwala Date: Tue, 5 May 2026 16:52:30 +0200 Subject: [PATCH 2/5] ci(scripts): add test suite and coverage for generate_config.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - scripts/tests/test_generate_config.py — 7 tests covering default ssh_keys_dir behaviour, custom/absolute paths, key generation location, and end-to-end main() runs - scripts/requirements-test.txt — pytest + pyyaml - .github/workflows/test_scripts.yml — CI job mirroring parser/webserver pattern; uploads coverage to Codecov with flag 'scripts' - pyproject.toml — add scripts/tests to testpaths; add scripts to coverage source so generate_config.py appears in coverage reports --- .github/workflows/test_scripts.yml | 41 +++++++ pyproject.toml | 4 +- scripts/requirements-test.txt | 2 + scripts/tests/test_generate_config.py | 160 ++++++++++++++++++++++++++ 4 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test_scripts.yml create mode 100644 scripts/requirements-test.txt create mode 100644 scripts/tests/test_generate_config.py diff --git a/.github/workflows/test_scripts.yml b/.github/workflows/test_scripts.yml new file mode 100644 index 0000000..384d241 --- /dev/null +++ b/.github/workflows/test_scripts.yml @@ -0,0 +1,41 @@ +name: Scripts Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + paths: + - 'scripts/**' + - 'users.yml' + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + cd scripts + pip install -r requirements-test.txt + + - name: Run tests with coverage + run: | + cd scripts + pytest tests/ -v --cov=. --cov-report=xml --cov-report=term + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./scripts/coverage.xml + flags: scripts + fail_ci_if_error: false + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index 2ecc45e..bc45d01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.pytest.ini_options] -testpaths = ["tests"] +testpaths = ["tests", "scripts/tests"] python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] @@ -17,7 +17,7 @@ addopts = """ # `pytest-timeout` if you need per-test timeouts. [tool.coverage.run] -source = ["tests"] +source = ["tests", "scripts"] omit = [ "*/venv/*", "*/__pycache__/*", diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt new file mode 100644 index 0000000..3d16dc3 --- /dev/null +++ b/scripts/requirements-test.txt @@ -0,0 +1,2 @@ +pytest>=7.4.0 +pyyaml>=6.0 diff --git a/scripts/tests/test_generate_config.py b/scripts/tests/test_generate_config.py new file mode 100644 index 0000000..a298b97 --- /dev/null +++ b/scripts/tests/test_generate_config.py @@ -0,0 +1,160 @@ +"""Tests for scripts/generate_config.py — focused on the --ssh-keys-dir option.""" + +from pathlib import Path + +import pytest +import yaml + +# Import helpers directly from the script. +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from generate_config import ( + generate_compose_override, + ensure_ssh_keys, + main, +) + + +# --------------------------------------------------------------------------- +# Minimal users fixture +# --------------------------------------------------------------------------- + +USERS = [ + { + "id": "alice", + "uid": 2001, + "display_name": "Alice", + "grafana_org_id": 1, + "sources": [{"id": "main", "parser": "openmrg"}], + } +] + + +def _make_users_yml(repo_root: Path, users: list[dict] | None = None) -> Path: + data = {"users": users or USERS} + p = repo_root / "users.yml" + p.write_text(yaml.dump(data)) + return p + + +# --------------------------------------------------------------------------- +# generate_compose_override — ssh_keys_dir param +# --------------------------------------------------------------------------- + + +def test_default_ssh_keys_dir_uses_dot_slash(): + """Without --ssh-keys-dir the mount path is ./ssh_keys//...""" + output = generate_compose_override(USERS) + assert "./ssh_keys/alice/authorized_keys" in output + + +def test_custom_ssh_keys_dir_written_verbatim(): + """A relative path like ../ssh_keys is written as-is into the compose file.""" + output = generate_compose_override(USERS, ssh_keys_dir="../ssh_keys") + assert "../ssh_keys/alice/authorized_keys" in output + assert "./ssh_keys" not in output + + +def test_absolute_ssh_keys_dir_written_verbatim(tmp_path): + """An absolute path is also written verbatim.""" + abs_path = str(tmp_path / "keys") + output = generate_compose_override(USERS, ssh_keys_dir=abs_path) + assert f"{abs_path}/alice/authorized_keys" in output + + +# --------------------------------------------------------------------------- +# ensure_ssh_keys — uses the resolved absolute path for key generation +# --------------------------------------------------------------------------- + + +def test_ensure_ssh_keys_creates_keys_in_given_dir(tmp_path): + """Keys are generated under the supplied directory, not under repo_root.""" + keys_dir = tmp_path / "external_keys" + ensure_ssh_keys(USERS, keys_dir) + + priv = keys_dir / "alice" / "id_ed25519" + pub = keys_dir / "alice" / "id_ed25519.pub" + auth = keys_dir / "alice" / "authorized_keys" + + assert priv.exists(), "private key not generated" + assert pub.exists(), "public key not generated" + assert auth.exists(), "authorized_keys not created" + + +def test_ensure_ssh_keys_skips_existing_key(tmp_path): + """Existing private key is left untouched (no overwrite).""" + keys_dir = tmp_path / "keys" + user_dir = keys_dir / "alice" + user_dir.mkdir(parents=True) + priv = user_dir / "id_ed25519" + priv.write_text("EXISTING") + + ensure_ssh_keys(USERS, keys_dir) + assert priv.read_text() == "EXISTING" + + +# --------------------------------------------------------------------------- +# main() — end-to-end with --ssh-keys-dir +# --------------------------------------------------------------------------- + + +def test_main_ssh_keys_dir_override_uses_external_dir(tmp_path, monkeypatch): + """ + Running main() with --ssh-keys-dir writes ../ssh_keys paths into the + generated compose override and places keys in the external directory. + """ + # Build a minimal repo layout inside tmp_path + repo_root = tmp_path / "repo" + for subdir in ("sftp_receiver", "webserver/configs", "database/migrations", + "grafana/provisioning/datasources", "grafana", "ssh_keys"): + (repo_root / subdir).mkdir(parents=True) + + # Minimal users.yml (deployment-level, one real user) + _make_users_yml(tmp_path) # tmp_path/users.yml + + # Stub out files that main() reads/updates + (repo_root / "webserver" / "configs" / "users.json").write_text("{}") + (repo_root / "grafana" / "init_grafana.py").write_text( + "ORGS = []\nUSERS = []\n" + ) + + # External ssh_keys dir lives next to the repo (simulating deployment layout) + ext_keys_dir = tmp_path / "ssh_keys" + ext_keys_dir.mkdir() + + main([ + "--users-file", str(tmp_path / "users.yml"), + "--repo-root", str(repo_root), + "--ssh-keys-dir", str(ext_keys_dir), + ]) + + # 1. Compose override uses the external path + override = (repo_root / "docker-compose.override.yml").read_text() + assert str(ext_keys_dir) + "/alice/authorized_keys" in override + + # 2. Keys were generated in the external directory + assert (ext_keys_dir / "alice" / "id_ed25519").exists() + + # 3. Default ssh_keys inside repo_root was NOT used + assert not (repo_root / "ssh_keys" / "alice").exists() + + +def test_main_default_ssh_keys_dir_stays_inside_repo(tmp_path): + """Without --ssh-keys-dir, keys go into /ssh_keys as before.""" + repo_root = tmp_path / "repo" + for subdir in ("sftp_receiver", "webserver/configs", "database/migrations", + "grafana/provisioning/datasources", "grafana", "ssh_keys"): + (repo_root / subdir).mkdir(parents=True) + + _make_users_yml(repo_root) + (repo_root / "webserver" / "configs" / "users.json").write_text("{}") + (repo_root / "grafana" / "init_grafana.py").write_text( + "ORGS = []\nUSERS = []\n" + ) + + main(["--repo-root", str(repo_root)]) + + override = (repo_root / "docker-compose.override.yml").read_text() + assert "./ssh_keys/alice/authorized_keys" in override + assert (repo_root / "ssh_keys" / "alice" / "id_ed25519").exists() From 4b38b091c3dfc44edb559510c1f08ee608dd1c63 Mon Sep 17 00:00:00 2001 From: Christian Chwala Date: Tue, 5 May 2026 16:54:33 +0200 Subject: [PATCH 3/5] fix(scripts-ci): add pytest-cov to test requirements --- scripts/requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 3d16dc3..9a62f48 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,2 +1,3 @@ pytest>=7.4.0 +pytest-cov>=4.1.0 pyyaml>=6.0 From 02e6bcbdedb55c3e30680271d2f442434843d2ae Mon Sep 17 00:00:00 2001 From: Christian Chwala Date: Tue, 5 May 2026 17:00:34 +0200 Subject: [PATCH 4/5] fix(scripts): fix 4 test failures - generate_config.py: key_dir.mkdir(parents=True) so ensure_ssh_keys works when the parent ssh_keys dir does not pre-exist - test_generate_config.py: assert './ssh_keys/' not in output (the substring './ssh_keys' is present inside '../ssh_keys', so the trailing slash makes the check unambiguous) - test_generate_config.py: mkdir(parents=True, exist_ok=True) in both main() tests so grafana/ is not created twice from the subdir list --- scripts/generate_config.py | 2 +- scripts/tests/test_generate_config.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/generate_config.py b/scripts/generate_config.py index 5512c9b..0552419 100644 --- a/scripts/generate_config.py +++ b/scripts/generate_config.py @@ -476,7 +476,7 @@ def ensure_ssh_keys(users: list[dict], ssh_keys_dir: Path) -> None: for u in users: uid = u["id"] key_dir = ssh_keys_dir / uid - key_dir.mkdir(exist_ok=True) + key_dir.mkdir(parents=True, exist_ok=True) priv_key = key_dir / "id_ed25519" auth_keys = key_dir / "authorized_keys" diff --git a/scripts/tests/test_generate_config.py b/scripts/tests/test_generate_config.py index a298b97..6370368 100644 --- a/scripts/tests/test_generate_config.py +++ b/scripts/tests/test_generate_config.py @@ -53,7 +53,9 @@ def test_custom_ssh_keys_dir_written_verbatim(): """A relative path like ../ssh_keys is written as-is into the compose file.""" output = generate_compose_override(USERS, ssh_keys_dir="../ssh_keys") assert "../ssh_keys/alice/authorized_keys" in output - assert "./ssh_keys" not in output + # Must not contain the default prefix (note: ../ssh_keys contains the + # substring ssh_keys, so check for the full default prefix ./ssh_keys/) + assert "./ssh_keys/" not in output def test_absolute_ssh_keys_dir_written_verbatim(tmp_path): @@ -108,7 +110,7 @@ def test_main_ssh_keys_dir_override_uses_external_dir(tmp_path, monkeypatch): repo_root = tmp_path / "repo" for subdir in ("sftp_receiver", "webserver/configs", "database/migrations", "grafana/provisioning/datasources", "grafana", "ssh_keys"): - (repo_root / subdir).mkdir(parents=True) + (repo_root / subdir).mkdir(parents=True, exist_ok=True) # Minimal users.yml (deployment-level, one real user) _make_users_yml(tmp_path) # tmp_path/users.yml @@ -145,7 +147,7 @@ def test_main_default_ssh_keys_dir_stays_inside_repo(tmp_path): repo_root = tmp_path / "repo" for subdir in ("sftp_receiver", "webserver/configs", "database/migrations", "grafana/provisioning/datasources", "grafana", "ssh_keys"): - (repo_root / subdir).mkdir(parents=True) + (repo_root / subdir).mkdir(parents=True, exist_ok=True) _make_users_yml(repo_root) (repo_root / "webserver" / "configs" / "users.json").write_text("{}") From fcfa89054576ab4b61b6f15907f8251c2d288963 Mon Sep 17 00:00:00 2001 From: Christian Chwala Date: Tue, 5 May 2026 17:04:04 +0200 Subject: [PATCH 5/5] fix(scripts-tests): drop unfixable negative substring assertion '../ssh_keys/' contains './ssh_keys/' as a substring (starting at index 1), so the not-in check always fails. The positive assertion that '../ssh_keys/alice/authorized_keys' is present is sufficient. --- scripts/tests/test_generate_config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/tests/test_generate_config.py b/scripts/tests/test_generate_config.py index 6370368..83e9802 100644 --- a/scripts/tests/test_generate_config.py +++ b/scripts/tests/test_generate_config.py @@ -53,9 +53,7 @@ def test_custom_ssh_keys_dir_written_verbatim(): """A relative path like ../ssh_keys is written as-is into the compose file.""" output = generate_compose_override(USERS, ssh_keys_dir="../ssh_keys") assert "../ssh_keys/alice/authorized_keys" in output - # Must not contain the default prefix (note: ../ssh_keys contains the - # substring ssh_keys, so check for the full default prefix ./ssh_keys/) - assert "./ssh_keys/" not in output + def test_absolute_ssh_keys_dir_written_verbatim(tmp_path):