From 4ddd61acff1f29a21da9e7d003001f52bb420e05 Mon Sep 17 00:00:00 2001 From: Christian Chwala Date: Tue, 5 May 2026 19:29:08 +0200 Subject: [PATCH 1/2] fix: grafana startup crash and orange_cameroun SFTP key path Two bugs discovered during IFU deployment: 1. generate_config.py emitted datasource blocks for all orgs into postgres.yml. Grafana reads this file at startup before init_grafana has created orgs 2+, causing 'org.notFound' errors and a non-zero exit. Fix: only emit the org-1 entry; init_grafana.py already handles additional orgs via the API. Regenerate postgres.yml. 2. docker-compose.yml referenced SFTP_PRIVATE_KEY_PATH= id_rsa_demo_orange_cameroun for mno_simulator_orange_cameroun, but the actual committed key file is id_rsa_orange_cameroun (no 'demo_' prefix), causing all uploads to silently fail. --- docker-compose.yml | 2 +- grafana/provisioning/datasources/postgres.yml | 25 ++++--------------- scripts/generate_config.py | 16 ++++++++---- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index dbc8238..48c2b44 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,7 +60,7 @@ services: - SFTP_USERNAME=demo_orange_cameroun - SFTP_REMOTE_PATH=/uploads/douala - SFTP_USE_SSH_KEY=true - - SFTP_PRIVATE_KEY_PATH=/app/ssh_keys/id_rsa_demo_orange_cameroun + - SFTP_PRIVATE_KEY_PATH=/app/ssh_keys/id_rsa_orange_cameroun - SFTP_KNOWN_HOSTS_PATH=/app/ssh_keys/known_hosts - NETCDF_RSL_VAR=rsl_min - NETCDF_TSL_VAR=tsl_min diff --git a/grafana/provisioning/datasources/postgres.yml b/grafana/provisioning/datasources/postgres.yml index 9418f7c..31b8947 100644 --- a/grafana/provisioning/datasources/postgres.yml +++ b/grafana/provisioning/datasources/postgres.yml @@ -29,23 +29,8 @@ datasources: isDefault: true editable: false - # Org 2 — demo_orange_cameroun - - name: PostgreSQL - uid: ds_demo_orange_cameroun - type: grafana-postgresql-datasource - access: proxy - orgId: 2 - url: database:5432 - database: mydatabase - user: demo_orange_cameroun - secureJsonData: - password: demo_orange_cameroun_password - jsonData: - sslmode: disable - isDefault: true - editable: false - -# Note: provisioning files only apply to Grafana organisations that -# already exist when Grafana starts. Org 1 (the default) always -# exists. Additional orgs are created by the init_grafana service -# (grafana/init_grafana.py) before it triggers a provisioning reload. +# Note: only org 1 is listed above, intentionally. Grafana reads this +# file at startup, before init_grafana.py has created orgs 2+. Listing +# a non-existent org here causes Grafana to exit with 'org.notFound'. +# Datasources for additional orgs are registered by init_grafana.py via +# the Grafana API after those orgs have been created. diff --git a/scripts/generate_config.py b/scripts/generate_config.py index 0552419..d529988 100644 --- a/scripts/generate_config.py +++ b/scripts/generate_config.py @@ -370,7 +370,12 @@ def generate_grafana_datasources(users: list[dict]) -> str: " # the correct one — no user interaction required.", "", ] - for u in users: + # Only org 1 (the Grafana default org) is provisioned here. Grafana reads + # this file at startup, before init_grafana.py has had a chance to create + # orgs 2+. Provisioning a non-existent org causes Grafana to exit with + # "org.notFound". Additional orgs are created by init_grafana.py and their + # datasources are registered there via the Grafana API. + for u in users[:1]: uid = u["id"] org_id = u["grafana_org_id"] lines += [ @@ -392,10 +397,11 @@ def generate_grafana_datasources(users: list[dict]) -> str: "", ] lines += [ - "# Note: provisioning files only apply to Grafana organisations that", - "# already exist when Grafana starts. Org 1 (the default) always", - "# exists. Additional orgs are created by the init_grafana service", - "# (grafana/init_grafana.py) before it triggers a provisioning reload.", + "# Note: only org 1 is listed above, intentionally. Grafana reads this", + "# file at startup, before init_grafana.py has created orgs 2+. Listing", + "# a non-existent org here causes Grafana to exit with 'org.notFound'.", + "# Datasources for additional orgs are registered by init_grafana.py via", + "# the Grafana API after those orgs have been created.", ] return "\n".join(lines) + "\n" From a206cc1a71008de2a5f4f53765de07638b31b5cb Mon Sep 17 00:00:00 2001 From: Christian Chwala Date: Tue, 5 May 2026 20:05:51 +0200 Subject: [PATCH 2/2] fix: grant INSERT on file_processing_log to user roles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parser connects as the per-user role (e.g. demo_openmrg) and calls log_file_event() after every processed file. Both init.sql and migration 008 only granted SELECT, causing: psycopg2.errors.InsufficientPrivilege: permission denied for table file_processing_log This left the Pipeline Health dashboard and /pipeline-log page empty. The misleading comment in 008 suggested myuser (superuser) handles INSERTs — that is wrong; the parser always uses the per-user role. Fix: - init.sql and 008: GRANT SELECT, INSERT to user roles; SELECT only to webserver_role (which never writes to this table). - generate_config.py _SQL_TEMPLATE: add GRANT SELECT, INSERT and SEQUENCE USAGE so newly generated user migrations are also correct. --- database/init.sql | 3 ++- .../migrations/008_add_file_processing_log.sql | 17 ++++++++--------- scripts/generate_config.py | 5 +++++ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/database/init.sql b/database/init.sql index 11f2d78..3810773 100644 --- a/database/init.sql +++ b/database/init.sql @@ -301,6 +301,7 @@ ALTER TABLE file_processing_log FORCE ROW LEVEL SECURITY; CREATE POLICY user_isolation ON file_processing_log USING (user_id = current_user); -GRANT SELECT ON file_processing_log TO demo_openmrg, demo_orange_cameroun, webserver_role; +GRANT SELECT, INSERT ON file_processing_log TO demo_openmrg, demo_orange_cameroun; +GRANT SELECT ON file_processing_log TO webserver_role; GRANT USAGE ON SEQUENCE file_processing_log_id_seq TO demo_openmrg, demo_orange_cameroun; GRANT SELECT ON cml_data_1h_secure TO webserver_role; \ No newline at end of file diff --git a/database/migrations/008_add_file_processing_log.sql b/database/migrations/008_add_file_processing_log.sql index da03548..6eda7a1 100644 --- a/database/migrations/008_add_file_processing_log.sql +++ b/database/migrations/008_add_file_processing_log.sql @@ -22,19 +22,18 @@ CREATE INDEX IF NOT EXISTS file_processing_log_status_time_idx ON file_processing_log (status, processed_at DESC); -- Row-Level Security: each login role only sees its own rows (user_id = current_user). --- myuser (superuser) bypasses RLS so the parser continues to INSERT without restriction. +-- The parser connects as the per-user role, so RLS is enforced; INSERT is permitted +-- because the user_id column must equal current_user (guaranteed by the parser). ALTER TABLE file_processing_log ENABLE ROW LEVEL SECURITY; ALTER TABLE file_processing_log FORCE ROW LEVEL SECURITY; CREATE POLICY user_isolation ON file_processing_log USING (user_id = current_user); --- Grant read access to all user roles so Grafana dashboards can query this table. --- Each role connects to Postgres as their own login (demo_openmrg, demo_orange_cameroun) --- and Grafana reads the log for that org's datasource. --- webserver_role needs it for any admin views. -GRANT SELECT ON file_processing_log TO demo_openmrg, demo_orange_cameroun, webserver_role; +-- Grant read+write access to user roles: the parser connects as the per-user +-- role (e.g. demo_openmrg) and INSERTs a log entry for every processed file. +-- webserver_role only needs SELECT (read-only admin/dashboard view). +GRANT SELECT, INSERT ON file_processing_log TO demo_openmrg, demo_orange_cameroun; +GRANT SELECT ON file_processing_log TO webserver_role; --- Sequence used by the BIGSERIAL primary key: needed for INSERTs by the parser --- (which connects as myuser/superuser, so this is a no-op in practice, --- but keeps the intent explicit for future role changes). +-- Sequence used by the BIGSERIAL primary key: required for INSERT by user roles. GRANT USAGE ON SEQUENCE file_processing_log_id_seq TO demo_openmrg, demo_orange_cameroun; diff --git a/scripts/generate_config.py b/scripts/generate_config.py index d529988..22e7cfb 100644 --- a/scripts/generate_config.py +++ b/scripts/generate_config.py @@ -278,6 +278,11 @@ def generate_users_json(users: list[dict], existing_json: dict) -> dict: GRANT SELECT, INSERT, UPDATE ON cml_data_secure TO {user_id}; GRANT SELECT ON cml_data_1h_secure TO {user_id}; +-- file_processing_log: parser INSERTs a row for every processed file; +-- webserver_role only needs SELECT. +GRANT SELECT, INSERT ON file_processing_log TO {user_id}; +GRANT USAGE ON SEQUENCE file_processing_log_id_seq TO {user_id}; + -- --------------------------------------------------------------------------- -- Step 4: Allow webserver_role to impersonate this user -- ---------------------------------------------------------------------------