From 931f5485f39efeff68d9694343499f9028f582bc Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Tue, 28 Apr 2026 10:23:10 +0200 Subject: [PATCH 1/2] Make sure we do not remove full gpkg when diff is missing --- server/mergin/sync/tasks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/mergin/sync/tasks.py b/server/mergin/sync/tasks.py index 3c47ddee..480222e6 100644 --- a/server/mergin/sync/tasks.py +++ b/server/mergin/sync/tasks.py @@ -102,6 +102,12 @@ def optimize_storage(project_id): if not os.path.exists(item.abs_path): continue + # skip cleanup if missing corresponding diff file - this should never happen but in case of some inconsistency keep full gpkg on disk + if not os.path.exists( + os.path.join(project.storage.project_dir, item.diff_file.location) + ): + continue + age = time.time() - os.path.getmtime(item.abs_path) if age > Configuration.FILE_EXPIRATION: move_to_tmp(item.abs_path) From 0c32c8461436058291cd85d52ed4b67942953850 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Tue, 28 Apr 2026 10:47:59 +0200 Subject: [PATCH 2/2] Fix the issue with construct gpkg diff in case of geodiff copy error Make temp changeset and destination changeset have unique names which differ to avoid removing of live diff file. --- server/mergin/sync/storages/disk.py | 2 +- .../mergin/tests/test_project_controller.py | 44 ++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/server/mergin/sync/storages/disk.py b/server/mergin/sync/storages/disk.py index 02c0c8ed..f4cb34fc 100644 --- a/server/mergin/sync/storages/disk.py +++ b/server/mergin/sync/storages/disk.py @@ -333,7 +333,7 @@ def construct_diff( # create changeset next to uploaded file copy changeset_tmp = os.path.join( os.path.dirname(uploaded_file_tmp), - diff_name, + diff_name + "_tmp", ) self.flush_geodiff_logger() logging.info( diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index 25e0e055..9318eff1 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -38,6 +38,7 @@ PushChangeType, ProjectFilePath, ) +from ..sync.storages.disk import copy_file as real_copy_file from ..sync.files import files_changes_from_upload from ..sync.schemas import ProjectListSchema from ..sync.utils import Checkpoint, generate_checksum, is_versioned_file @@ -57,6 +58,7 @@ create_project, create_workspace, DateTimeEncoder, + execute_query, login, file_info, login_as_admin, @@ -1620,6 +1622,44 @@ def test_push_no_diff_finish(client): ) assert not os.path.exists(upload.project.storage.geodiff_working_dir) + # test the same with geodiff working dir error (fallback to _tmp next to original file) + working_file = os.path.join(working_dir, "base.gpkg") + sql = "INSERT INTO simple (geometry, name) VALUES (GeomFromText('POINT(24.5, 38.2)', 4326), 'insert_test')" + execute_query(working_file, sql) + changes = { + "added": [], + "removed": [], + "updated": [ + file_info(working_dir, "base.gpkg", chunk_size=CHUNK_SIZE), + ], + } + upload, upload_dir = create_transaction("mergin", changes, version=2) + upload_chunks(upload_dir, upload.changes, src_dir=working_dir) + + def copy_file_failing_for_geodiff(src, dest): + """Mocked implementation of copy_file, failing if dest is geodiff directory.""" + if current_app.config["GEODIFF_WORKING_DIR"] in dest: + raise OSError("Mocked: copy to geodiff dir failed") + return real_copy_file(src, dest) + + with patch( + "mergin.sync.storages.disk.copy_file", + side_effect=copy_file_failing_for_geodiff, + ): + resp = client.post("/v1/project/push/finish/{}".format(upload.id)) + assert resp.status_code == 200 + latest_version = upload.project.get_latest_version() + file_meta = latest_version.changes.filter( + FileHistory.change == PushChangeType.UPDATE_DIFF.value + ).first() + assert file_meta.diff_file is not None + assert os.path.exists( + os.path.join( + upload.project.storage.project_dir, file_meta.diff_file.location + ) + ) + assert not os.path.exists(upload.project.storage.geodiff_working_dir) + # change structure of gpkg file so diff would not be available -> hard overwrite gpkg_conn = pysqlite3.connect(os.path.join(working_dir, "base.gpkg")) gpkg_conn.enable_load_extension(True) @@ -1635,7 +1675,7 @@ def test_push_no_diff_finish(client): file_info(working_dir, "test.txt", chunk_size=CHUNK_SIZE), ], } - upload, upload_dir = create_transaction("mergin", changes, version=2) + upload, upload_dir = create_transaction("mergin", changes, version=3) upload_chunks(upload_dir, upload.changes, src_dir=working_dir) resp = client.post("/v1/project/push/finish/{}".format(upload.id)) assert resp.status_code == 200 @@ -1652,7 +1692,7 @@ def test_push_no_diff_finish(client): ).count() == 2 ) - version_files = os.listdir(os.path.join(upload.project.storage.project_dir, "v3")) + version_files = os.listdir(os.path.join(upload.project.storage.project_dir, "v4")) diff_files = [f for f in version_files if re.findall("-diff-", f)] assert not diff_files