From 45e70c0052ee24cf04bab30b9656b47e4ff17a41 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 19 May 2026 23:39:01 +0530 Subject: [PATCH 1/2] =?UTF-8?q?Fix=20OSM=E2=86=92OSW=20zone=20=5Fw=5Fid=20?= =?UTF-8?q?references=20missing=20from=20nodes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Dev Board Ticket - https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/3665 - https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/3191/ ## Changes - **Zone boundary protection in `construct_geometries()`**: collect zone boundary IDs from `_w_id` lists and filter them out of `internal_nodes` before `self.G.remove_nodes_from(...)`. This prevents zone boundary nodes from being deleted by simplification edge cases (e.g., circular-way neighbor combination), so they remain in `node_id_map` for `_w_id` → sequential `_id` remapping in `to_geojson()`. - **Regression fixture**: added [tests/unit_tests/test_files/zone_boundary.xml](tests/unit_tests/test_files/zone_boundary.xml) — a minimal `highway=pedestrian` plaza plus connecting footways with simplifiable intermediate nodes. - **Regression test**: `test_osm2osw_zone_boundary_is_osw_compliant` in [test_osm_compliance.py](tests/unit_tests/test_osm_compliance/test_osm_compliance.py) runs `Formatter.osm2osw()`, zips the output, and asserts `OSWValidation` reports zero issues. - **README fixes**: - Corrected invalid Python import `from osm-osw-reformatter` → `from osm_osw_reformatter`. - Aligned conda command (`python==3.10.3` → `python=3.10`) with the documented 3.10.x requirement. - Updated sample snippet to `return` the result and access `results.generated_files`. - **Version bump**: `0.3.4` → `0.3.5` in [version.py](src/osm_osw_reformatter/version.py) with matching [CHANGELOG.md](CHANGELOG.md) entry. ## Testing - Ran the full compliance test suite — all 3 pass, including the new zone-boundary regression test: ``` pytest tests/unit_tests/test_osm_compliance/ ===== 3 passed in 4.81s ===== ``` - Validated end-to-end via the `example.py` flow (`osm2osw` → zip → `python-osw-validation`) on the fixture: zero validation issues reported. --- CHANGELOG.md | 5 ++ README.md | 11 ++-- requirements.txt | 2 +- setup.py | 1 + .../serializer/osm/osm_graph.py | 32 +++++++++++- src/osm_osw_reformatter/version.py | 2 +- tests/unit_tests/test_files/zone_boundary.xml | 44 ++++++++++++++++ .../test_osm_compliance.py | 20 +++++++ .../test_serializer/test_osm_graph.py | 52 +++++++++++++++++++ 9 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 tests/unit_tests/test_files/zone_boundary.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index adacb26..f781e04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change log +### 0.3.5 +- [BUG-3665](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/3665) - Fix OSM→OSW export so zone boundary nodes are retained in `nodes.geojson` and zone `_w_id` references resolve to remapped sequential `_id`s, restoring OSW validation compliance for pedestrian-area geometries. +- Add regression coverage that converts a `highway=pedestrian` plaza fixture and asserts `python-osw-validation` reports zero issues. +- [ISSUE-3191](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/3191/) - Fixed Documentation + ### 0.3.4 - Fix OSM→OSW conversion when OSM Way contains consecutive duplicate nodes (bug 3286). Instead of generating an invalid 0 length geometry, the segment is ignored. diff --git a/README.md b/README.md index 52f8a64..9ca1c93 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The conversion of OSW data to OSM is beneficial for incorporating detailed pedes Running the code base requires a proper Python environment set up. The following lines of code helps one establish such env named `tdei-osw`. replace `tdei-osw` with the name of your choice. ``` -conda create -n tdei-osw python==3.10.3 gdal +conda create -n tdei-osw python=3.10 gdal conda activate tdei-osw pip install -r requirements.txt ``` @@ -92,24 +92,25 @@ To install the GDAL library (Geospatial Data Abstraction Library) on your system ```python import asyncio -from osm-osw-reformatter import Formatter +from osm_osw_reformatter import Formatter async def osm_convert(): f = Formatter(workdir=, file_path=) - await f.osm2osw() + return await f.osm2osw() # Uncomment below line to clean up the generated files # f.cleanup() def osw_convert(): f = Formatter(workdir=, file_path=) - f.osw2osm() + return f.osw2osm() # Uncomment below line to clean up the generated files # f.cleanup() if __name__ == '__main__': - asyncio.run(osm_convert()) + results = asyncio.run(osm_convert()) + print(results.generated_files) osw_convert() ``` diff --git a/requirements.txt b/requirements.txt index f075ae7..133877e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ shapely~=2.0.2 pyproj~=3.6.1 coverage~=7.5.1 ogr2osm==1.2.0 -python-osw-validation==0.3.5 \ No newline at end of file +python-osw-validation==0.4.0 \ No newline at end of file diff --git a/setup.py b/setup.py index e040e16..ce66339 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ 'asyncio~=3.4.3', 'networkx~=3.2', 'shapely~=2.0.2', + 'pyproj~=3.6.1', 'ogr2osm==1.2.0' ], packages=find_packages(where='src'), diff --git a/src/osm_osw_reformatter/serializer/osm/osm_graph.py b/src/osm_osw_reformatter/serializer/osm/osm_graph.py index ea5b965..88045eb 100644 --- a/src/osm_osw_reformatter/serializer/osm/osm_graph.py +++ b/src/osm_osw_reformatter/serializer/osm/osm_graph.py @@ -588,6 +588,21 @@ def construct_geometries(self, progressbar: Optional[callable] = None) -> None: if progressbar: progressbar.update(1) + # Protect zone boundary nodes: even if they ended up in internal_nodes + # due to circular-way simplification edge cases, they must not be removed + # because zones' _w_id still references them and to_geojson() needs to + # remap those IDs to sequential _id values. + zone_boundary_ids = set() + for _n, _d in self.G.nodes(data=True): + w_ids = _d.get("_w_id") + if isinstance(w_ids, list): + for ref in w_ids: + try: + zone_boundary_ids.add(int(ref)) + except (TypeError, ValueError): + pass + if zone_boundary_ids: + internal_nodes = [n for n in internal_nodes if n not in zone_boundary_ids] self.G.remove_nodes_from(internal_nodes) def to_undirected(self): @@ -675,6 +690,20 @@ def _remap_node_ref(ref, node_id_map): zone_features = [] polygon_features = [] node_id_map = {} + zone_node_refs = set() + for _, d in self.G.nodes(data=True): + if OSWZoneNormalizer.osw_zone_filter(d): + w_ids = d.get("_w_id", d.get("ndref", [])) + if not isinstance(w_ids, list): + w_ids = [w_ids] + for ref in w_ids: + zone_node_refs.add(ref) + zone_node_refs.add(str(ref)) + try: + zone_node_refs.add(int(ref)) + except (TypeError, ValueError): + pass + for n, d in self.G.nodes(data=True): d_copy = {**d} source_id = _source_id(n) @@ -682,8 +711,9 @@ def _remap_node_ref(ref, node_id_map): geometry = mapping(geometry_obj) geometry_type = geometry_obj.geom_type is_topology_node = geometry_type == "Point" and self.G.degree(n) > 0 + is_zone_node = geometry_type == "Point" and n in zone_node_refs - if is_topology_node: + if is_topology_node or is_zone_node: _assign_ids(d_copy, node_id_counter, source_id) node_id_map[n] = d_copy["_id"] node_id_counter += 1 diff --git a/src/osm_osw_reformatter/version.py b/src/osm_osw_reformatter/version.py index bfeb9e7..40ed83d 100644 --- a/src/osm_osw_reformatter/version.py +++ b/src/osm_osw_reformatter/version.py @@ -1 +1 @@ -__version__ = '0.3.4' +__version__ = '0.3.5' diff --git a/tests/unit_tests/test_files/zone_boundary.xml b/tests/unit_tests/test_files/zone_boundary.xml new file mode 100644 index 0000000..297335a --- /dev/null +++ b/tests/unit_tests/test_files/zone_boundary.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/unit_tests/test_osm_compliance/test_osm_compliance.py b/tests/unit_tests/test_osm_compliance/test_osm_compliance.py index c21f1b7..841fbca 100644 --- a/tests/unit_tests/test_osm_compliance/test_osm_compliance.py +++ b/tests/unit_tests/test_osm_compliance/test_osm_compliance.py @@ -10,6 +10,7 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) OUTPUT_DIR = os.path.join(os.path.dirname(os.path.dirname(ROOT_DIR)), 'output') TEST_DATA_WITH_INCLINE_ZIP_FILE = os.path.join(ROOT_DIR, 'test_files/dataset_with_incline.zip') +TEST_ZONE_BOUNDARY_FILE = os.path.join(ROOT_DIR, 'test_files/zone_boundary.xml') class TestOSMCompliance(unittest.IsolatedAsyncioTestCase): @@ -37,6 +38,25 @@ async def test_output_is_osm_compliant(self): os.remove(zip_path) formatter.cleanup() + async def test_osm2osw_zone_boundary_is_osw_compliant(self): + formatter = Formatter(workdir=OUTPUT_DIR, file_path=TEST_ZONE_BOUNDARY_FILE, prefix='zone_boundary') + res = await formatter.osm2osw() + osw_files = res.generated_files + + zip_path = os.path.join(OUTPUT_DIR, 'zone_boundary_osw.zip') + with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + for f in osw_files: + zipf.write(f, os.path.basename(f)) + + validator = OSWValidation(zipfile_path=zip_path) + result = validator.validate() + self.assertEqual(len(result.issues), 0, f'OSW Validation issues: {json.dumps(result.issues)}') + + for f in osw_files: + os.remove(f) + os.remove(zip_path) + formatter.cleanup() + async def test_incline_tag_preserved(self): osw2osm = OSW2OSM( zip_file_path=TEST_DATA_WITH_INCLINE_ZIP_FILE, diff --git a/tests/unit_tests/test_serializer/test_osm_graph.py b/tests/unit_tests/test_serializer/test_osm_graph.py index 7560f5f..ea6aef5 100644 --- a/tests/unit_tests/test_serializer/test_osm_graph.py +++ b/tests/unit_tests/test_serializer/test_osm_graph.py @@ -1058,6 +1058,58 @@ def test_to_geojson_zones_reference_remapped_node_ids_in_w_id(self): zone_w_ids = zone_data["features"][0]["properties"]["_w_id"] self.assertTrue(all(wid in node_ids for wid in zone_w_ids)) + def test_to_geojson_zone_boundary_custom_points_stay_in_nodes_file(self): + graph = nx.MultiDiGraph() + graph.add_node( + 10, + geometry=Point(0, 0), + lon=0.0, + lat=0.0, + **{"ext:entrance": "yes"}, + ) + graph.add_node(20, geometry=Point(1, 0), lon=1.0, lat=0.0) + graph.add_node( + "z100", + geometry=Polygon([(0, 0), (1, 0), (0, 1), (0, 0)]), + highway="pedestrian", + _w_id=["10", "20"], + ) + + osm_graph = OSMGraph(G=graph) + + with TemporaryDirectory() as tmpdir: + nodes_path = os.path.join(tmpdir, 'nodes.geojson') + edges_path = os.path.join(tmpdir, 'edges.geojson') + points_path = os.path.join(tmpdir, 'points.geojson') + lines_path = os.path.join(tmpdir, 'lines.geojson') + zones_path = os.path.join(tmpdir, 'zones.geojson') + polygons_path = os.path.join(tmpdir, 'polygons.geojson') + + osm_graph.to_geojson( + nodes_path, + edges_path, + points_path, + lines_path, + zones_path, + polygons_path, + ) + + with open(nodes_path) as f: + node_data = json.load(f) + with open(zones_path) as f: + zone_data = json.load(f) + + node_ids = {feat["properties"]["_id"] for feat in node_data["features"]} + node_ext_ids = { + feat["properties"].get("ext:osm_id") + for feat in node_data["features"] + } + zone_w_ids = zone_data["features"][0]["properties"]["_w_id"] + + self.assertIn("10", node_ext_ids) + self.assertFalse(os.path.exists(points_path)) + self.assertTrue(all(wid in node_ids for wid in zone_w_ids)) + def test_to_undirected_on_simple_graph(self): g = nx.Graph() g.add_edge(1, 2) From 26e448cd32d98fd582393f8f34625b66403674bb Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 19 May 2026 23:59:29 +0530 Subject: [PATCH 2/2] Add coverage reporting and zone boundary repro fixture - Add coverage configuration - Add Codecov badge and coverage upload workflow - Add zone boundary XML fixture for _w_id validation repro --- .coveragerc | 5 + .github/workflows/unit_tests.yml | 10 +- README.md | 101 ++---------------- tests/unit_tests/test_files/zone_boundary.xml | 56 ++++------ 4 files changed, 39 insertions(+), 133 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..19b6e19 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,5 @@ +[run] +source = src +omit = + src/example.py + src/osm_osw_reformatter/version.py diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 53de4a9..9e4cb27 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -60,6 +60,14 @@ jobs: python -m coverage run --source=src/osm_osw_reformatter -m unittest discover -v tests/unit_tests >> $log_file 2>&1 echo -e "\nCoverage Report\n" >> $log_file coverage report >> $log_file + coverage xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + fail_ci_if_error: false - name: Check coverage run: | @@ -73,4 +81,4 @@ jobs: connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} container_name: 'osm-osw-reformatter-package' clean_destination_folder: false - delete_if_exists: false \ No newline at end of file + delete_if_exists: false diff --git a/README.md b/README.md index 9ca1c93..bd62adf 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # TDEI python lib formatter library + +[![codecov](https://codecov.io/gh/TaskarCenterAtUW/TDEI-python-lib-osw-formatter/branch/main/graph/badge.svg)](https://codecov.io/gh/TaskarCenterAtUW/TDEI-python-lib-osw-formatter) This python package designed to convert geospatial data from one format to another. In this case, it converts data from OpenStreetMap (OSM) format to OpenSideWalks (OSW) format and OpenSideWalks (OSW) format to OpenStreetMap (OSM) format. Let's break down the key components and processes involved in this converter: @@ -142,103 +144,12 @@ folder. - The terminal will show the output of coverage like this ```shell -> coverage run --source=src -m unittest discover -v tests/unit_tests -test_construct_geometries (helpers.test_osm.TestOSMHelper) ... ok -test_count_entities_with_nodes_counter (helpers.test_osm.TestOSMHelper) ... ok -test_count_entities_with_points_counter (helpers.test_osm.TestOSMHelper) ... ok -test_count_entities_with_ways_counter (helpers.test_osm.TestOSMHelper) ... ok -test_get_osm_graph (helpers.test_osm.TestOSMHelper) ... ok -test_osw_node_filter (helpers.test_osm.TestOSMHelper) ... ok -test_osw_point_filter (helpers.test_osm.TestOSMHelper) ... ok -test_osw_way_filter (helpers.test_osm.TestOSMHelper) ... ok -test_simplify_og (helpers.test_osm.TestOSMHelper) ... ok -test_cleanup_of_temp_files (helpers.test_osw.TestOSWHelper) ... ok -test_construct_geometries (helpers.test_osw.TestOSWHelper) ... ok -test_count_entities_with_nodes_counter (helpers.test_osw.TestOSWHelper) ... ok -test_count_entities_with_points_counter (helpers.test_osw.TestOSWHelper) ... ok -test_count_entities_with_ways_counter (helpers.test_osw.TestOSWHelper) ... ok -test_count_lines (helpers.test_osw.TestOSWHelper) ... ok -test_count_nodes (helpers.test_osw.TestOSWHelper) ... ok -test_count_points (helpers.test_osw.TestOSWHelper) ... ok -test_count_polygons (helpers.test_osw.TestOSWHelper) ... ok -test_count_ways (helpers.test_osw.TestOSWHelper) ... ok -test_count_zones (helpers.test_osw.TestOSWHelper) ... ok -test_get_osm_graph (helpers.test_osw.TestOSWHelper) ... ok -test_merge (helpers.test_osw.TestOSWHelper) ... ok -test_missing_files (helpers.test_osw.TestOSWHelper) ... ok -test_osw_node_filter (helpers.test_osw.TestOSWHelper) ... ok -test_osw_point_filter (helpers.test_osw.TestOSWHelper) ... ok -test_osw_way_filter (helpers.test_osw.TestOSWHelper) ... ok -test_simplify_og (helpers.test_osw.TestOSWHelper) ... ok -test_unzip (helpers.test_osw.TestOSWHelper) ... ok -test_unzip_should_return_3_files (helpers.test_osw.TestOSWHelper) ... ok -test_custom_values (helpers.test_response.TestResponseClass) ... ok -test_default_values (helpers.test_response.TestResponseClass) ... ok -test_error_none (helpers.test_response.TestResponseClass) ... ok -test_error_string (helpers.test_response.TestResponseClass) ... ok -test_generated_files_list (helpers.test_response.TestResponseClass) ... ok -test_generated_files_string (helpers.test_response.TestResponseClass) ... ok -test_cleanup_existing_files (test_formatter.TestFormatter) ... ok -test_cleanup_non_existent_files (test_formatter.TestFormatter) ... ok -test_osm2osw_error (test_formatter.TestFormatter) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Open failed for 'test.pbf': No such file or directory -ok -test_osm2osw_successful (test_formatter.TestFormatter) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Creating networks from region extracts... -Created OSW files! -ok -test_workdir_already_exists (test_formatter.TestFormatter) ... ok -test_workdir_creation (test_formatter.TestFormatter) ... ok -test_convert_error (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Open failed for 'test.pbf': No such file or directory -ok -test_convert_successful (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Creating networks from region extracts... -Created OSW files! -ok -test_generated_3_files (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Creating networks from region extracts... -Created OSW files! -ok -test_generated_files_are_string (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Creating networks from region extracts... -Created OSW files! -ok -test_generated_files_include_nodes_points_edges (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Creating networks from region extracts... -Created OSW files! -ok -test_convert_error (test_osw2osm.test_osw2osm.TestOSW2OSM) ... [Errno 2] No such file or directory: 'test.zip' -ok -test_convert_generated_files_are_string (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok -test_convert_successful (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok -test_generated_file (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok -test_generated_file_should_be_xml (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok -test_crossing_markings (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok -test_incline (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok -test_surface (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok -test_tactile_paving (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok -test_is_kerb (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok -test_is_kerb_invalid (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok -test_normalize_invalid_node (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok -test_normalize_kerb (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok -test_is_powerpole (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok -test_is_powerpole_invalid (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok -test_normalize_invalid_point (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok -test_normalize_powerpole (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok -test_is_crossing (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_footway (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_living_street (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_pedestrian (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_sidewalk (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_stairs (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_traffic_island (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_normalize_crossing (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_normalize_invalid_way (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_normalize_sidewalk (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok +> python -m coverage run --source=src -m unittest discover tests/unit_tests +................................. +.................................. ---------------------------------------------------------------------- -Ran 73 tests in 79.494s +Ran 225 tests in 44.601s OK ``` diff --git a/tests/unit_tests/test_files/zone_boundary.xml b/tests/unit_tests/test_files/zone_boundary.xml index 297335a..cf9f2a3 100644 --- a/tests/unit_tests/test_files/zone_boundary.xml +++ b/tests/unit_tests/test_files/zone_boundary.xml @@ -1,44 +1,26 @@ - - - - - - + + - - - - - + + + + + + - - - - - - - + + + + + - - - - - - - - - - - - - - - - - - - +