Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 30 additions & 8 deletions .github/workflows/unit_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,19 +60,41 @@ jobs:
python -m coverage run --source=src/osm_osw_reformatter -m unittest discover -v tests/unit_tests >> $log_file 2>&1
echo -e "\nCoverage Report\n" >> $log_file
coverage report >> $log_file
coverage xml

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
fail_ci_if_error: false

- name: Check coverage
run: |
coverage report --fail-under=85

- name: Generate coverage badge
run: |
pip install coverage-badge
mkdir -p badge-out
coverage-badge -f -o badge-out/coverage.svg

- name: Publish coverage badge to badges branch
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -e
cp badge-out/coverage.svg /tmp/coverage.svg
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git fetch origin badges || true
if git show-ref --verify --quiet refs/remotes/origin/badges; then
git checkout badges
else
git checkout --orphan badges
git rm -rf . >/dev/null 2>&1 || true
fi
cp /tmp/coverage.svg coverage.svg
git add coverage.svg
if ! git diff --cached --quiet; then
git commit -m "chore: update coverage badge"
git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:badges
else
echo "No badge changes to commit."
fi

- name: Upload report to Azure
uses: LanceMcCarthy/Action-AzureBlobUpload@v2
with:
Expand Down
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# Change log

### 0.3.6
- [BUG-3726](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/3726/) - Fix OSW→OSM conversion so duplicate source/generated node IDs do not corrupt way geometry. Way and relation references now preserve the exact `ogr2osm` node/way/relation objects during sequential ID remapping, preventing QGIS spider-line artifacts from ways snapping to unrelated duplicate node IDs.
- Add regression coverage for overlapping OSW node IDs and generated IDs, asserting converted OSM ways keep short local segments instead of cross-map jumps.
- Update README badges for PyPI package version, unit test workflow status, and coverage with short cache timing.

### 0.3.5
- [BUG-3665](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/3665) - Fix OSM→OSW export so zone boundary nodes are retained in `nodes.geojson` and zone `_w_id` references resolve to remapped sequential `_id`s, restoring OSW validation compliance for pedestrian-area geometries.
- Add regression coverage that converts a `highway=pedestrian` plaza fixture and asserts `python-osw-validation` reports zero issues.
Expand Down
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@

# TDEI python lib formatter library

[![codecov](https://codecov.io/gh/TaskarCenterAtUW/TDEI-python-lib-osw-formatter/branch/main/graph/badge.svg)](https://codecov.io/gh/TaskarCenterAtUW/TDEI-python-lib-osw-formatter)
[![osm-osw-reformatter](https://img.shields.io/pypi/v/osm-osw-reformatter?label=osm-osw-reformatter&cacheSeconds=60&t=1)](https://pypi.org/project/osm-osw-reformatter/)
[![Unit Tests](https://github.com/TaskarCenterAtUW/TDEI-python-lib-osw-formatter/actions/workflows/unit_tests.yml/badge.svg)](https://github.com/TaskarCenterAtUW/TDEI-python-lib-osw-formatter/actions/workflows/unit_tests.yml)
![Coverage](https://raw.githubusercontent.com/TaskarCenterAtUW/TDEI-python-lib-osw-formatter/badges/coverage.svg?cacheSeconds=60&t=1)

This python package designed to convert geospatial data from one format to another. In this case, it converts data from OpenStreetMap (OSM) format to OpenSideWalks (OSW) format and OpenSideWalks (OSW) format to OpenStreetMap (OSM) format. Let's break down the key components and processes involved in this converter:

Expand Down
52 changes: 42 additions & 10 deletions src/osm_osw_reformatter/serializer/osm/osm_normalizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,13 @@ def _as_int(val):
elif '_id' in osmgeometry.tags and osmgeometry.tags['_id'][0]:
osm_id = _as_int(osmgeometry.tags['_id'][0])

if osm_id is not None:
is_generated_node = (
hasattr(osmgeometry, "x")
and hasattr(osmgeometry, "y")
and not hasattr(osmgeometry, "nodes")
and not hasattr(osmgeometry, "members")
)
if osm_id is not None and not is_generated_node:
osmgeometry.id = osm_id
elevation = self._extract_elevation(ogrgeometry)
if elevation is not None:
Expand Down Expand Up @@ -231,8 +237,11 @@ def process_output(self, osmnodes, osmways, osmrelations):
references accordingly.
"""
# Capture original IDs for mapping
node_identity_map = {}
node_id_map = {}
way_identity_map = {}
way_id_map = {}
rel_identity_map = {}
rel_id_map = {}
next_node_id = 1
next_way_id = 1
Expand Down Expand Up @@ -279,7 +288,8 @@ def _member_type(member):
continue
new_id = next_node_id
next_node_id += 1
# Keep first mapping for refs that still point to source IDs.
node_identity_map[id(node)] = new_id
# Keep first mapping only as a fallback for raw integer refs.
node_id_map.setdefault(old_id, new_id)
node.id = new_id
_set_id_tag(node, new_id)
Expand All @@ -290,6 +300,7 @@ def _member_type(member):
if old_id is not None:
new_id = next_way_id
next_way_id += 1
way_identity_map[id(way)] = new_id
way_id_map.setdefault(old_id, new_id)
way.id = new_id
_set_id_tag(way, new_id)
Expand All @@ -305,11 +316,16 @@ def _member_type(member):
node_id_map[ref] = new_id
new_refs.append(node_id_map.get(ref, ref))
elif hasattr(ref, "id"):
if ref.id not in node_id_map:
ref_identity = id(ref)
if ref_identity in node_identity_map:
ref.id = node_identity_map[ref_identity]
elif ref.id not in node_id_map:
new_id = next_node_id
next_node_id += 1
node_id_map[ref.id] = new_id
ref.id = node_id_map.get(ref.id, ref.id)
ref.id = new_id
else:
ref.id = node_id_map.get(ref.id, ref.id)
new_refs.append(ref)
else:
new_refs.append(ref)
Expand All @@ -329,6 +345,7 @@ def _member_type(member):
if old_id is not None:
new_id = next_rel_id
next_rel_id += 1
rel_identity_map[id(rel)] = new_id
rel_id_map.setdefault(old_id, new_id)
rel.id = new_id
_set_id_tag(rel, new_id)
Expand Down Expand Up @@ -365,24 +382,39 @@ def _member_type(member):
rel_id_map[ref] = new_id
member.ref = node_id_map.get(ref, way_id_map.get(ref, rel_id_map.get(ref, ref)))
elif hasattr(ref, "id"):
ref_identity = id(ref)
if m_type == "node":
if ref.id not in node_id_map:
if ref_identity in node_identity_map:
ref.id = node_identity_map[ref_identity]
elif ref.id not in node_id_map:
new_id = next_node_id
next_node_id += 1
node_id_map[ref.id] = new_id
ref.id = node_id_map.get(ref.id, ref.id)
ref.id = new_id
else:
ref.id = node_id_map.get(ref.id, ref.id)
elif m_type == "way":
if ref.id not in way_id_map:
ref_identity = id(ref)
if ref_identity in way_identity_map:
ref.id = way_identity_map[ref_identity]
elif ref.id not in way_id_map:
new_id = next_way_id
next_way_id += 1
way_id_map[ref.id] = new_id
ref.id = way_id_map.get(ref.id, ref.id)
ref.id = new_id
else:
ref.id = way_id_map.get(ref.id, ref.id)
elif m_type == "relation":
if ref.id not in rel_id_map:
ref_identity = id(ref)
if ref_identity in rel_identity_map:
ref.id = rel_identity_map[ref_identity]
elif ref.id not in rel_id_map:
new_id = next_rel_id
next_rel_id += 1
rel_id_map[ref.id] = new_id
ref.id = rel_id_map.get(ref.id, ref.id)
ref.id = new_id
else:
ref.id = rel_id_map.get(ref.id, ref.id)
else:
if ref.id not in node_id_map and ref.id not in way_id_map and ref.id not in rel_id_map:
new_id = next_rel_id
Expand Down
2 changes: 1 addition & 1 deletion src/osm_osw_reformatter/version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '0.3.5'
__version__ = '0.3.6'
78 changes: 78 additions & 0 deletions tests/unit_tests/test_files/bug_3726/edges.geojson
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {
"type": "LineString",
"coordinates": [
[
-117.1589921,
32.7043326
],
[
-117.1590819,
32.7043175
],
[
-117.1591807,
32.7042797
],
[
-117.1592436,
32.7042419
],
[
-117.1593784,
32.7042041
],
[
-117.1595581,
32.7041663
],
[
-117.1596749,
32.7041512
],
[
-117.1597826,
32.7041512
],
[
-117.1599443,
32.7041587
],
[
-117.1601958,
32.7041814
],
[
-117.160636,
32.7044762
],
[
-117.1607708,
32.7045821
],
[
-117.1608786,
32.7046501
],
[
-117.1609504,
32.7047181
]
]
},
"properties": {
"_id": "359",
"_u_id": "612",
"_v_id": "613",
"highway": "footway",
"footway": "sidewalk",
"surface": "concrete",
"surface:score": 0.94
}
}
]
}
44 changes: 44 additions & 0 deletions tests/unit_tests/test_files/bug_3726/nodes.geojson
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [
-117.1589921,
32.7043326
]
},
"properties": {
"_id": "612"
}
},
{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [
-117.1609504,
32.7047181
]
},
"properties": {
"_id": "613"
}
},
{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [
-117.1631527,
32.7320171
]
},
"properties": {
"_id": "1"
}
}
]
}
39 changes: 39 additions & 0 deletions tests/unit_tests/test_osw2osm/test_osw2osm.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
import math
import os
from pathlib import Path
import tempfile
Expand All @@ -14,6 +15,8 @@
TEST_DATA_WITH_INCLINE_ZIP_FILE = os.path.join(ROOT_DIR, 'test_files/dataset_with_incline.zip')
TEST_EDGES_WITH_INVALID_INCLINE_FILE = os.path.join(ROOT_DIR, 'test_files/edges_invalid_incline.geojson')
TEST_NODES_WITH_INVALID_INCLINE_FILE = os.path.join(ROOT_DIR, 'test_files/nodes_invalid_incline.geojson')
TEST_BUG_3726_NODES_FILE = os.path.join(ROOT_DIR, 'test_files/bug_3726/nodes.geojson')
TEST_BUG_3726_EDGES_FILE = os.path.join(ROOT_DIR, 'test_files/bug_3726/edges.geojson')


def _create_invalid_incline_zip(zip_path: str) -> str:
Expand Down Expand Up @@ -91,6 +94,15 @@ def _create_3d_node_zip(zip_path: str, z_value: float) -> str:


class TestOSW2OSM(unittest.IsolatedAsyncioTestCase):
@staticmethod
def _distance_meters(first, second):
lon1, lat1 = first
lon2, lat2 = second
return math.hypot(
(lon1 - lon2) * 111320 * math.cos(math.radians((lat1 + lat2) / 2)),
(lat1 - lat2) * 110540,
)

def test_convert_successful(self):
zip_file = TEST_ZIP_FILE
osw2osm = OSW2OSM(zip_file_path=zip_file, workdir=OUTPUT_DIR, prefix='test')
Expand Down Expand Up @@ -366,6 +378,33 @@ def test_remap_ids_rewrites_refs(self):
rel_tag_ids = [tag.get("v") for tag in root.findall(".//relation/tag[@k='_id']")]
self.assertEqual(rel_tag_ids, ["1"])

def test_convert_preserves_way_geometry_when_osw_node_ids_overlap_generated_ids(self):
with tempfile.TemporaryDirectory() as tmpdir:
zip_path = Path(tmpdir, "overlapping_ids.zip")
with zipfile.ZipFile(zip_path, "w") as zf:
zf.write(TEST_BUG_3726_NODES_FILE, arcname="nodes.geojson")
zf.write(TEST_BUG_3726_EDGES_FILE, arcname="edges.geojson")

result = OSW2OSM(zip_file_path=str(zip_path), workdir=tmpdir, prefix="overlapping").convert()
self.assertTrue(result.status, msg=getattr(result, "error", "Conversion failed"))

root = ET.parse(result.generated_files).getroot()
nodes = {
node.get("id"): (float(node.get("lon")), float(node.get("lat")))
for node in root.findall(".//node")
}
node_ids = [node.get("id") for node in root.findall(".//node")]
self.assertEqual(len(node_ids), len(set(node_ids)))

for way in root.findall(".//way"):
refs = [nd.get("ref") for nd in way.findall("nd")]
distances = [
self._distance_meters(nodes[start_ref], nodes[end_ref])
for start_ref, end_ref in zip(refs, refs[1:])
]
self.assertTrue(distances)
self.assertLess(max(distances), 200)


if __name__ == '__main__':
unittest.main()
Loading
Loading