Upgrade osm-osw-reformatter to 0.3.6

.github/workflows/unit_tests.yaml

@@ -76,6 +76,36 @@ jobs:
         run: |
           coverage report --fail-under=85
 
+      - name: Generate coverage badge
+        run: |
+          pip install coverage-badge
+          mkdir -p badge-out
+          coverage-badge -f -o badge-out/coverage.svg
+
+      - name: Publish coverage badge to badges branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -e
+          cp badge-out/coverage.svg /tmp/coverage.svg
+          git config user.name  "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git fetch origin badges || true
+          if git show-ref --verify --quiet refs/remotes/origin/badges; then
+            git checkout badges
+          else
+            git checkout --orphan badges
+            git rm -rf . >/dev/null 2>&1 || true
+          fi
+          cp /tmp/coverage.svg coverage.svg
+          git add coverage.svg
+          if ! git diff --cached --quiet; then
+            git commit -m "chore: update coverage badge"
+            git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:badges
+          else
+            echo "No badge changes to commit."
+          fi
+
       - name: Upload report to Azure
         uses: LanceMcCarthy/Action-AzureBlobUpload@v2
         with:

README.md

@@ -1,5 +1,10 @@
 # TDEI-python-osw-formatter
 
+[![Unit Tests](https://github.com/TaskarCenterAtUW/TDEI-python-osw-formatter/actions/workflows/unit_tests.yaml/badge.svg)](https://github.com/TaskarCenterAtUW/TDEI-python-osw-formatter/actions/workflows/unit_tests.yaml)
+[![Coverage](https://raw.githubusercontent.com/TaskarCenterAtUW/TDEI-python-osw-formatter/badges/coverage.svg)](https://github.com/TaskarCenterAtUW/TDEI-python-osw-formatter/tree/badges)
+[![osm-osw-reformatter](https://img.shields.io/badge/dynamic/regex?url=https%3A%2F%2Fraw.githubusercontent.com%2FTaskarCenterAtUW%2FTDEI-python-osw-formatter%2Fdev%2Frequirements.txt&search=%28%3Fm%29%5Eosm-osw-reformatter%3D%3D%28%5B%5E%5Cr%5Cn%5D%2B%29&replace=%241&label=osm-osw-reformatter&color=blue&cacheSeconds=60)](https://pypi.org/project/osm-osw-reformatter/)
+[![python-ms-core](https://img.shields.io/badge/dynamic/regex?url=https%3A%2F%2Fraw.githubusercontent.com%2FTaskarCenterAtUW%2FTDEI-python-osw-formatter%2Fdev%2Frequirements.txt&search=%28%3Fm%29%5Epython-ms-core%3D%3D%28%5B%5E%5Cr%5Cn%5D%2B%29&replace=%241&label=python-ms-core&color=blue&cacheSeconds=60)](https://pypi.org/project/python-ms-core/)
+
 ## Introduction 
 Service to Convert the OSW files to OSM files and OSM to OSW files. At the moment, the service does the following:
 - Listens to the topic which is mentioned in `.env` file for any new message (that is triggered when a file is uploaded), example  `UPLOAD_TOPIC=osw-validation` 
@@ -216,4 +221,3 @@ The format is mentioned in [osw-upload.json](./src/assets/osw-upload.json)
 #### Outgoing
 The outgoing messages will be to the `osw-validation` topic.
 The format of the message is at [osw-format.json](./src/assets/osw-format.json)
-

requirements.txt

@@ -1,8 +1,8 @@
 fastapi==0.88.0
 pydantic==1.10.4
-python-ms-core==0.0.25
+python-ms-core==0.0.26
 uvicorn==0.20.0
 html_testRunner==1.2.1
-osm-osw-reformatter==0.3.5
+osm-osw-reformatter==0.3.6
 numpy==1.26.4
 pyproj~=3.6.1

src/config.py

@@ -17,13 +17,13 @@ class EventBusSettings:
 class Settings(BaseSettings):
     app_name: str = 'python-osw-formatter'
     event_bus = EventBusSettings()
-    max_concurrent_messages: int = os.environ.get('MAX_CONCURRENT_MESSAGES', 1)
+    max_concurrent_messages: int = int(os.environ.get('MAX_CONCURRENT_MESSAGES', 1))
     # Single-run worker should consume one message and then shut down.
-    max_receivable_messages: int = os.environ.get('MAX_RECEIVABLE_MESSAGES', 1)
+    max_receivable_messages: int = int(os.environ.get('MAX_RECEIVABLE_MESSAGES', 1))
     # Wait for queue message completion/abandon settlement before terminating process.
-    message_settle_wait_seconds: float = os.environ.get('MESSAGE_SETTLE_WAIT_SECONDS', 10.0)
+    message_settle_wait_seconds: float = float(os.environ.get('MESSAGE_SETTLE_WAIT_SECONDS', 10.0))
     # Delay gives queue client time to settle/complete the in-flight message before exit.
-    shutdown_delay_seconds: float = os.environ.get('SHUTDOWN_DELAY_SECONDS', 2.0)
+    shutdown_delay_seconds: float = float(os.environ.get('SHUTDOWN_DELAY_SECONDS', 2.0))
 
     def get_root_directory(self) -> str:
         return os.path.dirname(os.path.abspath(__file__))

src/service/osw_formatter_service.py

@@ -29,6 +29,11 @@ class OSWFomatterService:
     _settings = Settings()
 
     def __init__(self):
+        # Keep Service Bus receiver and lock renewal in the parent process while
+        # long-running formatting work runs in a Linux forked child process.
+        os.environ["TOPIC_CALLBACK_EXECUTION_MODE"] = "process"
+        os.environ["TOPIC_CALLBACK_PROCESS_START_METHOD"] = "fork"
+        os.environ["TOPIC_CALLBACK_PROCESS_FALLBACK_MODE"] = "error"
         self.core = Core()
         listening_topic_name = self._settings.event_bus.validation_topic or ""
         self.subscription_name = self._settings.event_bus.validation_subscription or ""
@@ -88,8 +93,9 @@ def process(message: QueueMessage) -> None:
             subscription=self.subscription_name, callback=process,
             max_receivable_messages=self._settings.max_receivable_messages
         )
-        logger.info('Listener finished processing available messages; stopping server/container.')
-        self._stop_server_and_container(delay_seconds=self._settings.shutdown_delay_seconds)
+        if self._settings.max_receivable_messages > 0:
+            logger.info('Listener finished processing available messages; stopping server/container.')
+            self._stop_server_and_container(delay_seconds=self._settings.shutdown_delay_seconds)
 
     def format(self, received_message: OSWValidationMessage):
         tdei_record_id: str = ""
@@ -288,7 +294,7 @@ def upload_to_azure_on_demand(self, remote_path: str, local_url: str):
         container = self.storage_client.get_container(
             container_name=self.container_name
         )
-        file = container.create_file(remote_path)
+        file = container.create_file(name=remote_path)
         with open(local_url, "rb") as data:
             file.upload(data)
         return file.get_remote_url()
@@ -328,4 +334,4 @@ def _terminate():
                 logger.info('Forcing process exit to stop server/container.')
                 os._exit(0)
 
-        threading.Thread(target=_terminate, daemon=True).start()
+        threading.Thread(target=_terminate, daemon=True).start()

tests/unit_tests/service/test_osw_formatter_service.py

@@ -76,6 +76,19 @@ def test_start_listening_stops_container_after_subscribe_returns(self, mock_stop
             delay_seconds=self.formatter._settings.shutdown_delay_seconds
         )
 
+    @patch.object(OSWFomatterService, '_stop_server_and_container')
+    def test_start_listening_does_not_stop_container_for_unlimited_receivable_messages(self, mock_stop_server_and_container):
+        self.formatter._settings.max_receivable_messages = -1
+
+        self.formatter.start_listening()
+
+        self.formatter.listening_topic.subscribe.assert_called_once_with(
+            subscription=self.formatter.subscription_name,
+            callback=ANY,
+            max_receivable_messages=self.formatter._settings.max_receivable_messages,
+        )
+        mock_stop_server_and_container.assert_not_called()
+
     @patch('src.service.osw_formatter_service.OSWFormat')
     @patch.object(OSWFormat, 'download_single_file')
     @patch.object(OSWFomatterService, 'send_status')

tests/unit_tests/service/test_service.py

@@ -405,7 +405,7 @@ def test_upload_to_azure_on_demand(self, mock_service, mock_open_file):
 
         # Assert
         mock_service.storage_client.get_container.assert_called_once_with(container_name="mock_container")
-        mock_container.create_file.assert_called_once_with(remote_path)
+        mock_container.create_file.assert_called_once_with(name=remote_path)
         mock_file.upload.assert_called_once()
         mock_open_file.assert_called_once_with(local_url, "rb")
         self.assertEqual(result, "https://example.com/mock_remote_url")