diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 67486f1..eb5af60 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -76,6 +76,36 @@ jobs: run: | coverage report --fail-under=85 + - name: Generate coverage badge + run: | + pip install coverage-badge + mkdir -p badge-out + coverage-badge -f -o badge-out/coverage.svg + + - name: Publish coverage badge to badges branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -e + cp badge-out/coverage.svg /tmp/coverage.svg + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git fetch origin badges || true + if git show-ref --verify --quiet refs/remotes/origin/badges; then + git checkout badges + else + git checkout --orphan badges + git rm -rf . >/dev/null 2>&1 || true + fi + cp /tmp/coverage.svg coverage.svg + git add coverage.svg + if ! git diff --cached --quiet; then + git commit -m "chore: update coverage badge" + git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:badges + else + echo "No badge changes to commit." + fi + - name: Upload report to Azure uses: LanceMcCarthy/Action-AzureBlobUpload@v2 with: diff --git a/README.md b/README.md index 17e1dcb..0425214 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # TDEI-python-osw-formatter +[![Unit Tests](https://github.com/TaskarCenterAtUW/TDEI-python-osw-formatter/actions/workflows/unit_tests.yaml/badge.svg)](https://github.com/TaskarCenterAtUW/TDEI-python-osw-formatter/actions/workflows/unit_tests.yaml) +[![Coverage](https://raw.githubusercontent.com/TaskarCenterAtUW/TDEI-python-osw-formatter/badges/coverage.svg)](https://github.com/TaskarCenterAtUW/TDEI-python-osw-formatter/tree/badges) +[![osm-osw-reformatter](https://img.shields.io/badge/dynamic/regex?url=https%3A%2F%2Fraw.githubusercontent.com%2FTaskarCenterAtUW%2FTDEI-python-osw-formatter%2Fdev%2Frequirements.txt&search=%28%3Fm%29%5Eosm-osw-reformatter%3D%3D%28%5B%5E%5Cr%5Cn%5D%2B%29&replace=%241&label=osm-osw-reformatter&color=blue&cacheSeconds=60)](https://pypi.org/project/osm-osw-reformatter/) +[![python-ms-core](https://img.shields.io/badge/dynamic/regex?url=https%3A%2F%2Fraw.githubusercontent.com%2FTaskarCenterAtUW%2FTDEI-python-osw-formatter%2Fdev%2Frequirements.txt&search=%28%3Fm%29%5Epython-ms-core%3D%3D%28%5B%5E%5Cr%5Cn%5D%2B%29&replace=%241&label=python-ms-core&color=blue&cacheSeconds=60)](https://pypi.org/project/python-ms-core/) + ## Introduction Service to Convert the OSW files to OSM files and OSM to OSW files. At the moment, the service does the following: - Listens to the topic which is mentioned in `.env` file for any new message (that is triggered when a file is uploaded), example `UPLOAD_TOPIC=osw-validation` @@ -216,4 +221,3 @@ The format is mentioned in [osw-upload.json](./src/assets/osw-upload.json) #### Outgoing The outgoing messages will be to the `osw-validation` topic. The format of the message is at [osw-format.json](./src/assets/osw-format.json) - diff --git a/requirements.txt b/requirements.txt index c87461f..4c240f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ fastapi==0.88.0 pydantic==1.10.4 -python-ms-core==0.0.25 +python-ms-core==0.0.26 uvicorn==0.20.0 html_testRunner==1.2.1 -osm-osw-reformatter==0.3.4 +osm-osw-reformatter==0.3.6 numpy==1.26.4 pyproj~=3.6.1 \ No newline at end of file diff --git a/src/config.py b/src/config.py index 389db87..39caeec 100644 --- a/src/config.py +++ b/src/config.py @@ -17,13 +17,13 @@ class EventBusSettings: class Settings(BaseSettings): app_name: str = 'python-osw-formatter' event_bus = EventBusSettings() - max_concurrent_messages: int = os.environ.get('MAX_CONCURRENT_MESSAGES', 1) + max_concurrent_messages: int = int(os.environ.get('MAX_CONCURRENT_MESSAGES', 1)) # Single-run worker should consume one message and then shut down. - max_receivable_messages: int = os.environ.get('MAX_RECEIVABLE_MESSAGES', 1) + max_receivable_messages: int = int(os.environ.get('MAX_RECEIVABLE_MESSAGES', 1)) # Wait for queue message completion/abandon settlement before terminating process. - message_settle_wait_seconds: float = os.environ.get('MESSAGE_SETTLE_WAIT_SECONDS', 10.0) + message_settle_wait_seconds: float = float(os.environ.get('MESSAGE_SETTLE_WAIT_SECONDS', 10.0)) # Delay gives queue client time to settle/complete the in-flight message before exit. - shutdown_delay_seconds: float = os.environ.get('SHUTDOWN_DELAY_SECONDS', 2.0) + shutdown_delay_seconds: float = float(os.environ.get('SHUTDOWN_DELAY_SECONDS', 2.0)) def get_root_directory(self) -> str: return os.path.dirname(os.path.abspath(__file__)) diff --git a/src/service/osw_formatter_service.py b/src/service/osw_formatter_service.py index 564b3cc..b3d555d 100644 --- a/src/service/osw_formatter_service.py +++ b/src/service/osw_formatter_service.py @@ -29,6 +29,11 @@ class OSWFomatterService: _settings = Settings() def __init__(self): + # Keep Service Bus receiver and lock renewal in the parent process while + # long-running formatting work runs in a Linux forked child process. + os.environ["TOPIC_CALLBACK_EXECUTION_MODE"] = "process" + os.environ["TOPIC_CALLBACK_PROCESS_START_METHOD"] = "fork" + os.environ["TOPIC_CALLBACK_PROCESS_FALLBACK_MODE"] = "error" self.core = Core() listening_topic_name = self._settings.event_bus.validation_topic or "" self.subscription_name = self._settings.event_bus.validation_subscription or "" @@ -88,8 +93,9 @@ def process(message: QueueMessage) -> None: subscription=self.subscription_name, callback=process, max_receivable_messages=self._settings.max_receivable_messages ) - logger.info('Listener finished processing available messages; stopping server/container.') - self._stop_server_and_container(delay_seconds=self._settings.shutdown_delay_seconds) + if self._settings.max_receivable_messages > 0: + logger.info('Listener finished processing available messages; stopping server/container.') + self._stop_server_and_container(delay_seconds=self._settings.shutdown_delay_seconds) def format(self, received_message: OSWValidationMessage): tdei_record_id: str = "" @@ -288,7 +294,7 @@ def upload_to_azure_on_demand(self, remote_path: str, local_url: str): container = self.storage_client.get_container( container_name=self.container_name ) - file = container.create_file(remote_path) + file = container.create_file(name=remote_path) with open(local_url, "rb") as data: file.upload(data) return file.get_remote_url() @@ -328,4 +334,4 @@ def _terminate(): logger.info('Forcing process exit to stop server/container.') os._exit(0) - threading.Thread(target=_terminate, daemon=True).start() \ No newline at end of file + threading.Thread(target=_terminate, daemon=True).start() diff --git a/tests/unit_tests/service/test_osw_formatter_service.py b/tests/unit_tests/service/test_osw_formatter_service.py index 163750f..fa45331 100644 --- a/tests/unit_tests/service/test_osw_formatter_service.py +++ b/tests/unit_tests/service/test_osw_formatter_service.py @@ -76,6 +76,19 @@ def test_start_listening_stops_container_after_subscribe_returns(self, mock_stop delay_seconds=self.formatter._settings.shutdown_delay_seconds ) + @patch.object(OSWFomatterService, '_stop_server_and_container') + def test_start_listening_does_not_stop_container_for_unlimited_receivable_messages(self, mock_stop_server_and_container): + self.formatter._settings.max_receivable_messages = -1 + + self.formatter.start_listening() + + self.formatter.listening_topic.subscribe.assert_called_once_with( + subscription=self.formatter.subscription_name, + callback=ANY, + max_receivable_messages=self.formatter._settings.max_receivable_messages, + ) + mock_stop_server_and_container.assert_not_called() + @patch('src.service.osw_formatter_service.OSWFormat') @patch.object(OSWFormat, 'download_single_file') @patch.object(OSWFomatterService, 'send_status') diff --git a/tests/unit_tests/service/test_service.py b/tests/unit_tests/service/test_service.py index addc118..01f52bf 100644 --- a/tests/unit_tests/service/test_service.py +++ b/tests/unit_tests/service/test_service.py @@ -405,7 +405,7 @@ def test_upload_to_azure_on_demand(self, mock_service, mock_open_file): # Assert mock_service.storage_client.get_container.assert_called_once_with(container_name="mock_container") - mock_container.create_file.assert_called_once_with(remote_path) + mock_container.create_file.assert_called_once_with(name=remote_path) mock_file.upload.assert_called_once() mock_open_file.assert_called_once_with(local_url, "rb") self.assertEqual(result, "https://example.com/mock_remote_url")