Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,9 @@ shipped by Vector (`vector.toml`) to Axiom, where they feed dashboards.
outage. Beat has to run with `--scheduler core.beat:QueueDepthScheduler`.
- Refresh failures: `fio_refresh_failed`, `planet_refresh_failed`,
`planet_infrastructure_refresh_failed`, `cxpc_refresh_failed`,
`exchanges_refresh_failed`. A FIO error status is a warning with
`status_code`, anything else `logger.exception`.
`exchanges_refresh_failed`. A FIO error status (with `status_code`) or
transport error (timeout, dropped connection) is a warning, anything else
`logger.exception`.
- Levels: `info` for a business event, `warning` for an expected failure
(bad user FIO key, invalid webhook payload), `logger.exception` inside
`except` for what needs a look. Log a failure once, where it's handled.
Expand Down
6 changes: 6 additions & 0 deletions backend/gamedata/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ def gamedata_refresh_planet() -> bool:
planet_natural_id=to_update.planet_natural_id,
status_code=exc.response.status_code,
)
elif isinstance(exc, httpx.TransportError):
# FIO slow or unreachable (timeout, dropped connection): expected, fio_request_failed has the error
logger.warning('planet_refresh_failed', planet_natural_id=to_update.planet_natural_id)
else:
logger.exception('planet_refresh_failed', planet_natural_id=to_update.planet_natural_id)
to_update.update_refresh_result(error=exc)
Expand Down Expand Up @@ -269,6 +272,9 @@ def gamedata_refresh_user_fiodata(user_id: int, *_legacy_args: str) -> bool:
# retrying won't fix a rejected key: fail now; new credentials reset the row (user/signals.py)
to_update.fio_status_code = 401
to_update.automation_error_count = GameFIOPlayerData.MAX_RETRIES - 1
elif isinstance(exc, httpx.TransportError):
# FIO slow or unreachable: expected, fio_request_failed has the error
log.warning('fio_refresh_failed')
else:
log.exception('fio_refresh_failed')
to_update.update_refresh_result(error=exc, commit=False)
Expand Down
8 changes: 7 additions & 1 deletion backend/tests/gamedata/test_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ def test_refresh_planet(self, scenario):
'error, level',
[
(httpx.HTTPStatusError('503', request=MagicMock(), response=MagicMock(status_code=503)), 'WARNING'),
(httpx.ReadTimeout('The read operation timed out'), 'WARNING'),
(ValueError('bug'), 'ERROR'),
],
)
Expand All @@ -83,8 +84,12 @@ def test_refresh_planet_logs_its_failure(
if isinstance(r.msg, dict) and r.msg['event'] == 'planet_refresh_failed'
]
assert [(lvl, msg['planet_natural_id']) for lvl, msg in failed] == [(level, 'M')]
# a traceback only for what is not an HTTP error status
# a traceback only for what is not an HTTP error status or transport failure
assert ('exception' in failed[0][1]) is (level == 'ERROR')
# the failure is recorded for a retry either way
planet = GamePlanet.objects.get(planet_natural_id='M')
assert (planet.automation_refresh_status, planet.automation_error_count) == ('retrying', 1)
assert planet.automation_next_retry_at is not None

@pytest.mark.parametrize('scenario', ['missing', 'fio_fail', 'success'])
@patch('gamedata.tasks.get_fio_service')
Expand All @@ -100,6 +105,7 @@ def test_refresh_user_fiodata(self, mock_get_fio, scenario):
'error, level',
[
(httpx.HTTPStatusError('401', request=MagicMock(), response=MagicMock(status_code=401)), 'WARNING'),
(httpx.ReadTimeout('The read operation timed out'), 'WARNING'),
(ValueError('bug'), 'ERROR'),
],
)
Expand Down
Loading