From b902fc1840b00df87e7bab5cf9dbfa609bd638e6 Mon Sep 17 00:00:00 2001 From: Joe Farrelly Date: Wed, 1 Jul 2026 00:10:00 +0100 Subject: [PATCH] Parse addon gold/played-time into a DB table on upload Replaces the page=addon read-through cache (which re-parsed the raw .lua file on every cache miss, with no expiry) with parse-on-upload into a new ProfileAltAddonData model, matching how every other Profile* model is populated. Adds the altaddondata endpoint and drops the now-dead generic file-read branch from ProfileUserView.list. --- CLAUDE.md | 6 +- .../migrations/0016_profilealtaddondata.py | 26 +++++++ apicore/models.py | 16 +++++ apicore/serializers.py | 9 +++ apicore/views.py | 71 +++++++++++-------- backend/urls.py | 1 + tests/test_views.py | 60 +++++++++++++--- 7 files changed, 150 insertions(+), 39 deletions(-) create mode 100644 apicore/migrations/0016_profilealtaddondata.py diff --git a/CLAUDE.md b/CLAUDE.md index ccd2cfc..10f5586 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -100,6 +100,7 @@ conftest.py pytest env-var setup (pytest_configure hook) - `altreputations` — `ProfileAltReputation`: Faction standing per alt - `altmythicplus` — `ProfileAltMythicPlus`: Current-season M+ rating summary per alt - `altmythicplusdungeons` — `ProfileAltMythicPlusDungeon`: Best run per dungeon per alt +- `altaddondata` — `ProfileAltAddonData`: Gold + played time per alt, parsed from the uploaded `.lua` file (addon-only, no Blizzard API equivalent) ### Data endpoints (static WoW data, synced via DataScan task) - `professions`, `professiontiers`, `professionrecipes`, `reagents`, `recipereagents` @@ -150,14 +151,13 @@ Dispatches six independent subtasks: `scanProfessionData`, `scanMountData`, `sca - `fullDataScan` — weekly, Sunday 03:00 UTC, with `BLIZZ_CLIENT`/`BLIZZ_SECRET` baked into the schedule args at startup. The admin-triggered `/api/custom/datascan/` endpoints still work for ad-hoc/manual scans (e.g. testing a single category). ### Lua addon file -`ProfileUser.perform_update` validates and stores a `FazzToolsScraper.lua` addon export. -`ProfileUserView.list` with `?page=header` returns the last-update timestamp; the file is parsed via `LuaParser` and cached on read (`userfile:{user_id}`), ready for future addon-only data (gold, currencies, lockouts) to consume — no page handler reads it yet. +`ProfileUser.perform_update` validates and stores a `FazzToolsScraper.lua` addon export, then parses it via `LuaParser` and upserts `ProfileAltAddonData` for each alt found in the file (matched by `f"{alt.alt_name}-{alt.alt_realm}"`, the display name + display realm key `core.lua` writes). This is parse-on-upload, not parse-on-read — there's no Celery sync task for this data since it only ever exists in the addon export, never the Blizzard API, so the upload itself is the sync point. A failed parse logs a warning and leaves the stored file/timestamp update intact rather than failing the upload. Currencies/lockouts/keystone/vault are captured by the addon but not yet parsed into a model — same pattern, not built. ## Database tables (all prefixed `ft_`) **Data (static):** `ft_data_profession`, `ft_data_professiontier`, `ft_data_professionrecipe`, `ft_data_reagent`, `ft_data_recipereagent`, `ft_data_equipment`, `ft_data_equipmentvariant`, `ft_data_mount`, `ft_data_pet`, `ft_data_achievement`, `ft_data_faction`, `ft_data_mythicdungeon` -**Profile (user):** `ft_profile_user`, `ft_profile_alt`, `ft_profile_altprofession`, `ft_profile_altprofessiondata`, `ft_profile_altequipment`, `ft_profile_usermount`, `ft_profile_userpet`, `ft_profile_altachievement`, `ft_profile_altreputation`, `ft_profile_altmythicplus`, `ft_profile_altmythicplusdungeon` +**Profile (user):** `ft_profile_user`, `ft_profile_alt`, `ft_profile_altprofession`, `ft_profile_altprofessiondata`, `ft_profile_altequipment`, `ft_profile_usermount`, `ft_profile_userpet`, `ft_profile_altachievement`, `ft_profile_altreputation`, `ft_profile_altmythicplus`, `ft_profile_altmythicplusdungeon`, `ft_profile_altaddondata` ## Things to know diff --git a/apicore/migrations/0016_profilealtaddondata.py b/apicore/migrations/0016_profilealtaddondata.py new file mode 100644 index 0000000..c208aa1 --- /dev/null +++ b/apicore/migrations/0016_profilealtaddondata.py @@ -0,0 +1,26 @@ +# Generated by Django 5.2 on 2026-06-30 23:06 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('apicore', '0015_datamythicdungeon_profilealtmythicplus_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='ProfileAltAddonData', + fields=[ + ('alt', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='apicore.profilealt')), + ('gold', models.PositiveBigIntegerField(default=0)), + ('played_time_total', models.PositiveIntegerField(default=0)), + ('played_time_level', models.PositiveIntegerField(default=0)), + ], + options={ + 'db_table': 'ft_profile_altaddondata', + }, + ), + ] diff --git a/apicore/models.py b/apicore/models.py index 573262a..ae47491 100644 --- a/apicore/models.py +++ b/apicore/models.py @@ -412,3 +412,19 @@ class Meta: def __str__(self): return f"{self.alt} - {self.dungeon}" + + +class ProfileAltAddonData(models.Model): + """Per-alt data that only exists in the uploaded addon export (no Blizzard API + equivalent), parsed and upserted from the .lua file on upload.""" + + alt = models.OneToOneField(ProfileAlt, on_delete=models.CASCADE, primary_key=True) + gold = models.PositiveBigIntegerField(default=0) + played_time_total = models.PositiveIntegerField(default=0) + played_time_level = models.PositiveIntegerField(default=0) + + class Meta: + db_table = "ft_profile_altaddondata" + + def __str__(self): + return f"{self.alt.alt_name} - {self.alt.alt_realm}" diff --git a/apicore/serializers.py b/apicore/serializers.py index 4cb0c27..59f8054 100644 --- a/apicore/serializers.py +++ b/apicore/serializers.py @@ -15,6 +15,7 @@ DataRecipeReagent, ProfileAlt, ProfileAltAchievement, + ProfileAltAddonData, ProfileAltEquipment, ProfileAltMythicPlus, ProfileAltMythicPlusDungeon, @@ -288,3 +289,11 @@ class Meta: "completed_timestamp", "is_completed_within_time", ) + + +class ProfileAltAddonDataSerializer(serializers.ModelSerializer): + alt_id = serializers.ReadOnlyField(source="alt.alt_id") + + class Meta: + model = ProfileAltAddonData + fields = ("alt_id", "gold", "played_time_total", "played_time_level") diff --git a/apicore/views.py b/apicore/views.py index 47f8da8..6729bc1 100644 --- a/apicore/views.py +++ b/apicore/views.py @@ -8,7 +8,6 @@ import environ import requests -from django.core.cache import cache from django.core.files.base import ContentFile from django.db import models from django.utils import timezone @@ -36,6 +35,7 @@ DataRecipeReagent, ProfileAlt, ProfileAltAchievement, + ProfileAltAddonData, ProfileAltEquipment, ProfileAltMythicPlus, ProfileAltMythicPlusDungeon, @@ -61,6 +61,7 @@ DataReagentSerializer, DataRecipeReagentSerializer, ProfileAltAchievementSerializer, + ProfileAltAddonDataSerializer, ProfileAltEquipmentSerializer, ProfileAltMythicPlusDungeonSerializer, ProfileAltMythicPlusSerializer, @@ -195,7 +196,29 @@ def perform_update(self, serializer): logger.warning("Could not remove old file: %s", exc) serializer.save(user_id=user_id, user_file=normalised_file, user_last_update=update_date) - cache.delete(f"userfile:{user_id}") + self._sync_addon_data(user_id, normalised) + + @staticmethod + def _sync_addon_data(user_id, normalised_content): + try: + parsed = LuaParser(normalised_content.splitlines(keepends=True)).parse() + except (ValueError, IndexError) as exc: + logger.warning("Could not parse addon file for %s: %s", user_id, exc) + return + + addon_alts = parsed.get("alts", {}) + for alt in ProfileAlt.objects.filter(user=user_id): + addon_alt = addon_alts.get(f"{alt.alt_name}-{alt.alt_realm}") + if addon_alt is None: + continue + ProfileAltAddonData.objects.update_or_create( + alt=alt, + defaults={ + "gold": addon_alt.get("gold") or 0, + "played_time_total": addon_alt.get("playedTimeTotal") or 0, + "played_time_level": addon_alt.get("playedTimeLevel") or 0, + }, + ) def list(self, request): user_id = request.query_params.get("user") @@ -213,32 +236,6 @@ def list(self, request): ts = time.mktime(user_obj.user_last_update.timetuple()) * 1000 return response.Response([ts]) - if not user_obj.user_file: - return response.Response([]) - - cache_key = f"userfile:{user_id}" - data = cache.get(cache_key) - if data is None: - with user_obj.user_file.open("rb") as f: - lines = [line.decode("utf-8") for line in f.readlines()] - data = LuaParser(lines).parse() - cache.set(cache_key, data, timeout=None) - - if page == "addon": - addon_alts = data.get("alts", {}) - result = [] - for alt in ProfileAlt.objects.filter(user=user_id): - addon_alt = addon_alts.get(f"{alt.alt_name}-{alt.alt_realm}", {}) - result.append( - { - "alt_id": alt.alt_id, - "gold": addon_alt.get("gold"), - "played_time_total": addon_alt.get("playedTimeTotal"), - "played_time_level": addon_alt.get("playedTimeLevel"), - } - ) - return response.Response(result) - return response.Response([]) @@ -936,6 +933,24 @@ def list(self, request): return response.Response(serializer.data) +class ProfileAltAddonDataView(viewsets.ModelViewSet): + serializer_class = ProfileAltAddonDataSerializer + queryset = ProfileAltAddonData.objects.all() + + def list(self, request): + user_id = request.query_params.get("user") + + if not user_id: + return response.Response([]) + if request.session.get("user_id") != user_id: + return response.Response([], status=403) + + alt_ids = ProfileAlt.objects.filter(user=user_id).values_list("alt_id", flat=True) + qs = ProfileAltAddonData.objects.filter(alt__in=alt_ids).select_related("alt") + serializer = self.get_serializer(qs, many=True) + return response.Response(serializer.data) + + class ScanAlt(viewsets.ViewSet): def create(self, request): user_id = request.data.get("userid") diff --git a/backend/urls.py b/backend/urls.py index efa70b5..4493e8c 100644 --- a/backend/urls.py +++ b/backend/urls.py @@ -33,6 +33,7 @@ profile.register(r"altreputations", views.ProfileAltReputationView) profile.register(r"altmythicplus", views.ProfileAltMythicPlusView) profile.register(r"altmythicplusdungeons", views.ProfileAltMythicPlusDungeonView) +profile.register(r"altaddondata", views.ProfileAltAddonDataView) data = routers.DefaultRouter() data.register(r"professions", views.DataProfessionView) diff --git a/tests/test_views.py b/tests/test_views.py index 8d10494..ed6c28a 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -12,7 +12,7 @@ from django.utils import timezone from apicore.libs.expansion_order import tier_sort_key -from apicore.models import ProfileUser +from apicore.models import ProfileAlt, ProfileAltAddonData, ProfileUser # --------------------------------------------------------------------------- # Pure helper: tier_sort_key @@ -156,13 +156,26 @@ def test_upload_does_not_500(self): ) assert resp.status_code == 200 - def test_addon_page_reads_uploaded_file(self): - """Regression test: page=addon (and any non-header page reading the - uploaded file) used to crash with 'str' object has no attribute 'decode' - because the file was opened in text mode ("r") despite the parser - expecting bytes to decode itself.""" + def test_upload_syncs_addon_data_for_matching_alts(self): + """Gold/played-time are parsed from the uploaded file and upserted into + ProfileAltAddonData at upload time, not re-parsed on every read.""" user_id = "u1" ProfileUser.objects.create(user_id=user_id, user_file="", user_last_update=timezone.now()) + ProfileAlt.objects.create( + alt_id=1, + alt_account_id=1, + alt_level=80, + alt_name="Testchar", + alt_realm="Realm", + alt_realm_id=1, + alt_realm_slug="realm", + alt_class=1, + alt_race=1, + alt_gender="Male", + alt_faction="Alliance", + alt_expiry_date=timezone.now(), + user_id=user_id, + ) c = self._authed_client(user_id) body = ( @@ -171,6 +184,8 @@ def test_addon_page_reads_uploaded_file(self): '["alts"] = {\n' '["Testchar-Realm"] = {\n' '["gold"] = 12345,\n' + '["playedTimeTotal"] = 6000,\n' + '["playedTimeLevel"] = 100,\n' "},\n" "},\n" "}\n" @@ -178,7 +193,7 @@ def test_addon_page_reads_uploaded_file(self): upload = SimpleUploadedFile( "FazzToolsScraper.lua", body.encode(), content_type="text/plain" ) - c.put( + resp = c.put( f"/api/profile/users/{user_id}/", data=encode_multipart( BOUNDARY, @@ -190,8 +205,37 @@ def test_addon_page_reads_uploaded_file(self): ), content_type=MULTIPART_CONTENT, ) + assert resp.status_code == 200 + + addon_data = ProfileAltAddonData.objects.get(alt_id=1) + assert addon_data.gold == 12345 + assert addon_data.played_time_total == 6000 + assert addon_data.played_time_level == 100 + + +# --------------------------------------------------------------------------- +# ProfileAltAddonDataView — IsSessionUser enforcement +# --------------------------------------------------------------------------- + - resp = c.get(f"/api/profile/users/?user={user_id}&page=addon") +@pytest.mark.django_db +class TestProfileAltAddonDataView: + def test_no_session_returns_403(self, client: Client): + resp = client.get("/api/profile/altaddondata/?user=u1") + assert resp.status_code == 403 + + def test_wrong_session_returns_403(self, client: Client): + session = client.session + session["user_id"] = "other" + session.save() + resp = client.get("/api/profile/altaddondata/?user=u1") + assert resp.status_code == 403 + + def test_correct_session_returns_200(self, client: Client): + session = client.session + session["user_id"] = "u1" + session.save() + resp = client.get("/api/profile/altaddondata/?user=u1") assert resp.status_code == 200 assert resp.json() == []