From eb996e9314ed6f0d82a07aa2f0992e04a7f2e450 Mon Sep 17 00:00:00 2001 From: Joe Farrelly Date: Fri, 19 Jun 2026 20:00:49 +0100 Subject: [PATCH 01/18] Add cross-repo suite badges to README --- readme.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/readme.md b/readme.md index 6563b96..cdb7d6a 100644 --- a/readme.md +++ b/readme.md @@ -5,6 +5,8 @@ [![codecov](https://codecov.io/gh/joefarrelly/FazzToolsAPI/graph/badge.svg)](https://codecov.io/gh/joefarrelly/FazzToolsAPI) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +**Frontend** [![Deploy](https://github.com/joefarrelly/FazzToolsFrontend/actions/workflows/deploy.yml/badge.svg)](https://github.com/joefarrelly/FazzToolsFrontend/actions/workflows/deploy.yml) [![Lint and Test](https://github.com/joefarrelly/FazzToolsFrontend/actions/workflows/lint.yml/badge.svg)](https://github.com/joefarrelly/FazzToolsFrontend/actions/workflows/lint.yml) | **Scraper** [![Lint](https://github.com/joefarrelly/FazzToolsScraper/actions/workflows/lint.yml/badge.svg)](https://github.com/joefarrelly/FazzToolsScraper/actions/workflows/lint.yml) [![Release](https://github.com/joefarrelly/FazzToolsScraper/actions/workflows/release.yml/badge.svg)](https://github.com/joefarrelly/FazzToolsScraper/actions/workflows/release.yml) + Django REST Framework backend for **FazzTools** — a World of Warcraft companion app. Syncs character data (professions, equipment, mounts, pets) from the Blizzard Battle.net API and parses WoW Lua addon exports to serve keybind data. From 68b4faa333e26e31dbe7ddee89bec10fb59f5606 Mon Sep 17 00:00:00 2001 From: Joe Farrelly Date: Fri, 19 Jun 2026 20:13:38 +0100 Subject: [PATCH 02/18] Replace cross-repo badges with suite links --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index cdb7d6a..36b59c2 100644 --- a/readme.md +++ b/readme.md @@ -5,7 +5,7 @@ [![codecov](https://codecov.io/gh/joefarrelly/FazzToolsAPI/graph/badge.svg)](https://codecov.io/gh/joefarrelly/FazzToolsAPI) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) -**Frontend** [![Deploy](https://github.com/joefarrelly/FazzToolsFrontend/actions/workflows/deploy.yml/badge.svg)](https://github.com/joefarrelly/FazzToolsFrontend/actions/workflows/deploy.yml) [![Lint and Test](https://github.com/joefarrelly/FazzToolsFrontend/actions/workflows/lint.yml/badge.svg)](https://github.com/joefarrelly/FazzToolsFrontend/actions/workflows/lint.yml) | **Scraper** [![Lint](https://github.com/joefarrelly/FazzToolsScraper/actions/workflows/lint.yml/badge.svg)](https://github.com/joefarrelly/FazzToolsScraper/actions/workflows/lint.yml) [![Release](https://github.com/joefarrelly/FazzToolsScraper/actions/workflows/release.yml/badge.svg)](https://github.com/joefarrelly/FazzToolsScraper/actions/workflows/release.yml) +**Suite:** [Backend](https://github.com/joefarrelly/FazzToolsAPI) · [Frontend](https://github.com/joefarrelly/FazzToolsFrontend) · [Addon](https://github.com/joefarrelly/FazzToolsScraper) Django REST Framework backend for **FazzTools** — a World of Warcraft companion app. From b626acd5ec66f5dc7852b96309d0b7ff1ddd90f8 Mon Sep 17 00:00:00 2001 From: Joe Farrelly Date: Sun, 21 Jun 2026 22:27:09 +0100 Subject: [PATCH 03/18] Add Celery reliability: rate limiting, error handling, stale purge --- apicore/tasks.py | 107 ++++++++++++++++++++++++++++++++------------ backend/settings.py | 7 +++ docker-compose.yml | 13 ++++++ 3 files changed, 98 insertions(+), 29 deletions(-) diff --git a/apicore/tasks.py b/apicore/tasks.py index 8449929..bacc692 100644 --- a/apicore/tasks.py +++ b/apicore/tasks.py @@ -90,7 +90,17 @@ } _session = requests.Session() -_session.mount("https://", HTTPAdapter(max_retries=Retry(total=3, backoff_factor=1))) +_session.mount( + "https://", + HTTPAdapter( + max_retries=Retry( + total=5, + backoff_factor=1, + status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=["HEAD", "GET", "POST"], + ) + ), +) # --------------------------------------------------------------------------- @@ -129,35 +139,47 @@ def fullAltScan(user_id: str, client: str, secret: str) -> None: group(scan_single_alt.s(alt_id, user_id, token) for alt_id in alt_ids).apply_async() -@shared_task -def scan_single_alt(alt_id: int, user_id: str, token: str) -> None: - alt = ProfileAlt.objects.get(alt_id=alt_id) - user = ProfileUser.objects.get(user_id=user_id) - auth_headers = {"Authorization": f"Bearer {token}"} - - char_base = f"{_EU_API_BASE}/profile/wow/character/{alt.alt_realm_slug}/{alt.alt_name.lower()}" - endpoints = { - "professions": f"{char_base}/professions", - "equipment": f"{char_base}/equipment", - "mounts": f"{char_base}/collections/mounts", - "pets": f"{char_base}/collections/pets", - } - - for key, url in endpoints.items(): - resp = _api_get(url, _PROFILE_PARAMS, auth_headers) - if resp.status_code != 200: - logger.warning("Blizzard API %s %s", resp.status_code, url) - continue - if key == "professions": - _sync_professions(alt, resp.json(), auth_headers) - elif key == "equipment": - _sync_equipment(alt, resp.json()) - elif key == "mounts": - _sync_mounts(user, resp.json()) - elif key == "pets": - _sync_pets(user, resp.json()) +@shared_task(bind=True) +def scan_single_alt(self, alt_id: int, user_id: str, token: str) -> None: + try: + alt = ProfileAlt.objects.get(alt_id=alt_id) + user = ProfileUser.objects.get(user_id=user_id) + auth_headers = {"Authorization": f"Bearer {token}"} - logger.info("Completed alt scan: %s-%s", alt.alt_name, alt.alt_realm_slug) + char_base = ( + f"{_EU_API_BASE}/profile/wow/character/{alt.alt_realm_slug}/{alt.alt_name.lower()}" + ) + endpoints = { + "professions": f"{char_base}/professions", + "equipment": f"{char_base}/equipment", + "mounts": f"{char_base}/collections/mounts", + "pets": f"{char_base}/collections/pets", + } + + for key, url in endpoints.items(): + resp = _api_get(url, _PROFILE_PARAMS, auth_headers) + if resp.status_code != 200: + logger.warning("Blizzard API %s %s", resp.status_code, url) + continue + if key == "professions": + _sync_professions(alt, resp.json(), auth_headers) + elif key == "equipment": + _sync_equipment(alt, resp.json()) + elif key == "mounts": + _sync_mounts(user, resp.json()) + elif key == "pets": + _sync_pets(user, resp.json()) + + logger.info("Completed alt scan: %s-%s", alt.alt_name, alt.alt_realm_slug) + except Exception as exc: + logger.error( + "scan_single_alt failed: alt_id=%s user_id=%s task_id=%s error=%s", + alt_id, + user_id, + self.request.id, + exc, + ) + raise def _sync_professions(alt: ProfileAlt, data: dict, auth_headers: dict) -> None: @@ -595,3 +617,30 @@ def _sync_pet_data(index_data: dict, auth_headers: dict) -> None: ) except (KeyError, TypeError) as exc: logger.warning("Failed to parse pet: %s", exc) + + +# --------------------------------------------------------------------------- +# Maintenance — purge stale profile data +# --------------------------------------------------------------------------- + + +@shared_task +def purge_stale_profiles() -> None: + now = timezone.now() + prof_data_deleted, _ = ProfileAltProfessionData.objects.filter( + alt_profession_data_expiry_date__lt=now + ).delete() + prof_deleted, _ = ProfileAltProfession.objects.filter( + alt_profession_expiry_date__lt=now + ).delete() + equip_deleted, _ = ProfileAltEquipment.objects.filter( + alt_equipment_expiry_date__lt=now + ).delete() + alt_deleted, _ = ProfileAlt.objects.filter(alt_expiry_date__lt=now).delete() + logger.info( + "purge_stale_profiles: deleted %d profession_data, %d professions, %d equipment, %d alts", + prof_data_deleted, + prof_deleted, + equip_deleted, + alt_deleted, + ) diff --git a/backend/settings.py b/backend/settings.py index 02813c7..1f7d0d0 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -106,6 +106,13 @@ "apicore.tasks.scan_single_alt": {"queue": "alt_scan"}, } +CELERY_BEAT_SCHEDULE = { + "purge-stale-profiles-daily": { + "task": "apicore.tasks.purge_stale_profiles", + "schedule": 86400, + }, +} + REST_FRAMEWORK = { "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination", "PAGE_SIZE": 100, diff --git a/docker-compose.yml b/docker-compose.yml index 61fb278..9bf2e57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -69,6 +69,19 @@ services: redis: condition: service_started + beat: + build: . + command: celery -A backend beat -l info + volumes: + - .:/app + env_file: + - .env + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + volumes: db_data: media_data: From 9384c6270c223a5b3d96facc39d96527c410b966 Mon Sep 17 00:00:00 2001 From: Joe Farrelly Date: Sun, 21 Jun 2026 22:32:15 +0100 Subject: [PATCH 04/18] Rename DataEquipmentVariant.armour to armor --- .../migrations/0010_rename_armour_to_armor.py | 17 +++++++++++++++++ apicore/models.py | 2 +- apicore/serializers.py | 2 +- apicore/tasks.py | 4 ++-- 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 apicore/migrations/0010_rename_armour_to_armor.py diff --git a/apicore/migrations/0010_rename_armour_to_armor.py b/apicore/migrations/0010_rename_armour_to_armor.py new file mode 100644 index 0000000..8d9307b --- /dev/null +++ b/apicore/migrations/0010_rename_armour_to_armor.py @@ -0,0 +1,17 @@ +# Generated by Django 5.2 on 2026-06-21 21:30 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("apicore", "0009_dataprofessionrecipe_recipe_icon"), + ] + + operations = [ + migrations.RenameField( + model_name="dataequipmentvariant", + old_name="armour", + new_name="armor", + ), + ] diff --git a/apicore/models.py b/apicore/models.py index ebc850a..8c0f830 100644 --- a/apicore/models.py +++ b/apicore/models.py @@ -90,7 +90,7 @@ class DataEquipmentVariant(models.Model): equipment = models.ForeignKey(DataEquipment, on_delete=models.CASCADE) variant = models.CharField(max_length=64) stamina = models.PositiveSmallIntegerField() - armour = models.PositiveSmallIntegerField() + armor = models.PositiveSmallIntegerField() strength = models.PositiveSmallIntegerField() agility = models.PositiveSmallIntegerField() intellect = models.PositiveSmallIntegerField() diff --git a/apicore/serializers.py b/apicore/serializers.py index 2a07738..c2e4250 100644 --- a/apicore/serializers.py +++ b/apicore/serializers.py @@ -69,7 +69,7 @@ class Meta: "equipment", "variant", "stamina", - "armour", + "armor", "strength", "agility", "intellect", diff --git a/apicore/tasks.py b/apicore/tasks.py index bacc692..844b7f8 100644 --- a/apicore/tasks.py +++ b/apicore/tasks.py @@ -320,14 +320,14 @@ def _sync_equipment(alt: ProfileAlt, data: dict) -> None: def _parse_item_stats(item: dict) -> dict: defaults = {v: 0 for v in _STAT_FIELDS.values()} - defaults["armour"] = 0 + defaults["armor"] = 0 for stat in item.get("stats", []): field = _STAT_FIELDS.get(stat.get("type", {}).get("type", "")) if field: defaults[field] = stat.get("value", 0) - defaults["armour"] = item.get("armor", {}).get("value", 0) + defaults["armor"] = item.get("armor", {}).get("value", 0) defaults["level"] = item.get("level", {}).get("value", 0) defaults["quality"] = item.get("quality", {}).get("name", "") return defaults From 55a4c54ca94f0464462c56bbf9b1f54857d4e8f5 Mon Sep 17 00:00:00 2001 From: Joe Farrelly Date: Sun, 28 Jun 2026 19:45:42 +0100 Subject: [PATCH 05/18] Add celerybeat-schedule to .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 33b94bb..540e90c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,10 @@ test.py # claude local settings .claude/settings.local.json +# Celery +celerybeat-schedule +celerybeat.pid + # Python __pycache__/ *.pyc From 24581ed13f8d7ac0ba736e32b6b8be7851d2326b Mon Sep 17 00:00:00 2001 From: Joe Farrelly Date: Sun, 28 Jun 2026 20:04:38 +0100 Subject: [PATCH 06/18] Add achievements and reputations (Phase 2a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New models: DataAchievement, DataFaction, ProfileAltAchievement, ProfileAltReputation with migration 0011. fullDataScan now fetches achievement and reputation-faction indexes from the Blizzard static API to populate the Data* tables. scan_single_alt now fetches /achievements and /reputations per alt; both sync helpers create Data* records lazily on first encounter. purge_stale_profiles extended to clean up expired achievement and reputation records. New endpoints: /api/profile/altachievements/, /api/profile/altreputations/, /api/data/achievements/, /api/data/factions/ — all support ?user=, ?alt=, ?realm= query params with session auth enforcement. Excludes migrations from ruff lint. --- ...afaction_profilealtachievement_and_more.py | 65 ++++++++++ apicore/models.py | 57 +++++++++ apicore/serializers.py | 52 ++++++++ apicore/tasks.py | 120 +++++++++++++++++- apicore/views.py | 92 ++++++++++++++ backend/urls.py | 4 + celerybeat-schedule | Bin 0 -> 16384 bytes pyproject.toml | 1 + 8 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 apicore/migrations/0011_dataachievement_datafaction_profilealtachievement_and_more.py create mode 100644 celerybeat-schedule diff --git a/apicore/migrations/0011_dataachievement_datafaction_profilealtachievement_and_more.py b/apicore/migrations/0011_dataachievement_datafaction_profilealtachievement_and_more.py new file mode 100644 index 0000000..fc76afd --- /dev/null +++ b/apicore/migrations/0011_dataachievement_datafaction_profilealtachievement_and_more.py @@ -0,0 +1,65 @@ +# Generated by Django 5.2 on 2026-06-28 18:52 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('apicore', '0010_rename_armour_to_armor'), + ] + + operations = [ + migrations.CreateModel( + name='DataAchievement', + fields=[ + ('achievement_id', models.PositiveIntegerField(primary_key=True, serialize=False)), + ('achievement_name', models.CharField(max_length=256)), + ('achievement_points', models.PositiveSmallIntegerField(default=0)), + ('achievement_category', models.CharField(default='', max_length=128)), + ], + options={ + 'db_table': 'ft_data_achievement', + }, + ), + migrations.CreateModel( + name='DataFaction', + fields=[ + ('faction_id', models.PositiveIntegerField(primary_key=True, serialize=False)), + ('faction_name', models.CharField(max_length=256)), + ], + options={ + 'db_table': 'ft_data_faction', + }, + ), + migrations.CreateModel( + name='ProfileAltAchievement', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('completed_timestamp', models.DateTimeField(blank=True, null=True)), + ('alt_achievement_expiry_date', models.DateTimeField()), + ('achievement', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='apicore.dataachievement')), + ('alt', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='apicore.profilealt')), + ], + options={ + 'db_table': 'ft_profile_altachievement', + 'constraints': [models.UniqueConstraint(fields=('alt', 'achievement'), name='unique_altachievement')], + }, + ), + migrations.CreateModel( + name='ProfileAltReputation', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('standing_type', models.CharField(max_length=32)), + ('standing_value', models.PositiveIntegerField()), + ('alt_reputation_expiry_date', models.DateTimeField()), + ('alt', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='apicore.profilealt')), + ('faction', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='apicore.datafaction')), + ], + options={ + 'db_table': 'ft_profile_altreputation', + 'constraints': [models.UniqueConstraint(fields=('alt', 'faction'), name='unique_altreputation')], + }, + ), + ] diff --git a/apicore/models.py b/apicore/models.py index 8c0f830..ea9b4b9 100644 --- a/apicore/models.py +++ b/apicore/models.py @@ -310,3 +310,60 @@ class Meta: def __str__(self): return f"{self.alt.alt_name} - {self.alt.alt_realm}" + + +class DataAchievement(models.Model): + achievement_id = models.PositiveIntegerField(primary_key=True) + achievement_name = models.CharField(max_length=256) + achievement_points = models.PositiveSmallIntegerField(default=0) + achievement_category = models.CharField(max_length=128, default="") + + class Meta: + db_table = "ft_data_achievement" + + def __str__(self): + return f"{self.achievement_id} - {self.achievement_name}" + + +class DataFaction(models.Model): + faction_id = models.PositiveIntegerField(primary_key=True) + faction_name = models.CharField(max_length=256) + + class Meta: + db_table = "ft_data_faction" + + def __str__(self): + return f"{self.faction_id} - {self.faction_name}" + + +class ProfileAltAchievement(models.Model): + alt = models.ForeignKey(ProfileAlt, on_delete=models.CASCADE) + achievement = models.ForeignKey(DataAchievement, on_delete=models.CASCADE) + completed_timestamp = models.DateTimeField(null=True, blank=True) + alt_achievement_expiry_date = models.DateTimeField() + + class Meta: + db_table = "ft_profile_altachievement" + constraints = [ + models.UniqueConstraint(fields=["alt", "achievement"], name="unique_altachievement") + ] + + def __str__(self): + return f"{self.alt} - {self.achievement}" + + +class ProfileAltReputation(models.Model): + alt = models.ForeignKey(ProfileAlt, on_delete=models.CASCADE) + faction = models.ForeignKey(DataFaction, on_delete=models.CASCADE) + standing_type = models.CharField(max_length=32) + standing_value = models.PositiveIntegerField() + alt_reputation_expiry_date = models.DateTimeField() + + class Meta: + db_table = "ft_profile_altreputation" + constraints = [ + models.UniqueConstraint(fields=["alt", "faction"], name="unique_altreputation") + ] + + def __str__(self): + return f"{self.alt} - {self.faction}" diff --git a/apicore/serializers.py b/apicore/serializers.py index c2e4250..fa29842 100644 --- a/apicore/serializers.py +++ b/apicore/serializers.py @@ -1,8 +1,10 @@ from rest_framework import serializers from apicore.models import ( + DataAchievement, DataEquipment, DataEquipmentVariant, + DataFaction, DataMount, DataPet, DataProfession, @@ -11,9 +13,11 @@ DataReagent, DataRecipeReagent, ProfileAlt, + ProfileAltAchievement, ProfileAltEquipment, ProfileAltProfession, ProfileAltProfessionData, + ProfileAltReputation, ProfileUser, ProfileUserMount, ProfileUserPet, @@ -197,3 +201,51 @@ class Meta: "weapon2", "alt_equipment_expiry_date", ) + + +class DataAchievementSerializer(serializers.ModelSerializer): + class Meta: + model = DataAchievement + fields = ( + "achievement_id", + "achievement_name", + "achievement_points", + "achievement_category", + ) + + +class DataFactionSerializer(serializers.ModelSerializer): + class Meta: + model = DataFaction + fields = ("faction_id", "faction_name") + + +class ProfileAltAchievementSerializer(serializers.ModelSerializer): + achievement_name = serializers.ReadOnlyField(source="achievement.achievement_name") + achievement_points = serializers.ReadOnlyField(source="achievement.achievement_points") + achievement_category = serializers.ReadOnlyField(source="achievement.achievement_category") + + class Meta: + model = ProfileAltAchievement + fields = ( + "alt", + "achievement", + "achievement_name", + "achievement_points", + "achievement_category", + "completed_timestamp", + ) + + +class ProfileAltReputationSerializer(serializers.ModelSerializer): + faction_name = serializers.ReadOnlyField(source="faction.faction_name") + + class Meta: + model = ProfileAltReputation + fields = ( + "alt", + "faction", + "faction_name", + "standing_type", + "standing_value", + ) diff --git a/apicore/tasks.py b/apicore/tasks.py index 844b7f8..60f1892 100644 --- a/apicore/tasks.py +++ b/apicore/tasks.py @@ -10,8 +10,10 @@ from apicore.libs.mount_icons import MOUNT_ICONS from apicore.models import ( + DataAchievement, DataEquipment, DataEquipmentVariant, + DataFaction, DataMount, DataPet, DataProfession, @@ -20,9 +22,11 @@ DataReagent, DataRecipeReagent, ProfileAlt, + ProfileAltAchievement, ProfileAltEquipment, ProfileAltProfession, ProfileAltProfessionData, + ProfileAltReputation, ProfileUser, ProfileUserMount, ProfileUserPet, @@ -154,6 +158,8 @@ def scan_single_alt(self, alt_id: int, user_id: str, token: str) -> None: "equipment": f"{char_base}/equipment", "mounts": f"{char_base}/collections/mounts", "pets": f"{char_base}/collections/pets", + "achievements": f"{char_base}/achievements", + "reputations": f"{char_base}/reputations", } for key, url in endpoints.items(): @@ -169,6 +175,10 @@ def scan_single_alt(self, alt_id: int, user_id: str, token: str) -> None: _sync_mounts(user, resp.json()) elif key == "pets": _sync_pets(user, resp.json()) + elif key == "achievements": + _sync_achievements(alt, resp.json()) + elif key == "reputations": + _sync_reputations(alt, resp.json()) logger.info("Completed alt scan: %s-%s", alt.alt_name, alt.alt_realm_slug) except Exception as exc: @@ -375,6 +385,8 @@ def fullDataScan(client: str, secret: str) -> str: f"{_EU_API_BASE}/data/wow/profession/index", f"{_EU_API_BASE}/data/wow/mount/index", f"{_EU_API_BASE}/data/wow/pet/index", + f"{_EU_API_BASE}/data/wow/achievement/index", + f"{_EU_API_BASE}/data/wow/reputation-faction/index", ] for url in index_urls: @@ -388,6 +400,10 @@ def fullDataScan(client: str, secret: str) -> str: _sync_mount_data(resp.json(), auth_headers) elif "pet" in url: _sync_pet_data(resp.json(), auth_headers) + elif "achievement" in url: + _sync_achievement_data(resp.json(), auth_headers) + elif "reputation-faction" in url: + _sync_faction_data(resp.json()) return "Done" @@ -636,11 +652,113 @@ def purge_stale_profiles() -> None: equip_deleted, _ = ProfileAltEquipment.objects.filter( alt_equipment_expiry_date__lt=now ).delete() + ach_deleted, _ = ProfileAltAchievement.objects.filter( + alt_achievement_expiry_date__lt=now + ).delete() + rep_deleted, _ = ProfileAltReputation.objects.filter( + alt_reputation_expiry_date__lt=now + ).delete() alt_deleted, _ = ProfileAlt.objects.filter(alt_expiry_date__lt=now).delete() logger.info( - "purge_stale_profiles: deleted %d profession_data, %d professions, %d equipment, %d alts", + "purge_stale_profiles: deleted %d profession_data, %d professions, %d equipment, " + "%d achievements, %d reputations, %d alts", prof_data_deleted, prof_deleted, equip_deleted, + ach_deleted, + rep_deleted, alt_deleted, ) + + +# --------------------------------------------------------------------------- +# Per-alt sync helpers — achievements + reputations +# --------------------------------------------------------------------------- + + +def _sync_achievements(alt: ProfileAlt, data: dict) -> None: + expiry = timezone.now() + datetime.timedelta(days=30) + for ach_data in data.get("achievements", []): + try: + ach_ref = ach_data.get("achievement", {}) + ach_id = ach_ref.get("id") or ach_data.get("id") + if not ach_id: + continue + achievement, _ = DataAchievement.objects.get_or_create( + achievement_id=ach_id, + defaults={"achievement_name": ach_ref.get("name", "Unknown")}, + ) + ts = ach_data.get("completed_timestamp") + completed = timezone.datetime.fromtimestamp(ts / 1000, tz=datetime.UTC) if ts else None + ProfileAltAchievement.objects.update_or_create( + alt=alt, + achievement=achievement, + defaults={ + "completed_timestamp": completed, + "alt_achievement_expiry_date": expiry, + }, + ) + except (KeyError, TypeError) as exc: + logger.warning("Failed to sync achievement for %s: %s", alt.alt_name, exc) + + +def _sync_reputations(alt: ProfileAlt, data: dict) -> None: + expiry = timezone.now() + datetime.timedelta(days=30) + for rep_data in data.get("reputations", []): + try: + faction_ref = rep_data.get("faction", {}) + faction_id = faction_ref.get("id") + if not faction_id: + continue + faction, _ = DataFaction.objects.get_or_create( + faction_id=faction_id, + defaults={"faction_name": faction_ref.get("name", "Unknown")}, + ) + standing = rep_data.get("standing", {}) + ProfileAltReputation.objects.update_or_create( + alt=alt, + faction=faction, + defaults={ + "standing_type": standing.get("type", ""), + "standing_value": standing.get("raw", 0), + "alt_reputation_expiry_date": expiry, + }, + ) + except (KeyError, TypeError) as exc: + logger.warning("Failed to sync reputation for %s: %s", alt.alt_name, exc) + + +# --------------------------------------------------------------------------- +# Static data sync helpers — achievements + factions +# --------------------------------------------------------------------------- + + +def _sync_achievement_data(index_data: dict, auth_headers: dict) -> None: + for ach_ref in index_data.get("achievements", []): + resp = _api_get(ach_ref["key"]["href"], _STATIC_PARAMS, auth_headers) + if resp.status_code != 200: + logger.warning("Blizzard API %s %s", resp.status_code, ach_ref["key"]["href"]) + continue + try: + details = resp.json() + DataAchievement.objects.update_or_create( + achievement_id=details["id"], + defaults={ + "achievement_name": details["name"], + "achievement_points": details.get("points", 0), + "achievement_category": details.get("category", {}).get("name", ""), + }, + ) + except (KeyError, TypeError) as exc: + logger.warning("Failed to parse achievement %s: %s", ach_ref.get("id"), exc) + + +def _sync_faction_data(index_data: dict) -> None: + for faction_ref in index_data.get("factions", []): + try: + DataFaction.objects.get_or_create( + faction_id=faction_ref["id"], + defaults={"faction_name": faction_ref["name"]}, + ) + except (KeyError, TypeError) as exc: + logger.warning("Failed to parse faction %s: %s", faction_ref.get("id"), exc) diff --git a/apicore/views.py b/apicore/views.py index 26d0a88..253ac27 100644 --- a/apicore/views.py +++ b/apicore/views.py @@ -21,8 +21,10 @@ from apicore.libs.keybind_builder import build_all_keybinds, build_single_keybinds, tier_sort_key from apicore.libs.lua_parser import LuaParser from apicore.models import ( + DataAchievement, DataEquipment, DataEquipmentVariant, + DataFaction, DataMount, DataPet, DataProfession, @@ -31,17 +33,21 @@ DataReagent, DataRecipeReagent, ProfileAlt, + ProfileAltAchievement, ProfileAltEquipment, ProfileAltProfession, ProfileAltProfessionData, + ProfileAltReputation, ProfileUser, ProfileUserMount, ProfileUserPet, ) from apicore.permissions import IsSessionUser from apicore.serializers import ( + DataAchievementSerializer, DataEquipmentSerializer, DataEquipmentVariantSerializer, + DataFactionSerializer, DataMountSerializer, DataPetSerializer, DataProfessionRecipeSerializer, @@ -49,9 +55,11 @@ DataProfessionTierSerializer, DataReagentSerializer, DataRecipeReagentSerializer, + ProfileAltAchievementSerializer, ProfileAltEquipmentSerializer, ProfileAltProfessionDataSerializer, ProfileAltProfessionSerializer, + ProfileAltReputationSerializer, ProfileAltSerializer, ProfileUserMountSerializer, ProfileUserPetSerializer, @@ -704,6 +712,90 @@ def create(self, request): return response.Response({"user": user_id, "alts": alt_ids}) +class DataAchievementView(viewsets.ModelViewSet): + serializer_class = DataAchievementSerializer + queryset = DataAchievement.objects.all() + + +class DataFactionView(viewsets.ModelViewSet): + serializer_class = DataFactionSerializer + queryset = DataFaction.objects.all() + + +class ProfileAltAchievementView(viewsets.ModelViewSet): + serializer_class = ProfileAltAchievementSerializer + queryset = ProfileAltAchievement.objects.all() + + def list(self, request): + user_id = request.query_params.get("user") + alt_name = request.query_params.get("alt", "").title() + realm_slug = request.query_params.get("realm", "") + + if not user_id: + return response.Response([]) + if request.session.get("user_id") != user_id: + return response.Response([], status=403) + + if alt_name and realm_slug: + alt = ProfileAlt.objects.filter( + alt_name=alt_name, alt_realm_slug=realm_slug, user=user_id + ).first() + if not alt: + return response.Response([]) + qs = ( + ProfileAltAchievement.objects.filter(alt=alt) + .select_related("achievement") + .order_by("-completed_timestamp") + ) + else: + alt_ids = ProfileAlt.objects.filter(user=user_id).values_list("alt_id", flat=True) + qs = ( + ProfileAltAchievement.objects.filter(alt__in=alt_ids) + .select_related("alt", "achievement") + .order_by("alt__alt_name", "-completed_timestamp") + ) + + serializer = self.get_serializer(qs, many=True) + return response.Response(serializer.data) + + +class ProfileAltReputationView(viewsets.ModelViewSet): + serializer_class = ProfileAltReputationSerializer + queryset = ProfileAltReputation.objects.all() + + def list(self, request): + user_id = request.query_params.get("user") + alt_name = request.query_params.get("alt", "").title() + realm_slug = request.query_params.get("realm", "") + + if not user_id: + return response.Response([]) + if request.session.get("user_id") != user_id: + return response.Response([], status=403) + + if alt_name and realm_slug: + alt = ProfileAlt.objects.filter( + alt_name=alt_name, alt_realm_slug=realm_slug, user=user_id + ).first() + if not alt: + return response.Response([]) + qs = ( + ProfileAltReputation.objects.filter(alt=alt) + .select_related("faction") + .order_by("-standing_value") + ) + else: + alt_ids = ProfileAlt.objects.filter(user=user_id).values_list("alt_id", flat=True) + qs = ( + ProfileAltReputation.objects.filter(alt__in=alt_ids) + .select_related("alt", "faction") + .order_by("alt__alt_name", "-standing_value") + ) + + serializer = self.get_serializer(qs, many=True) + return response.Response(serializer.data) + + class ScanAlt(viewsets.ViewSet): def create(self, request): user_id = request.data.get("userid") diff --git a/backend/urls.py b/backend/urls.py index a502d74..486f3ed 100644 --- a/backend/urls.py +++ b/backend/urls.py @@ -29,6 +29,8 @@ profile.register(r"altprofessions", views.ProfileAltProfessionView) profile.register(r"altprofessiondatas", views.ProfileAltProfessionDataView) profile.register(r"altequipments", views.ProfileAltEquipmentView) +profile.register(r"altachievements", views.ProfileAltAchievementView) +profile.register(r"altreputations", views.ProfileAltReputationView) data = routers.DefaultRouter() data.register(r"professions", views.DataProfessionView) @@ -40,6 +42,8 @@ data.register(r"equipmentvariants", views.DataEquipmentVariantView) data.register(r"mounts", views.DataMountView) data.register(r"pets", views.DataPetView) +data.register(r"achievements", views.DataAchievementView) +data.register(r"factions", views.DataFactionView) custom = routers.DefaultRouter() custom.register(r"bnetlogin", views.BnetLogin, "bnetlogin") diff --git a/celerybeat-schedule b/celerybeat-schedule new file mode 100644 index 0000000000000000000000000000000000000000..d5af20f8b2d31ee176dcd6a1622d3fe523325a5a GIT binary patch literal 16384 zcmeI3L2DC16vt<6Y!XwY*eXS(h#*EKON4q>C>|u(gRKZ2Tqe8I?viyk?9Pe_1ahht z=GG~Q;8l7Qzkpsn`2jq652?jzT^fy2!H?xfB*=900@8p2!H?xfB*=9z~4yVY>_l<4gws&!4N1OJ!~9Lje!n<*@7=FZfy+p7?!Rq7dKc`dK85 z=TyJ}947Y%>fvDO;r-hW7aZrIm?`wHM$67q1OJ2J56 zM7$jeZ&-R_|A@BpoNC8h16^0Ax>>Rz8mZ)=Xmpc!JqShG@OTjRw5qux6iLr+3of-f zy6XDE%R+I3^6KerUH2_@j&}n$PJ}Ia+DYwEG-nje8AsEq;&CZt@IXwu+j{v(`P1~7 z+TuLh{75tHEC^)~rCOCYgyd2tw1%qm<46S2dQ9#;a&H=!#;Z;i(bAnz=uLfF=l9&Q zjBi?tso5g?v`Cu*SH?a?@TXPP Date: Sun, 28 Jun 2026 20:05:13 +0100 Subject: [PATCH 07/18] Remove celerybeat-schedule from tracking --- celerybeat-schedule | Bin 16384 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 celerybeat-schedule diff --git a/celerybeat-schedule b/celerybeat-schedule deleted file mode 100644 index d5af20f8b2d31ee176dcd6a1622d3fe523325a5a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI3L2DC16vt<6Y!XwY*eXS(h#*EKON4q>C>|u(gRKZ2Tqe8I?viyk?9Pe_1ahht z=GG~Q;8l7Qzkpsn`2jq652?jzT^fy2!H?xfB*=900@8p2!H?xfB*=9z~4yVY>_l<4gws&!4N1OJ!~9Lje!n<*@7=FZfy+p7?!Rq7dKc`dK85 z=TyJ}947Y%>fvDO;r-hW7aZrIm?`wHM$67q1OJ2J56 zM7$jeZ&-R_|A@BpoNC8h16^0Ax>>Rz8mZ)=Xmpc!JqShG@OTjRw5qux6iLr+3of-f zy6XDE%R+I3^6KerUH2_@j&}n$PJ}Ia+DYwEG-nje8AsEq;&CZt@IXwu+j{v(`P1~7 z+TuLh{75tHEC^)~rCOCYgyd2tw1%qm<46S2dQ9#;a&H=!#;Z;i(bAnz=uLfF=l9&Q zjBi?tso5g?v`Cu*SH?a?@TXPP Date: Sun, 28 Jun 2026 22:08:43 +0100 Subject: [PATCH 08/18] Add achievements, reputations, and per-faction collection scan - Add DataAchievement, DataFaction, ProfileAltAchievement, ProfileAltReputation models - Add alt_ilvl field to ProfileAlt (populated from character summary on scan) - Refactor fullAltScan: move mounts/pets/achievements to scan_user_collection which scans one highest-level/ilvl alt per faction instead of all alts - Split fullDataScan into 5 independent subtasks with individual endpoints - Add logout endpoint that flushes Django session - Fix standing_value to IntegerField (Blizzard returns negative for Hated) --- .gitignore | 6 +- CLAUDE.md | 36 +++-- ...ter_profilealtreputation_standing_value.py | 18 +++ apicore/migrations/0013_add_alt_ilvl.py | 18 +++ apicore/models.py | 3 +- apicore/tasks.py | 147 ++++++++++++++---- apicore/views.py | 56 ++++++- backend/urls.py | 6 + 8 files changed, 246 insertions(+), 44 deletions(-) create mode 100644 apicore/migrations/0012_alter_profilealtreputation_standing_value.py create mode 100644 apicore/migrations/0013_add_alt_ilvl.py diff --git a/.gitignore b/.gitignore index 33b94bb..0c6c73b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,8 @@ __pycache__/ *.pyc *.pyo coverage.xml -.coverage \ No newline at end of file +.coverage + +# Celery +celerybeat-schedule +celerybeat.pid \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index fbc6dcf..ead6617 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,16 +96,26 @@ conftest.py pytest env-var setup (pytest_configure hook) - `altprofessiondatas` — `ProfileAltProfessionData`: Known recipes per alt/profession - `altequipments` — `ProfileAltEquipment`: Equipped gear slots per alt - `usermounts` / `userpets` — Collected mounts/pets per user +- `altachievements` — `ProfileAltAchievement`: Achievement completions per alt +- `altreputations` — `ProfileAltReputation`: Faction standing per alt ### Data endpoints (static WoW data, synced via DataScan task) - `professions`, `professiontiers`, `professionrecipes`, `reagents`, `recipereagents` - `equipments`, `equipmentvariants` - `mounts`, `pets` +- `achievements` — `DataAchievement`: All WoW achievements (name, points, category) +- `factions` — `DataFaction`: All WoW reputation factions ### Custom endpoints - `POST /api/custom/bnetlogin/` — Battle.net OAuth2 callback; creates/updates user and syncs alts +- `POST /api/custom/logout/` — Flushes Django session and removes auth state - `POST /api/custom/scanalt/` — Triggers `fullAltScan` Celery task for a user -- `POST /api/custom/datascan/` — Triggers `fullDataScan` Celery task (Django admin user required) +- `POST /api/custom/datascan/` — Triggers all data scans (Django admin required) +- `POST /api/custom/datascan/professions/` — Triggers profession data scan only +- `POST /api/custom/datascan/mounts/` — Triggers mount data scan only +- `POST /api/custom/datascan/pets/` — Triggers pet data scan only +- `POST /api/custom/datascan/achievements/` — Triggers achievement data scan only +- `POST /api/custom/datascan/factions/` — Triggers faction data scan only ## Key data flows @@ -113,14 +123,22 @@ conftest.py pytest env-var setup (pytest_configure hook) `BnetLogin.create` → exchanges auth code for token (using `Authorization: Bearer` header) → fetches WoW profile → HMAC-hashes Blizzard user ID → upserts `ProfileUser` and all `ProfileAlt` records. ### Alt scan (`fullAltScan` Celery task) -For each alt belonging to a user, fetches from Blizzard API: -1. `/professions` → upserts `ProfileAltProfession` + `ProfileAltProfessionData` (creates missing `DataProfessionRecipe` entries on the fly) -2. `/equipment` → upserts `ProfileAltEquipment` + `DataEquipment` / `DataEquipmentVariant` -3. `/collections/mounts` → links known `DataMount` records to user via `ProfileUserMount` -4. `/collections/pets` → links known `DataPet` records to user via `ProfileUserPet` +Dispatches two sets of tasks in parallel: + +**Per-alt** (`scan_single_alt` × N alts): +1. Character summary → updates `ProfileAlt.alt_ilvl` (equipped item level) +2. `/professions` → upserts `ProfileAltProfession` + `ProfileAltProfessionData` +3. `/equipment` → upserts `ProfileAltEquipment` + `DataEquipment` / `DataEquipmentVariant` +4. `/reputations` → upserts `ProfileAltReputation` per faction + +**Per-user** (`scan_user_collection` × 1): +Picks the highest-level, highest-ilvl alt per faction (Alliance + Horde) and fetches: +- `/collections/mounts` → links known `DataMount` to user via `ProfileUserMount` +- `/collections/pets` → links known `DataPet` to user via `ProfileUserPet` +- `/achievements` → upserts `ProfileAltAchievement` for that representative alt ### Data scan (`fullDataScan` Celery task) -Fetches Blizzard static data API indexes and walks all professions (tiers → categories → recipes → reagents) and all mounts/pets, creating `Data*` records. +Dispatches five independent subtasks: `scanProfessionData`, `scanMountData`, `scanPetData`, `scanAchievementData`, `scanFactionData`. Each can also be triggered individually via its own endpoint. ### Lua keybind file `ProfileUser.perform_update` validates and stores a `FazzToolsScraper.lua` addon export. @@ -128,9 +146,9 @@ Fetches Blizzard static data API indexes and walks all professions (tiers → ca ## Database tables (all prefixed `ft_`) -**Data (static):** `ft_data_profession`, `ft_data_professiontier`, `ft_data_professionrecipe`, `ft_data_reagent`, `ft_data_recipereagent`, `ft_data_equipment`, `ft_data_equipmentvariant`, `ft_data_mount`, `ft_data_pet` +**Data (static):** `ft_data_profession`, `ft_data_professiontier`, `ft_data_professionrecipe`, `ft_data_reagent`, `ft_data_recipereagent`, `ft_data_equipment`, `ft_data_equipmentvariant`, `ft_data_mount`, `ft_data_pet`, `ft_data_achievement`, `ft_data_faction` -**Profile (user):** `ft_profile_user`, `ft_profile_alt`, `ft_profile_altprofession`, `ft_profile_altprofessiondata`, `ft_profile_altequipment`, `ft_profile_usermount`, `ft_profile_userpet` +**Profile (user):** `ft_profile_user`, `ft_profile_alt`, `ft_profile_altprofession`, `ft_profile_altprofessiondata`, `ft_profile_altequipment`, `ft_profile_usermount`, `ft_profile_userpet`, `ft_profile_altachievement`, `ft_profile_altreputation` ## Things to know diff --git a/apicore/migrations/0012_alter_profilealtreputation_standing_value.py b/apicore/migrations/0012_alter_profilealtreputation_standing_value.py new file mode 100644 index 0000000..593397b --- /dev/null +++ b/apicore/migrations/0012_alter_profilealtreputation_standing_value.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2 on 2026-06-28 19:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('apicore', '0011_dataachievement_datafaction_profilealtachievement_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='profilealtreputation', + name='standing_value', + field=models.IntegerField(), + ), + ] diff --git a/apicore/migrations/0013_add_alt_ilvl.py b/apicore/migrations/0013_add_alt_ilvl.py new file mode 100644 index 0000000..7a7143b --- /dev/null +++ b/apicore/migrations/0013_add_alt_ilvl.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2 on 2026-06-28 20:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('apicore', '0012_alter_profilealtreputation_standing_value'), + ] + + operations = [ + migrations.AddField( + model_name='profilealt', + name='alt_ilvl', + field=models.PositiveSmallIntegerField(default=0), + ), + ] diff --git a/apicore/models.py b/apicore/models.py index ea9b4b9..36eda26 100644 --- a/apicore/models.py +++ b/apicore/models.py @@ -206,6 +206,7 @@ class AltClass(models.IntegerChoices): alt_id = models.PositiveIntegerField(primary_key=True) alt_account_id = models.PositiveIntegerField() alt_level = models.PositiveSmallIntegerField() + alt_ilvl = models.PositiveSmallIntegerField(default=0) alt_name = models.CharField(max_length=64) alt_realm = models.CharField(max_length=64) alt_realm_id = models.PositiveSmallIntegerField() @@ -356,7 +357,7 @@ class ProfileAltReputation(models.Model): alt = models.ForeignKey(ProfileAlt, on_delete=models.CASCADE) faction = models.ForeignKey(DataFaction, on_delete=models.CASCADE) standing_type = models.CharField(max_length=32) - standing_value = models.PositiveIntegerField() + standing_value = models.IntegerField() alt_reputation_expiry_date = models.DateTimeField() class Meta: diff --git a/apicore/tasks.py b/apicore/tasks.py index 60f1892..4796c93 100644 --- a/apicore/tasks.py +++ b/apicore/tasks.py @@ -141,24 +141,27 @@ def fullAltScan(user_id: str, client: str, secret: str) -> None: ProfileUser.objects.filter(user_id=user_id).update(user_last_update=timezone.now()) logger.info("Dispatching %d alt scan tasks for user %s", len(alt_ids), user_id) group(scan_single_alt.s(alt_id, user_id, token) for alt_id in alt_ids).apply_async() + scan_user_collection.delay(user_id, token) @shared_task(bind=True) def scan_single_alt(self, alt_id: int, user_id: str, token: str) -> None: try: alt = ProfileAlt.objects.get(alt_id=alt_id) - user = ProfileUser.objects.get(user_id=user_id) auth_headers = {"Authorization": f"Bearer {token}"} char_base = ( f"{_EU_API_BASE}/profile/wow/character/{alt.alt_realm_slug}/{alt.alt_name.lower()}" ) + + summary_resp = _api_get(char_base, _PROFILE_PARAMS, auth_headers) + if summary_resp.status_code == 200: + ilvl = summary_resp.json().get("equipped_item_level", 0) + ProfileAlt.objects.filter(alt_id=alt_id).update(alt_ilvl=ilvl) + endpoints = { "professions": f"{char_base}/professions", "equipment": f"{char_base}/equipment", - "mounts": f"{char_base}/collections/mounts", - "pets": f"{char_base}/collections/pets", - "achievements": f"{char_base}/achievements", "reputations": f"{char_base}/reputations", } @@ -171,12 +174,6 @@ def scan_single_alt(self, alt_id: int, user_id: str, token: str) -> None: _sync_professions(alt, resp.json(), auth_headers) elif key == "equipment": _sync_equipment(alt, resp.json()) - elif key == "mounts": - _sync_mounts(user, resp.json()) - elif key == "pets": - _sync_pets(user, resp.json()) - elif key == "achievements": - _sync_achievements(alt, resp.json()) elif key == "reputations": _sync_reputations(alt, resp.json()) @@ -192,6 +189,52 @@ def scan_single_alt(self, alt_id: int, user_id: str, token: str) -> None: raise +@shared_task +def scan_user_collection(user_id: str, token: str) -> None: + """Fetch account-wide data (mounts, pets, achievements) from one alt per faction.""" + try: + user = ProfileUser.objects.get(user_id=user_id) + auth_headers = {"Authorization": f"Bearer {token}"} + + alts_by_faction: dict[str, ProfileAlt] = {} + for alt in ProfileAlt.objects.filter(user=user_id).order_by("-alt_level", "-alt_ilvl"): + faction = alt.alt_faction.upper() + if faction not in alts_by_faction: + alts_by_faction[faction] = alt + if len(alts_by_faction) >= 2: + break + + for faction, alt in alts_by_faction.items(): + char_base = ( + f"{_EU_API_BASE}/profile/wow/character/{alt.alt_realm_slug}/{alt.alt_name.lower()}" + ) + for endpoint, key in [ + (f"{char_base}/collections/mounts", "mounts"), + (f"{char_base}/collections/pets", "pets"), + (f"{char_base}/achievements", "achievements"), + ]: + resp = _api_get(endpoint, _PROFILE_PARAMS, auth_headers) + if resp.status_code != 200: + logger.warning("Blizzard API %s %s", resp.status_code, endpoint) + continue + if key == "mounts": + _sync_mounts(user, resp.json()) + elif key == "pets": + _sync_pets(user, resp.json()) + elif key == "achievements": + _sync_achievements(alt, resp.json()) + + logger.info( + "Completed collection scan for %s faction via %s-%s", + faction, + alt.alt_name, + alt.alt_realm_slug, + ) + except Exception as exc: + logger.error("scan_user_collection failed: user_id=%s error=%s", user_id, exc) + raise + + def _sync_professions(alt: ProfileAlt, data: dict, auth_headers: dict) -> None: expiry = timezone.now() + datetime.timedelta(days=30) prof_record, _ = ProfileAltProfession.objects.get_or_create( @@ -378,33 +421,73 @@ def _sync_pets(user: ProfileUser, data: dict) -> None: @shared_task def fullDataScan(client: str, secret: str) -> str: + scanProfessionData.delay(client, secret) + scanMountData.delay(client, secret) + scanPetData.delay(client, secret) + scanAchievementData.delay(client, secret) + scanFactionData.delay(client, secret) + return "Dispatched all data scans" + + +@shared_task +def scanProfessionData(client: str, secret: str) -> str: token = _fetch_token(client, secret) auth_headers = {"Authorization": f"Bearer {token}"} + resp = _api_get(f"{_EU_API_BASE}/data/wow/profession/index", _STATIC_PARAMS, auth_headers) + if resp.status_code != 200: + logger.error("Blizzard API %s profession/index", resp.status_code) + return "Failed" + _sync_profession_data(resp.json(), auth_headers) + return "Done" - index_urls = [ - f"{_EU_API_BASE}/data/wow/profession/index", - f"{_EU_API_BASE}/data/wow/mount/index", - f"{_EU_API_BASE}/data/wow/pet/index", - f"{_EU_API_BASE}/data/wow/achievement/index", - f"{_EU_API_BASE}/data/wow/reputation-faction/index", - ] - for url in index_urls: - resp = _api_get(url, _STATIC_PARAMS, auth_headers) - if resp.status_code != 200: - logger.error("Blizzard API %s %s", resp.status_code, url) - continue - if "profession" in url: - _sync_profession_data(resp.json(), auth_headers) - elif "mount" in url: - _sync_mount_data(resp.json(), auth_headers) - elif "pet" in url: - _sync_pet_data(resp.json(), auth_headers) - elif "achievement" in url: - _sync_achievement_data(resp.json(), auth_headers) - elif "reputation-faction" in url: - _sync_faction_data(resp.json()) +@shared_task +def scanMountData(client: str, secret: str) -> str: + token = _fetch_token(client, secret) + auth_headers = {"Authorization": f"Bearer {token}"} + resp = _api_get(f"{_EU_API_BASE}/data/wow/mount/index", _STATIC_PARAMS, auth_headers) + if resp.status_code != 200: + logger.error("Blizzard API %s mount/index", resp.status_code) + return "Failed" + _sync_mount_data(resp.json(), auth_headers) + return "Done" + + +@shared_task +def scanPetData(client: str, secret: str) -> str: + token = _fetch_token(client, secret) + auth_headers = {"Authorization": f"Bearer {token}"} + resp = _api_get(f"{_EU_API_BASE}/data/wow/pet/index", _STATIC_PARAMS, auth_headers) + if resp.status_code != 200: + logger.error("Blizzard API %s pet/index", resp.status_code) + return "Failed" + _sync_pet_data(resp.json(), auth_headers) + return "Done" + +@shared_task +def scanAchievementData(client: str, secret: str) -> str: + token = _fetch_token(client, secret) + auth_headers = {"Authorization": f"Bearer {token}"} + resp = _api_get(f"{_EU_API_BASE}/data/wow/achievement/index", _STATIC_PARAMS, auth_headers) + if resp.status_code != 200: + logger.error("Blizzard API %s achievement/index", resp.status_code) + return "Failed" + _sync_achievement_data(resp.json(), auth_headers) + return "Done" + + +@shared_task +def scanFactionData(client: str, secret: str) -> str: + token = _fetch_token(client, secret) + auth_headers = {"Authorization": f"Bearer {token}"} + resp = _api_get( + f"{_EU_API_BASE}/data/wow/reputation-faction/index", _STATIC_PARAMS, auth_headers + ) + if resp.status_code != 200: + logger.error("Blizzard API %s reputation-faction/index", resp.status_code) + return "Failed" + _sync_faction_data(resp.json()) return "Done" diff --git a/apicore/views.py b/apicore/views.py index 253ac27..7dcdd84 100644 --- a/apicore/views.py +++ b/apicore/views.py @@ -65,7 +65,15 @@ ProfileUserPetSerializer, ProfileUserSerializer, ) -from apicore.tasks import fullAltScan, fullDataScan +from apicore.tasks import ( + fullAltScan, + fullDataScan, + scanAchievementData, + scanFactionData, + scanMountData, + scanPetData, + scanProfessionData, +) logger = logging.getLogger(__name__) @@ -807,9 +815,55 @@ def create(self, request): return response.Response(timezone.now()) +class Logout(viewsets.ViewSet): + def create(self, request): + request.session.flush() + return response.Response("ok") + + class DataScan(viewsets.ViewSet): permission_classes = [IsAdminUser] def create(self, request): fullDataScan.delay(BLIZZ_CLIENT, BLIZZ_SECRET) return response.Response("Scan started") + + +class DataScanProfessions(viewsets.ViewSet): + permission_classes = [IsAdminUser] + + def create(self, request): + scanProfessionData.delay(BLIZZ_CLIENT, BLIZZ_SECRET) + return response.Response("Profession scan started") + + +class DataScanMounts(viewsets.ViewSet): + permission_classes = [IsAdminUser] + + def create(self, request): + scanMountData.delay(BLIZZ_CLIENT, BLIZZ_SECRET) + return response.Response("Mount scan started") + + +class DataScanPets(viewsets.ViewSet): + permission_classes = [IsAdminUser] + + def create(self, request): + scanPetData.delay(BLIZZ_CLIENT, BLIZZ_SECRET) + return response.Response("Pet scan started") + + +class DataScanAchievements(viewsets.ViewSet): + permission_classes = [IsAdminUser] + + def create(self, request): + scanAchievementData.delay(BLIZZ_CLIENT, BLIZZ_SECRET) + return response.Response("Achievement scan started") + + +class DataScanFactions(viewsets.ViewSet): + permission_classes = [IsAdminUser] + + def create(self, request): + scanFactionData.delay(BLIZZ_CLIENT, BLIZZ_SECRET) + return response.Response("Faction scan started") diff --git a/backend/urls.py b/backend/urls.py index 486f3ed..b4e3866 100644 --- a/backend/urls.py +++ b/backend/urls.py @@ -48,7 +48,13 @@ custom = routers.DefaultRouter() custom.register(r"bnetlogin", views.BnetLogin, "bnetlogin") custom.register(r"scanalt", views.ScanAlt, "scanalt") +custom.register(r"logout", views.Logout, "logout") custom.register(r"datascan", views.DataScan, "datascan") +custom.register(r"datascan/professions", views.DataScanProfessions, "datascanprofessions") +custom.register(r"datascan/mounts", views.DataScanMounts, "datascanmounts") +custom.register(r"datascan/pets", views.DataScanPets, "datascanpets") +custom.register(r"datascan/achievements", views.DataScanAchievements, "datscanachievements") +custom.register(r"datascan/factions", views.DataScanFactions, "datascanfactions") # custom.register(r'fileupload', views.FileUpload, 'fileupload') urlpatterns = [ From d041c7db4088e4901c041607cebd3c4a6702a7cf Mon Sep 17 00:00:00 2001 From: Joe Farrelly Date: Mon, 29 Jun 2026 00:28:03 +0100 Subject: [PATCH 09/18] Add faction expansion mapping, fix rep standing, improve achievement view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DataFaction.faction_category: new field (migration 0014) populated via hardcoded FACTION_EXPANSION map (283 factions, Classic → Midnight) - _sync_reputations: use standing.name not standing.type (was always empty); derive faction_category from FACTION_EXPANSION map - ProfileAltAchievementView: add ?summary=1 (category aggregates) and ?category=X (per-category lazy load) to avoid returning all achievements - Achievement/rep serializers: expose alt_name and faction_category - _sync_achievement_data: add progress logging every 100 achievements --- CLAUDE.md | 9 +- apicore/libs/faction_expansion.py | 297 ++++++++++++++++++ .../migrations/0014_add_faction_category.py | 18 ++ apicore/models.py | 1 + apicore/serializers.py | 6 + apicore/tasks.py | 22 +- apicore/views.py | 55 +++- 7 files changed, 395 insertions(+), 13 deletions(-) create mode 100644 apicore/libs/faction_expansion.py create mode 100644 apicore/migrations/0014_add_faction_category.py diff --git a/CLAUDE.md b/CLAUDE.md index ead6617..d5f09b2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,10 +71,11 @@ apicore/ The single Django app serializers.py DRF serializers permissions.py IsSessionUser permission class libs/ - keybind_builder.py Pure keybind-building logic (build_all/single_keybinds, tier_sort_key) - keybind_mapping.py Slot→action-button mappings per addon - lua_parser.py Hand-rolled Lua-table-to-JSON converter - icon_mapping.py Mount/pet icon mappings + keybind_builder.py Pure keybind-building logic (build_all/single_keybinds, tier_sort_key) + keybind_mapping.py Slot→action-button mappings per addon + lua_parser.py Hand-rolled Lua-table-to-JSON converter + icon_mapping.py Mount/pet icon mappings + faction_expansion.py Hardcoded faction_id → expansion name mapping (283 factions) migrations/ DB migrations tests/ pytest suite (47 tests); run via pytest tests/ conftest.py pytest env-var setup (pytest_configure hook) diff --git a/apicore/libs/faction_expansion.py b/apicore/libs/faction_expansion.py new file mode 100644 index 0000000..11b204c --- /dev/null +++ b/apicore/libs/faction_expansion.py @@ -0,0 +1,297 @@ +FACTION_EXPANSION: dict[int, str] = { + # Classic + 21: "Classic", + 47: "Classic", + 54: "Classic", + 59: "Classic", + 67: "Classic", + 68: "Classic", + 69: "Classic", + 70: "Classic", + 72: "Classic", + 76: "Classic", + 81: "Classic", + 87: "Classic", + 92: "Classic", + 93: "Classic", + 270: "Classic", + 349: "Classic", + 369: "Classic", + 469: "Classic", + 470: "Classic", + 509: "Classic", + 510: "Classic", + 529: "Classic", + 530: "Classic", + 576: "Classic", + 577: "Classic", + 589: "Classic", + 609: "Classic", + 729: "Classic", + 730: "Classic", + 749: "Classic", + 809: "Classic", + 889: "Classic", + 890: "Classic", + 891: "Classic", + 892: "Classic", + 909: "Classic", + 910: "Classic", + 1118: "Classic", + # The Burning Crusade + 911: "The Burning Crusade", + 922: "The Burning Crusade", + 930: "The Burning Crusade", + 932: "The Burning Crusade", + 933: "The Burning Crusade", + 934: "The Burning Crusade", + 935: "The Burning Crusade", + 941: "The Burning Crusade", + 942: "The Burning Crusade", + 946: "The Burning Crusade", + 947: "The Burning Crusade", + 967: "The Burning Crusade", + 970: "The Burning Crusade", + 978: "The Burning Crusade", + 980: "The Burning Crusade", + 989: "The Burning Crusade", + 990: "The Burning Crusade", + 1011: "The Burning Crusade", + 1012: "The Burning Crusade", + 1015: "The Burning Crusade", + 1031: "The Burning Crusade", + 1038: "The Burning Crusade", + 1077: "The Burning Crusade", + # Wrath of the Lich King + 1037: "Wrath of the Lich King", + 1050: "Wrath of the Lich King", + 1052: "Wrath of the Lich King", + 1064: "Wrath of the Lich King", + 1067: "Wrath of the Lich King", + 1068: "Wrath of the Lich King", + 1073: "Wrath of the Lich King", + 1085: "Wrath of the Lich King", + 1090: "Wrath of the Lich King", + 1091: "Wrath of the Lich King", + 1094: "Wrath of the Lich King", + 1097: "Wrath of the Lich King", + 1098: "Wrath of the Lich King", + 1104: "Wrath of the Lich King", + 1105: "Wrath of the Lich King", + 1106: "Wrath of the Lich King", + 1119: "Wrath of the Lich King", + 1124: "Wrath of the Lich King", + 1126: "Wrath of the Lich King", + 1156: "Wrath of the Lich King", + # Cataclysm + 1133: "Cataclysm", + 1134: "Cataclysm", + 1135: "Cataclysm", + 1158: "Cataclysm", + 1162: "Cataclysm", + 1168: "Cataclysm", + 1169: "Cataclysm", + 1171: "Cataclysm", + 1172: "Cataclysm", + 1173: "Cataclysm", + 1174: "Cataclysm", + 1177: "Cataclysm", + 1178: "Cataclysm", + 1204: "Cataclysm", + # Mists of Pandaria + 1216: "Mists of Pandaria", + 1228: "Mists of Pandaria", + 1242: "Mists of Pandaria", + 1245: "Mists of Pandaria", + 1269: "Mists of Pandaria", + 1270: "Mists of Pandaria", + 1271: "Mists of Pandaria", + 1272: "Mists of Pandaria", + 1273: "Mists of Pandaria", + 1275: "Mists of Pandaria", + 1276: "Mists of Pandaria", + 1277: "Mists of Pandaria", + 1278: "Mists of Pandaria", + 1279: "Mists of Pandaria", + 1280: "Mists of Pandaria", + 1281: "Mists of Pandaria", + 1282: "Mists of Pandaria", + 1283: "Mists of Pandaria", + 1302: "Mists of Pandaria", + 1337: "Mists of Pandaria", + 1341: "Mists of Pandaria", + 1345: "Mists of Pandaria", + 1352: "Mists of Pandaria", + 1353: "Mists of Pandaria", + 1358: "Mists of Pandaria", + 1359: "Mists of Pandaria", + 1374: "Mists of Pandaria", + 1375: "Mists of Pandaria", + 1376: "Mists of Pandaria", + 1387: "Mists of Pandaria", + 1388: "Mists of Pandaria", + 1416: "Mists of Pandaria", + 1419: "Mists of Pandaria", + 1435: "Mists of Pandaria", + 1440: "Mists of Pandaria", + 1492: "Mists of Pandaria", + # Warlords of Draenor + 1444: "Warlords of Draenor", + 1445: "Warlords of Draenor", + 1515: "Warlords of Draenor", + 1681: "Warlords of Draenor", + 1682: "Warlords of Draenor", + 1690: "Warlords of Draenor", + 1691: "Warlords of Draenor", + 1708: "Warlords of Draenor", + 1710: "Warlords of Draenor", + 1711: "Warlords of Draenor", + 1731: "Warlords of Draenor", + 1733: "Warlords of Draenor", + 1735: "Warlords of Draenor", + 1736: "Warlords of Draenor", + 1737: "Warlords of Draenor", + 1738: "Warlords of Draenor", + 1739: "Warlords of Draenor", + 1740: "Warlords of Draenor", + 1741: "Warlords of Draenor", + 1847: "Warlords of Draenor", + 1848: "Warlords of Draenor", + 1849: "Warlords of Draenor", + 1850: "Warlords of Draenor", + # Legion + 1072: "Legion", + 1828: "Legion", + 1834: "Legion", + 1859: "Legion", + 1883: "Legion", + 1894: "Legion", + 1900: "Legion", + 1948: "Legion", + 1975: "Legion", + 1984: "Legion", + 2010: "Legion", + 2011: "Legion", + 2018: "Legion", + 2045: "Legion", + 2097: "Legion", + 2098: "Legion", + 2099: "Legion", + 2100: "Legion", + 2101: "Legion", + 2102: "Legion", + 2135: "Legion", + 2165: "Legion", + 2170: "Legion", + # Battle for Azeroth + 2103: "Battle for Azeroth", + 2104: "Battle for Azeroth", + 2156: "Battle for Azeroth", + 2157: "Battle for Azeroth", + 2158: "Battle for Azeroth", + 2159: "Battle for Azeroth", + 2160: "Battle for Azeroth", + 2161: "Battle for Azeroth", + 2162: "Battle for Azeroth", + 2163: "Battle for Azeroth", + 2164: "Battle for Azeroth", + 2371: "Battle for Azeroth", + 2372: "Battle for Azeroth", + 2373: "Battle for Azeroth", + 2391: "Battle for Azeroth", + 2395: "Battle for Azeroth", + 2400: "Battle for Azeroth", + 2415: "Battle for Azeroth", + 2417: "Battle for Azeroth", + # Shadowlands + 2407: "Shadowlands", + 2410: "Shadowlands", + 2413: "Shadowlands", + 2414: "Shadowlands", + 2432: "Shadowlands", + 2439: "Shadowlands", + 2445: "Shadowlands", + 2446: "Shadowlands", + 2447: "Shadowlands", + 2448: "Shadowlands", + 2449: "Shadowlands", + 2450: "Shadowlands", + 2451: "Shadowlands", + 2452: "Shadowlands", + 2453: "Shadowlands", + 2454: "Shadowlands", + 2455: "Shadowlands", + 2456: "Shadowlands", + 2457: "Shadowlands", + 2458: "Shadowlands", + 2459: "Shadowlands", + 2460: "Shadowlands", + 2461: "Shadowlands", + 2462: "Shadowlands", + 2464: "Shadowlands", + 2465: "Shadowlands", + 2469: "Shadowlands", + 2470: "Shadowlands", + 2472: "Shadowlands", + 2478: "Shadowlands", + # Dragonflight + 2503: "Dragonflight", + 2506: "Dragonflight", + 2507: "Dragonflight", + 2510: "Dragonflight", + 2511: "Dragonflight", + 2517: "Dragonflight", + 2518: "Dragonflight", + 2523: "Dragonflight", + 2524: "Dragonflight", + 2526: "Dragonflight", + 2544: "Dragonflight", + 2550: "Dragonflight", + 2553: "Dragonflight", + 2564: "Dragonflight", + 2568: "Dragonflight", + 2574: "Dragonflight", + # The War Within + 2569: "The War Within", + 2570: "The War Within", + 2590: "The War Within", + 2593: "The War Within", + 2594: "The War Within", + 2600: "The War Within", + 2601: "The War Within", + 2605: "The War Within", + 2607: "The War Within", + 2615: "The War Within", + 2640: "The War Within", + 2645: "The War Within", + 2653: "The War Within", + 2658: "The War Within", + 2662: "The War Within", + 2663: "The War Within", + 2664: "The War Within", + 2665: "The War Within", + 2666: "The War Within", + 2669: "The War Within", + 2671: "The War Within", + 2673: "The War Within", + 2675: "The War Within", + 2677: "The War Within", + 2685: "The War Within", + 2688: "The War Within", + 2766: "The War Within", + 2767: "The War Within", + 2770: "The War Within", + 2792: "The War Within", + # Midnight (Silvermoon / Blood Elf factions, next expansion) + 2696: "Midnight", + 2698: "Midnight", + 2699: "Midnight", + 2704: "Midnight", + 2710: "Midnight", + 2711: "Midnight", + 2712: "Midnight", + 2713: "Midnight", + 2714: "Midnight", + 2736: "Midnight", + 2744: "Midnight", +} diff --git a/apicore/migrations/0014_add_faction_category.py b/apicore/migrations/0014_add_faction_category.py new file mode 100644 index 0000000..e37e9da --- /dev/null +++ b/apicore/migrations/0014_add_faction_category.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2 on 2026-06-28 22:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('apicore', '0013_add_alt_ilvl'), + ] + + operations = [ + migrations.AddField( + model_name='datafaction', + name='faction_category', + field=models.CharField(default='', max_length=128), + ), + ] diff --git a/apicore/models.py b/apicore/models.py index 36eda26..df741c1 100644 --- a/apicore/models.py +++ b/apicore/models.py @@ -329,6 +329,7 @@ def __str__(self): class DataFaction(models.Model): faction_id = models.PositiveIntegerField(primary_key=True) faction_name = models.CharField(max_length=256) + faction_category = models.CharField(max_length=128, default="") class Meta: db_table = "ft_data_faction" diff --git a/apicore/serializers.py b/apicore/serializers.py index fa29842..d0bdb1d 100644 --- a/apicore/serializers.py +++ b/apicore/serializers.py @@ -221,6 +221,7 @@ class Meta: class ProfileAltAchievementSerializer(serializers.ModelSerializer): + alt_name = serializers.ReadOnlyField(source="alt.alt_name") achievement_name = serializers.ReadOnlyField(source="achievement.achievement_name") achievement_points = serializers.ReadOnlyField(source="achievement.achievement_points") achievement_category = serializers.ReadOnlyField(source="achievement.achievement_category") @@ -229,6 +230,7 @@ class Meta: model = ProfileAltAchievement fields = ( "alt", + "alt_name", "achievement", "achievement_name", "achievement_points", @@ -238,14 +240,18 @@ class Meta: class ProfileAltReputationSerializer(serializers.ModelSerializer): + alt_name = serializers.ReadOnlyField(source="alt.alt_name") faction_name = serializers.ReadOnlyField(source="faction.faction_name") + faction_category = serializers.ReadOnlyField(source="faction.faction_category") class Meta: model = ProfileAltReputation fields = ( "alt", + "alt_name", "faction", "faction_name", + "faction_category", "standing_type", "standing_value", ) diff --git a/apicore/tasks.py b/apicore/tasks.py index 4796c93..0281fdb 100644 --- a/apicore/tasks.py +++ b/apicore/tasks.py @@ -8,6 +8,7 @@ from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry +from apicore.libs.faction_expansion import FACTION_EXPANSION from apicore.libs.mount_icons import MOUNT_ICONS from apicore.models import ( DataAchievement, @@ -793,16 +794,21 @@ def _sync_reputations(alt: ProfileAlt, data: dict) -> None: faction_id = faction_ref.get("id") if not faction_id: continue - faction, _ = DataFaction.objects.get_or_create( + standing = rep_data.get("standing", {}) + if not standing: + continue + faction, _ = DataFaction.objects.update_or_create( faction_id=faction_id, - defaults={"faction_name": faction_ref.get("name", "Unknown")}, + defaults={ + "faction_name": faction_ref.get("name", "Unknown"), + "faction_category": FACTION_EXPANSION.get(faction_id, ""), + }, ) - standing = rep_data.get("standing", {}) ProfileAltReputation.objects.update_or_create( alt=alt, faction=faction, defaults={ - "standing_type": standing.get("type", ""), + "standing_type": standing.get("name", ""), "standing_value": standing.get("raw", 0), "alt_reputation_expiry_date": expiry, }, @@ -817,7 +823,12 @@ def _sync_reputations(alt: ProfileAlt, data: dict) -> None: def _sync_achievement_data(index_data: dict, auth_headers: dict) -> None: - for ach_ref in index_data.get("achievements", []): + achievements = index_data.get("achievements", []) + total = len(achievements) + logger.info("Achievement scan: %d achievements to process", total) + for i, ach_ref in enumerate(achievements, 1): + if i % 100 == 0 or i == total: + logger.info("Achievement scan: %d/%d", i, total) resp = _api_get(ach_ref["key"]["href"], _STATIC_PARAMS, auth_headers) if resp.status_code != 200: logger.warning("Blizzard API %s %s", resp.status_code, ach_ref["key"]["href"]) @@ -834,6 +845,7 @@ def _sync_achievement_data(index_data: dict, auth_headers: dict) -> None: ) except (KeyError, TypeError) as exc: logger.warning("Failed to parse achievement %s: %s", ach_ref.get("id"), exc) + logger.info("Achievement scan complete") def _sync_faction_data(index_data: dict) -> None: diff --git a/apicore/views.py b/apicore/views.py index 7dcdd84..ac1f632 100644 --- a/apicore/views.py +++ b/apicore/views.py @@ -738,6 +738,8 @@ def list(self, request): user_id = request.query_params.get("user") alt_name = request.query_params.get("alt", "").title() realm_slug = request.query_params.get("realm", "") + summary = request.query_params.get("summary") + category = request.query_params.get("category") if not user_id: return response.Response([]) @@ -757,12 +759,57 @@ def list(self, request): ) else: alt_ids = ProfileAlt.objects.filter(user=user_id).values_list("alt_id", flat=True) - qs = ( - ProfileAltAchievement.objects.filter(alt__in=alt_ids) - .select_related("alt", "achievement") - .order_by("alt__alt_name", "-completed_timestamp") + qs = ProfileAltAchievement.objects.filter(alt__in=alt_ids).select_related( + "alt", "achievement" + ) + + # Distinct earned achievement IDs for this user — eliminates duplicates + # that arise when the same achievement is stored against multiple alts + # (e.g. old per-alt scans before scan_user_collection was introduced). + earned_ids = qs.values_list("achievement_id", flat=True).distinct() + + if summary: + data = ( + DataAchievement.objects.filter(achievement_id__in=earned_ids) + .values("achievement_category") + .annotate( + count=models.Count("pk"), + points=models.Sum("achievement_points"), + ) + .order_by("achievement_category") + ) + return response.Response( + [ + { + "category_name": r["achievement_category"], + "count": r["count"], + "points": r["points"], + } + for r in data + ] + ) + + if category: + achievements = DataAchievement.objects.filter( + achievement_id__in=earned_ids, + achievement_category=category, + ).order_by("achievement_name") + return response.Response( + [ + { + "achievement": a.achievement_id, + "alt": None, + "alt_name": None, + "achievement_name": a.achievement_name, + "achievement_points": a.achievement_points, + "achievement_category": a.achievement_category, + "completed_timestamp": None, + } + for a in achievements + ] ) + qs = qs.order_by("achievement__achievement_name") serializer = self.get_serializer(qs, many=True) return response.Response(serializer.data) From f1c7062e3eff7d7ce72b3d7123f6311b6d07083d Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 30 Jun 2026 21:06:49 +0100 Subject: [PATCH 10/18] Remove keybind feature, keep generic Lua upload pipeline (#43) Keybind scraping/parsing/display was too niche to maintain. Deletes keybind_builder.py, keybind_mapping.py, and their tests; trims ProfileUserView.list to the generic page=header response plus the parse-and-cache scaffolding (renamed cache key from keybinds:{user} to userfile:{user}) for future addon-only data. tier_sort_key was also used for profession tier sorting, not just keybinds, so it moves to a new apicore/libs/expansion_order.py rather than being deleted. --- CLAUDE.md | 13 +- apicore/libs/expansion_order.py | 23 +++ apicore/libs/keybind_builder.py | 153 ------------------- apicore/libs/keybind_mapping.py | 261 -------------------------------- apicore/views.py | 16 +- readme.md | 2 +- tests/test_keybind_helpers.py | 150 ------------------ tests/test_views.py | 4 +- 8 files changed, 35 insertions(+), 587 deletions(-) create mode 100644 apicore/libs/expansion_order.py delete mode 100644 apicore/libs/keybind_builder.py delete mode 100644 apicore/libs/keybind_mapping.py delete mode 100644 tests/test_keybind_helpers.py diff --git a/CLAUDE.md b/CLAUDE.md index d5f09b2..23f06ec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # FazzToolsAPI -Django REST Framework backend for **FazzTools** — a World of Warcraft companion app. Integrates with the Blizzard Battle.net API to sync character data (professions, equipment, mounts, pets) and parses uploaded WoW Lua addon files to serve keybind data. +Django REST Framework backend for **FazzTools** — a World of Warcraft companion app. Integrates with the Blizzard Battle.net API to sync character data (professions, equipment, mounts, pets) and stores uploaded WoW Lua addon exports for future addon-only data (gold, currencies, lockouts). The companion frontend lives at `../FazzToolsFrontend` (React, port 3000 in dev). @@ -71,13 +71,12 @@ apicore/ The single Django app serializers.py DRF serializers permissions.py IsSessionUser permission class libs/ - keybind_builder.py Pure keybind-building logic (build_all/single_keybinds, tier_sort_key) - keybind_mapping.py Slot→action-button mappings per addon lua_parser.py Hand-rolled Lua-table-to-JSON converter icon_mapping.py Mount/pet icon mappings faction_expansion.py Hardcoded faction_id → expansion name mapping (283 factions) + expansion_order.py tier_sort_key — sorts profession tiers by expansion order migrations/ DB migrations -tests/ pytest suite (47 tests); run via pytest tests/ +tests/ pytest suite (33 tests); run via pytest tests/ conftest.py pytest env-var setup (pytest_configure hook) ``` @@ -141,9 +140,9 @@ Picks the highest-level, highest-ilvl alt per faction (Alliance + Horde) and fet ### Data scan (`fullDataScan` Celery task) Dispatches five independent subtasks: `scanProfessionData`, `scanMountData`, `scanPetData`, `scanAchievementData`, `scanFactionData`. Each can also be triggered individually via its own endpoint. -### Lua keybind file -`ProfileUser.perform_update` validates and stores a `FazzToolsScraper.lua` addon export. -`ProfileUserView.list` with `?page=all` or `?page=single` parses the stored Lua file using the `recursive()` function (a hand-rolled Lua-table-to-JSON converter) and joins results against `ProfileAlt` + Blizzard spell data. Returns per-spec keybind mappings. +### Lua addon file +`ProfileUser.perform_update` validates and stores a `FazzToolsScraper.lua` addon export. +`ProfileUserView.list` with `?page=header` returns the last-update timestamp; the file is parsed via `LuaParser` and cached on read (`userfile:{user_id}`), ready for future addon-only data (gold, currencies, lockouts) to consume — no page handler reads it yet. ## Database tables (all prefixed `ft_`) diff --git a/apicore/libs/expansion_order.py b/apicore/libs/expansion_order.py new file mode 100644 index 0000000..65b5e34 --- /dev/null +++ b/apicore/libs/expansion_order.py @@ -0,0 +1,23 @@ +_EXPANSION_ORDER: dict[str, int] = { + "classic": 0, + "outland": 1, + "northrend": 2, + "cataclysm": 3, + "pandaria": 4, + "draenor": 5, + "legion": 6, + "kul tiran": 7, + "zandalari": 7, + "shadowlands": 8, + "dragon isles": 9, + "khaz algar": 10, + "midnight": 11, +} + + +def tier_sort_key(tier_name: str) -> int: + name_lower = tier_name.lower() + for keyword, order in _EXPANSION_ORDER.items(): + if keyword in name_lower: + return order + return 999 diff --git a/apicore/libs/keybind_builder.py b/apicore/libs/keybind_builder.py deleted file mode 100644 index e171718..0000000 --- a/apicore/libs/keybind_builder.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Pure keybind-building logic, extracted from views.py. - -These functions transform a parsed Lua addon dict into the response shapes -consumed by the frontend. No Django models are touched here except for -_build_all_keybinds, which looks up ProfileAlt for class display names. -""" - -import logging - -from apicore.libs.keybind_mapping import getKeybindMap - -logger = logging.getLogger(__name__) - -_EXPANSION_ORDER: dict[str, int] = { - "classic": 0, - "outland": 1, - "northrend": 2, - "cataclysm": 3, - "pandaria": 4, - "draenor": 5, - "legion": 6, - "kul tiran": 7, - "zandalari": 7, - "shadowlands": 8, - "dragon isles": 9, - "khaz algar": 10, - "midnight": 11, -} - -_SPAM_FILTER = { - "Auto Attack", - "Mobile Banking", - "Revive Battle Pets", - "Vindicaar Matrix Crystal", - "Shoot", -} - -_SECTION_ORDER = {"Base": 0, "Talent": 1, "Misc": 2} - - -def tier_sort_key(tier_name: str) -> int: - name_lower = tier_name.lower() - for keyword, order in _EXPANSION_ORDER.items(): - if keyword in name_lower: - return order - return 999 - - -def build_all_keybinds(data: dict, user_id: str) -> list: - from apicore.models import ProfileAlt - - result = [] - for alt_key, alt_config in data.get("alts", {}).items(): - specs = [] - try: - if alt_config.get("kb") is not None: - specs = list(alt_config["kb"].keys()) - else: - specs = ["---", "---", "---", "---"] - except (KeyError, TypeError): - specs = ["---", "---", "---", "---"] - - specs.sort() - while len(specs) < 4: - specs.append("---") - - name, realm = (alt_key.split("-", 1) + [""])[:2] - try: - alt_obj = ProfileAlt.objects.get(alt_name=name, alt_realm=realm) - row = [name, realm, alt_obj.get_alt_class_display()] + specs - result.append(row) - except ProfileAlt.DoesNotExist: - logger.debug("Alt not in DB: %s", alt_key) - - result.sort(key=lambda x: (x[1], x[0])) - return result - - -def build_single_keybinds(data: dict, alt: str, realm: str, spec: str) -> list: - alt_key = f"{alt}-{realm}" - alt_config = data["alts"][alt_key] - keybind_map = getKeybindMap(alt_config["kbConfig"]["addon"]) - - user_keybind: dict[str, str] = {} - for slot, nice_spell in alt_config["kb"][spec].items(): - prefix = nice_spell.split(":")[0] - - if prefix == "spell": - try: - user_keybind[nice_spell] = alt_config["kbConfig"]["map"][keybind_map[int(slot)]] - except (KeyError, ValueError): - pass - - elif prefix == "macro": - macro_name = nice_spell.split(":")[1] - found = False - for tab in alt_config.get("spell", {}).get(spec, {}): - for spell_id, spell_info in alt_config["spell"][spec][tab].items(): - if spell_info[0] in alt_config["macro"][macro_name][2]: - found = True - spell_key = f"spell:{spell_id}" - try: - bound = alt_config["kbConfig"]["map"][keybind_map[int(slot)]] - except (KeyError, ValueError): - continue - if spell_key not in user_keybind: - user_keybind[spell_key] = bound - elif user_keybind[spell_key] != bound: - user_keybind[spell_key] += f" | {bound}" - if not found: - try: - user_keybind[nice_spell] = alt_config["kbConfig"]["map"][keybind_map[int(slot)]] - except (KeyError, ValueError): - pass - - elif prefix == "item": - item_name = nice_spell.split(":")[1] - if item_name in alt_config.get("item", {}): - try: - user_keybind[nice_spell] = alt_config["kbConfig"]["map"][keybind_map[int(slot)]] - except (KeyError, ValueError): - pass - - full_result = [] - for tab in alt_config.get("spell", {}).get(spec, {}): - spells = [] - for spell_id, spell_info in alt_config["spell"][spec][tab].items(): - if spell_info[0] in _SPAM_FILTER: - continue - entry = [spell_info[0]] - if len(spell_info) > 1: - entry.append(spell_info[1]) - entry.append(user_keybind.get(f"spell:{spell_id}", "UNBOUND")) - spells.append(entry) - spells.sort(key=lambda x: x[0]) - full_result.append([tab.title(), spells]) - - misc = [] - for item_name, item_info in alt_config.get("item", {}).items(): - if f"item:{item_name}" in user_keybind: - misc.append([item_info[0], user_keybind[f"item:{item_name}"]]) - for macro_name, macro_info in alt_config.get("macro", {}).items(): - if f"macro:{macro_name}" in user_keybind: - misc.append([f"[Macro] {macro_info[0]}", user_keybind[f"macro:{macro_name}"]]) - misc.sort(key=lambda x: x[0]) - full_result.append(["Misc", misc]) - - full_result.sort(key=lambda x: _SECTION_ORDER.get(x[0], 99)) - - if len(full_result) >= 2: - full_result[0][1] = [x for x in full_result[0][1] if x not in full_result[1][1]] - - return full_result diff --git a/apicore/libs/keybind_mapping.py b/apicore/libs/keybind_mapping.py deleted file mode 100644 index 34a7ce6..0000000 --- a/apicore/libs/keybind_mapping.py +++ /dev/null @@ -1,261 +0,0 @@ -MAPPING_DEFAULT = { - 1: "ACTIONBUTTON1", - 2: "ACTIONBUTTON2", - 3: "ACTIONBUTTON3", - 4: "ACTIONBUTTON4", - 5: "ACTIONBUTTON5", - 6: "ACTIONBUTTON6", - 7: "ACTIONBUTTON7", - 8: "ACTIONBUTTON8", - 9: "ACTIONBUTTON9", - 10: "ACTIONBUTTON10", - 11: "ACTIONBUTTON11", - 12: "ACTIONBUTTON12", - 13: "ACTIONBUTTON1", - 14: "ACTIONBUTTON2", - 15: "ACTIONBUTTON3", - 16: "ACTIONBUTTON4", - 17: "ACTIONBUTTON5", - 18: "ACTIONBUTTON6", - 19: "ACTIONBUTTON7", - 20: "ACTIONBUTTON8", - 21: "ACTIONBUTTON9", - 22: "ACTIONBUTTON10", - 23: "ACTIONBUTTON11", - 24: "ACTIONBUTTON12", - 25: "MULTIACTIONBAR3BUTTON1", - 26: "MULTIACTIONBAR3BUTTON2", - 27: "MULTIACTIONBAR3BUTTON3", - 28: "MULTIACTIONBAR3BUTTON4", - 29: "MULTIACTIONBAR3BUTTON5", - 30: "MULTIACTIONBAR3BUTTON6", - 31: "MULTIACTIONBAR3BUTTON7", - 32: "MULTIACTIONBAR3BUTTON8", - 33: "MULTIACTIONBAR3BUTTON9", - 34: "MULTIACTIONBAR3BUTTON10", - 35: "MULTIACTIONBAR3BUTTON11", - 36: "MULTIACTIONBAR3BUTTON12", - 37: "MULTIACTIONBAR4BUTTON1", - 38: "MULTIACTIONBAR4BUTTON2", - 39: "MULTIACTIONBAR4BUTTON3", - 40: "MULTIACTIONBAR4BUTTON4", - 41: "MULTIACTIONBAR4BUTTON5", - 42: "MULTIACTIONBAR4BUTTON6", - 43: "MULTIACTIONBAR4BUTTON7", - 44: "MULTIACTIONBAR4BUTTON8", - 45: "MULTIACTIONBAR4BUTTON9", - 46: "MULTIACTIONBAR4BUTTON10", - 47: "MULTIACTIONBAR4BUTTON11", - 48: "MULTIACTIONBAR4BUTTON12", - 49: "MULTIACTIONBAR2BUTTON1", - 50: "MULTIACTIONBAR2BUTTON2", - 51: "MULTIACTIONBAR2BUTTON3", - 52: "MULTIACTIONBAR2BUTTON4", - 53: "MULTIACTIONBAR2BUTTON5", - 54: "MULTIACTIONBAR2BUTTON6", - 55: "MULTIACTIONBAR2BUTTON7", - 56: "MULTIACTIONBAR2BUTTON8", - 57: "MULTIACTIONBAR2BUTTON9", - 58: "MULTIACTIONBAR2BUTTON10", - 59: "MULTIACTIONBAR2BUTTON11", - 60: "MULTIACTIONBAR2BUTTON12", - 61: "MULTIACTIONBAR1BUTTON1", - 62: "MULTIACTIONBAR1BUTTON2", - 63: "MULTIACTIONBAR1BUTTON3", - 64: "MULTIACTIONBAR1BUTTON4", - 65: "MULTIACTIONBAR1BUTTON5", - 66: "MULTIACTIONBAR1BUTTON6", - 67: "MULTIACTIONBAR1BUTTON7", - 68: "MULTIACTIONBAR1BUTTON8", - 69: "MULTIACTIONBAR1BUTTON9", - 70: "MULTIACTIONBAR1BUTTON10", - 71: "MULTIACTIONBAR1BUTTON11", - 72: "MULTIACTIONBAR1BUTTON12", - 73: "ACTIONBUTTON1", - 74: "ACTIONBUTTON2", - 75: "ACTIONBUTTON3", - 76: "ACTIONBUTTON4", - 77: "ACTIONBUTTON5", - 78: "ACTIONBUTTON6", - 79: "ACTIONBUTTON7", - 80: "ACTIONBUTTON8", - 81: "ACTIONBUTTON9", - 82: "ACTIONBUTTON10", - 83: "ACTIONBUTTON11", - 84: "ACTIONBUTTON12", - 85: "ACTIONBUTTON1", - 86: "ACTIONBUTTON2", - 87: "ACTIONBUTTON3", - 88: "ACTIONBUTTON4", - 89: "ACTIONBUTTON5", - 90: "ACTIONBUTTON6", - 91: "ACTIONBUTTON7", - 92: "ACTIONBUTTON8", - 93: "ACTIONBUTTON9", - 94: "ACTIONBUTTON10", - 95: "ACTIONBUTTON11", - 96: "ACTIONBUTTON12", - 97: "ACTIONBUTTON1", - 98: "ACTIONBUTTON2", - 99: "ACTIONBUTTON3", - 100: "ACTIONBUTTON4", - 101: "ACTIONBUTTON5", - 102: "ACTIONBUTTON6", - 103: "ACTIONBUTTON7", - 104: "ACTIONBUTTON8", - 105: "ACTIONBUTTON9", - 106: "ACTIONBUTTON10", - 107: "ACTIONBUTTON11", - 108: "ACTIONBUTTON12", - 109: "ACTIONBUTTON1", - 110: "ACTIONBUTTON2", - 111: "ACTIONBUTTON3", - 112: "ACTIONBUTTON4", - 113: "ACTIONBUTTON5", - 114: "ACTIONBUTTON6", - 115: "ACTIONBUTTON7", - 116: "ACTIONBUTTON8", - 117: "ACTIONBUTTON9", - 118: "ACTIONBUTTON10", - 119: "ACTIONBUTTON11", - 120: "ACTIONBUTTON12", -} - -MAPPING_DOMINOS = { - 1: "ACTIONBUTTON1", - 2: "ACTIONBUTTON2", - 3: "ACTIONBUTTON3", - 4: "ACTIONBUTTON4", - 5: "ACTIONBUTTON5", - 6: "ACTIONBUTTON6", - 7: "ACTIONBUTTON7", - 8: "ACTIONBUTTON8", - 9: "ACTIONBUTTON9", - 10: "ACTIONBUTTON10", - 11: "ACTIONBUTTON11", - 12: "ACTIONBUTTON12", - 13: "CLICK DominosActionButton1:HOTKEY", - 14: "CLICK DominosActionButton2:HOTKEY", - 15: "CLICK DominosActionButton3:HOTKEY", - 16: "CLICK DominosActionButton4:HOTKEY", - 17: "CLICK DominosActionButton5:HOTKEY", - 18: "CLICK DominosActionButton6:HOTKEY", - 19: "CLICK DominosActionButton7:HOTKEY", - 20: "CLICK DominosActionButton8:HOTKEY", - 21: "CLICK DominosActionButton9:HOTKEY", - 22: "CLICK DominosActionButton10:HOTKEY", - 23: "CLICK DominosActionButton11:HOTKEY", - 24: "CLICK DominosActionButton12:HOTKEY", - 25: "MULTIACTIONBAR3BUTTON1", - 26: "MULTIACTIONBAR3BUTTON2", - 27: "MULTIACTIONBAR3BUTTON3", - 28: "MULTIACTIONBAR3BUTTON4", - 29: "MULTIACTIONBAR3BUTTON5", - 30: "MULTIACTIONBAR3BUTTON6", - 31: "MULTIACTIONBAR3BUTTON7", - 32: "MULTIACTIONBAR3BUTTON8", - 33: "MULTIACTIONBAR3BUTTON9", - 34: "MULTIACTIONBAR3BUTTON10", - 35: "MULTIACTIONBAR3BUTTON11", - 36: "MULTIACTIONBAR3BUTTON12", - 37: "MULTIACTIONBAR4BUTTON1", - 38: "MULTIACTIONBAR4BUTTON2", - 39: "MULTIACTIONBAR4BUTTON3", - 40: "MULTIACTIONBAR4BUTTON4", - 41: "MULTIACTIONBAR4BUTTON5", - 42: "MULTIACTIONBAR4BUTTON6", - 43: "MULTIACTIONBAR4BUTTON7", - 44: "MULTIACTIONBAR4BUTTON8", - 45: "MULTIACTIONBAR4BUTTON9", - 46: "MULTIACTIONBAR4BUTTON10", - 47: "MULTIACTIONBAR4BUTTON11", - 48: "MULTIACTIONBAR4BUTTON12", - 49: "MULTIACTIONBAR2BUTTON1", - 50: "MULTIACTIONBAR2BUTTON2", - 51: "MULTIACTIONBAR2BUTTON3", - 52: "MULTIACTIONBAR2BUTTON4", - 53: "MULTIACTIONBAR2BUTTON5", - 54: "MULTIACTIONBAR2BUTTON6", - 55: "MULTIACTIONBAR2BUTTON7", - 56: "MULTIACTIONBAR2BUTTON8", - 57: "MULTIACTIONBAR2BUTTON9", - 58: "MULTIACTIONBAR2BUTTON10", - 59: "MULTIACTIONBAR2BUTTON11", - 60: "MULTIACTIONBAR2BUTTON12", - 61: "MULTIACTIONBAR1BUTTON1", - 62: "MULTIACTIONBAR1BUTTON2", - 63: "MULTIACTIONBAR1BUTTON3", - 64: "MULTIACTIONBAR1BUTTON4", - 65: "MULTIACTIONBAR1BUTTON5", - 66: "MULTIACTIONBAR1BUTTON6", - 67: "MULTIACTIONBAR1BUTTON7", - 68: "MULTIACTIONBAR1BUTTON8", - 69: "MULTIACTIONBAR1BUTTON9", - 70: "MULTIACTIONBAR1BUTTON10", - 71: "MULTIACTIONBAR1BUTTON11", - 72: "MULTIACTIONBAR1BUTTON12", - 73: "CLICK DominosActionButton13:HOTKEY", - 74: "CLICK DominosActionButton14:HOTKEY", - 75: "CLICK DominosActionButton15:HOTKEY", - 76: "CLICK DominosActionButton16:HOTKEY", - 77: "CLICK DominosActionButton17:HOTKEY", - 78: "CLICK DominosActionButton18:HOTKEY", - 79: "CLICK DominosActionButton19:HOTKEY", - 80: "CLICK DominosActionButton20:HOTKEY", - 81: "CLICK DominosActionButton21:HOTKEY", - 82: "CLICK DominosActionButton22:HOTKEY", - 83: "CLICK DominosActionButton23:HOTKEY", - 84: "CLICK DominosActionButton24:HOTKEY", - 85: "CLICK DominosActionButton25:HOTKEY", - 86: "CLICK DominosActionButton26:HOTKEY", - 87: "CLICK DominosActionButton27:HOTKEY", - 88: "CLICK DominosActionButton28:HOTKEY", - 89: "CLICK DominosActionButton29:HOTKEY", - 90: "CLICK DominosActionButton30:HOTKEY", - 91: "CLICK DominosActionButton31:HOTKEY", - 92: "CLICK DominosActionButton32:HOTKEY", - 93: "CLICK DominosActionButton33:HOTKEY", - 94: "CLICK DominosActionButton34:HOTKEY", - 95: "CLICK DominosActionButton35:HOTKEY", - 96: "CLICK DominosActionButton36:HOTKEY", - 97: "CLICK DominosActionButton37:HOTKEY", - 98: "CLICK DominosActionButton38:HOTKEY", - 99: "CLICK DominosActionButton39:HOTKEY", - 100: "CLICK DominosActionButton40:HOTKEY", - 101: "CLICK DominosActionButton41:HOTKEY", - 102: "CLICK DominosActionButton42:HOTKEY", - 103: "CLICK DominosActionButton43:HOTKEY", - 104: "CLICK DominosActionButton44:HOTKEY", - 105: "CLICK DominosActionButton45:HOTKEY", - 106: "CLICK DominosActionButton46:HOTKEY", - 107: "CLICK DominosActionButton47:HOTKEY", - 108: "CLICK DominosActionButton48:HOTKEY", - 109: "CLICK DominosActionButton49:HOTKEY", - 110: "CLICK DominosActionButton50:HOTKEY", - 111: "CLICK DominosActionButton51:HOTKEY", - 112: "CLICK DominosActionButton52:HOTKEY", - 113: "CLICK DominosActionButton53:HOTKEY", - 114: "CLICK DominosActionButton54:HOTKEY", - 115: "CLICK DominosActionButton55:HOTKEY", - 116: "CLICK DominosActionButton56:HOTKEY", - 117: "CLICK DominosActionButton57:HOTKEY", - 118: "CLICK DominosActionButton58:HOTKEY", - 119: "CLICK DominosActionButton59:HOTKEY", - 120: "CLICK DominosActionButton60:HOTKEY", -} - -MAPPING_BARTENDER = None - -MAPPING_ELVUI = None - - -def getKeybindMap(addon): - if addon == "Dominos": - result = MAPPING_DOMINOS - elif addon == "Bartender": - result = MAPPING_BARTENDER - elif addon == "Elvui": - result = MAPPING_ELVUI - else: - result = MAPPING_DEFAULT - return result diff --git a/apicore/views.py b/apicore/views.py index ac1f632..100a154 100644 --- a/apicore/views.py +++ b/apicore/views.py @@ -4,7 +4,6 @@ import logging import os import re -import string import time import environ @@ -18,7 +17,7 @@ from rest_framework.permissions import IsAdminUser from urllib3.util.retry import Retry -from apicore.libs.keybind_builder import build_all_keybinds, build_single_keybinds, tier_sort_key +from apicore.libs.expansion_order import tier_sort_key from apicore.libs.lua_parser import LuaParser from apicore.models import ( DataAchievement, @@ -191,7 +190,7 @@ def perform_update(self, serializer): logger.warning("Could not remove old file: %s", exc) serializer.save(user_id=user_id, user_file=file, user_last_update=update_date) - cache.delete(f"keybinds:{user_id}") + cache.delete(f"userfile:{user_id}") def list(self, request): user_id = request.query_params.get("user") @@ -212,7 +211,7 @@ def list(self, request): if not user_obj.user_file: return response.Response([]) - cache_key = f"keybinds:{user_id}" + cache_key = f"userfile:{user_id}" data = cache.get(cache_key) if data is None: with user_obj.user_file.open("r") as f: @@ -220,15 +219,6 @@ def list(self, request): data = LuaParser(lines).parse() cache.set(cache_key, data, timeout=None) - if page == "all": - return response.Response(build_all_keybinds(data, user_id)) - - if page == "single": - alt_name = request.query_params.get("alt", "").title() - realm = string.capwords(request.query_params.get("realm", "")) - spec = request.query_params.get("spec", "").title() - return response.Response(build_single_keybinds(data, alt_name, realm, spec)) - return response.Response([]) diff --git a/readme.md b/readme.md index 36b59c2..f9dd5d0 100644 --- a/readme.md +++ b/readme.md @@ -9,7 +9,7 @@ Django REST Framework backend for **FazzTools** — a World of Warcraft companion app. -Syncs character data (professions, equipment, mounts, pets) from the Blizzard Battle.net API and parses WoW Lua addon exports to serve keybind data. +Syncs character data (professions, equipment, mounts, pets) from the Blizzard Battle.net API and stores uploaded WoW Lua addon exports for future addon-only data. The companion frontend lives at [FazzToolsFrontend](../FazzToolsFrontend). diff --git a/tests/test_keybind_helpers.py b/tests/test_keybind_helpers.py deleted file mode 100644 index b8bd69a..0000000 --- a/tests/test_keybind_helpers.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Tests for keybind helper functions. - -Covers getKeybindMap dispatch and _build_single_keybinds transformation logic. -No DB access needed — these are pure functions. -""" - -from apicore.libs.keybind_builder import build_single_keybinds -from apicore.libs.keybind_mapping import MAPPING_DEFAULT, MAPPING_DOMINOS, getKeybindMap - -# --------------------------------------------------------------------------- -# getKeybindMap -# --------------------------------------------------------------------------- - - -class TestGetKeybindMap: - def test_dominos_returns_dominos_mapping(self): - assert getKeybindMap("Dominos") is MAPPING_DOMINOS - - def test_unknown_addon_returns_default(self): - assert getKeybindMap("SomeUnknownAddon") is MAPPING_DEFAULT - - def test_empty_string_returns_default(self): - assert getKeybindMap("") is MAPPING_DEFAULT - - def test_bartender_returns_none(self): - # Bartender mapping is not yet implemented — documents the current gap - assert getKeybindMap("Bartender") is None - - def test_default_mapping_slot1_is_actionbutton1(self): - assert MAPPING_DEFAULT[1] == "ACTIONBUTTON1" - - def test_dominos_slot13_is_dominos_button(self): - # Dominos re-maps slots 13+ to its own button names - assert MAPPING_DOMINOS[13] == "CLICK DominosActionButton1:HOTKEY" - - -# --------------------------------------------------------------------------- -# _build_single_keybinds -# --------------------------------------------------------------------------- - -# Minimal data structure matching what the Lua parser produces for a single alt -_BASE_ALT_CONFIG = { - "kbConfig": { - "addon": "Default", - "map": { - "ACTIONBUTTON1": "1", - "ACTIONBUTTON2": "2", - "ACTIONBUTTON3": "E", - }, - }, - "kb": { - "Frost": { - "1": "spell:100", - "2": "spell:200", - "3": "spell:300", - } - }, - "spell": { - "Frost": { - "Base": { - "100": ["Frostbolt"], - "200": ["Ice Lance", "spell_frost_frostlance"], - "300": ["Glacial Spike"], - } - } - }, -} - -_DATA = {"alts": {"Mage-Realm": _BASE_ALT_CONFIG}} - - -class TestBuildSingleKeybinds: - def _build(self, data=None, alt="Mage", realm="Realm", spec="Frost"): - return build_single_keybinds(data or _DATA, alt, realm, spec) - - def test_returns_list_of_sections(self): - result = self._build() - assert isinstance(result, list) - assert all(isinstance(s, list) and len(s) == 2 for s in result) - - def test_bound_spell_has_correct_keybind(self): - result = self._build() - base_section = next(s for s in result if s[0] == "Base") - spells_by_name = {entry[0]: entry for entry in base_section[1]} - assert spells_by_name["Frostbolt"][-1] == "1" - assert spells_by_name["Ice Lance"][-1] == "2" - - def test_spell_with_icon_includes_icon(self): - result = self._build() - base_section = next(s for s in result if s[0] == "Base") - spells_by_name = {entry[0]: entry for entry in base_section[1]} - # Ice Lance has icon data in spell_info - assert spells_by_name["Ice Lance"][1] == "spell_frost_frostlance" - - def test_unbound_slot_shows_unbound(self): - data = { - "alts": { - "Mage-Realm": { - **_BASE_ALT_CONFIG, - "kb": {"Frost": {}}, # no keybinds assigned - } - } - } - result = build_single_keybinds(data, "Mage", "Realm", "Frost") - base_section = next(s for s in result if s[0] == "Base") - assert all(entry[-1] == "UNBOUND" for entry in base_section[1]) - - def test_spells_sorted_alphabetically_within_section(self): - result = self._build() - base_section = next(s for s in result if s[0] == "Base") - names = [entry[0] for entry in base_section[1]] - assert names == sorted(names) - - def test_misc_section_always_present(self): - result = self._build() - section_names = [s[0] for s in result] - assert "Misc" in section_names - - def test_item_binding_appears_in_misc(self): - data = { - "alts": { - "Mage-Realm": { - **_BASE_ALT_CONFIG, - "kb": {"Frost": {"1": "item:Hearthstone"}}, - "spell": {"Frost": {}}, - "item": {"Hearthstone": ["Hearthstone", 6948, "Consumable"]}, - } - } - } - result = build_single_keybinds(data, "Mage", "Realm", "Frost") - misc_section = next(s for s in result if s[0] == "Misc") - misc_names = [entry[0] for entry in misc_section[1]] - assert "Hearthstone" in misc_names - - def test_missing_slot_in_keybind_map_is_silently_skipped(self): - data = { - "alts": { - "Mage-Realm": { - **_BASE_ALT_CONFIG, - "kb": {"Frost": {"999": "spell:100"}}, # slot 999 not in mapping - } - } - } - result = build_single_keybinds(data, "Mage", "Realm", "Frost") - base_section = next(s for s in result if s[0] == "Base") - # Frostbolt should still appear, but as UNBOUND since slot couldn't be mapped - names = [entry[0] for entry in base_section[1]] - assert "Frostbolt" in names - frostbolt = next(e for e in base_section[1] if e[0] == "Frostbolt") - assert frostbolt[-1] == "UNBOUND" diff --git a/tests/test_views.py b/tests/test_views.py index 586ae9d..7553615 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -8,10 +8,10 @@ import pytest from django.test import Client -from apicore.libs.keybind_builder import tier_sort_key +from apicore.libs.expansion_order import tier_sort_key # --------------------------------------------------------------------------- -# Pure helper: _tier_sort_key +# Pure helper: tier_sort_key # --------------------------------------------------------------------------- From 17bfa7ab92eb28e93db8e4ee9acbfb372efd55eb Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 30 Jun 2026 21:08:23 +0100 Subject: [PATCH 11/18] Fix admin session cookie breaking login via CSRF enforcement (#42) A lingering Django admin session (from triggering DataScan via /api/admin/) made DRF's SessionAuthentication enforce CSRF on every API call, since profile/custom endpoints use the custom IsSessionUser permission rather than Django auth. Removed SessionAuthentication from the default authenticators and scoped it explicitly to the IsAdminUser-gated DataScan* viewsets that actually need it. --- apicore/views.py | 7 +++++++ backend/settings.py | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/apicore/views.py b/apicore/views.py index 100a154..48bd05e 100644 --- a/apicore/views.py +++ b/apicore/views.py @@ -13,6 +13,7 @@ from django.utils import timezone from requests.adapters import HTTPAdapter from rest_framework import response, viewsets +from rest_framework.authentication import SessionAuthentication from rest_framework.exceptions import PermissionDenied from rest_framework.permissions import IsAdminUser from urllib3.util.retry import Retry @@ -859,6 +860,7 @@ def create(self, request): class DataScan(viewsets.ViewSet): + authentication_classes = [SessionAuthentication] permission_classes = [IsAdminUser] def create(self, request): @@ -867,6 +869,7 @@ def create(self, request): class DataScanProfessions(viewsets.ViewSet): + authentication_classes = [SessionAuthentication] permission_classes = [IsAdminUser] def create(self, request): @@ -875,6 +878,7 @@ def create(self, request): class DataScanMounts(viewsets.ViewSet): + authentication_classes = [SessionAuthentication] permission_classes = [IsAdminUser] def create(self, request): @@ -883,6 +887,7 @@ def create(self, request): class DataScanPets(viewsets.ViewSet): + authentication_classes = [SessionAuthentication] permission_classes = [IsAdminUser] def create(self, request): @@ -891,6 +896,7 @@ def create(self, request): class DataScanAchievements(viewsets.ViewSet): + authentication_classes = [SessionAuthentication] permission_classes = [IsAdminUser] def create(self, request): @@ -899,6 +905,7 @@ def create(self, request): class DataScanFactions(viewsets.ViewSet): + authentication_classes = [SessionAuthentication] permission_classes = [IsAdminUser] def create(self, request): diff --git a/backend/settings.py b/backend/settings.py index 1f7d0d0..5fa0e97 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -116,6 +116,11 @@ REST_FRAMEWORK = { "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination", "PAGE_SIZE": 100, + # Profile/custom endpoints use the custom IsSessionUser permission, not Django auth, + # so they don't need an authenticator. Leaving SessionAuthentication as the default + # meant any lingering Django admin session cookie (from /api/admin/) made DRF enforce + # CSRF on every API call, breaking bnetlogin and other endpoints for logged-in admins. + "DEFAULT_AUTHENTICATION_CLASSES": [], } CORS_ALLOWED_ORIGINS = [ From 3c3ff77bca2c2a5a05f23947bafdc8b299d8c0e3 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 30 Jun 2026 21:22:18 +0100 Subject: [PATCH 12/18] Add Phase 2b: Mythic+ rating tracking (#44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds DataMythicDungeon (static catalog), ProfileAltMythicPlus (per-alt season rating summary), and ProfileAltMythicPlusDungeon (per-alt per-dungeon best run) with migration 0015. scanMythicDungeonData populates the dungeon catalog via /data/wow/mythic-keystone/dungeon/index (namespace dynamic-eu, not static-eu) and is added to fullDataScan. _sync_mythic_plus is wired into scan_single_alt: the main mythic-keystone-profile endpoint only lists season refs with no "current" flag, so max(season.id) picks the current one, then a second call to the season-specific endpoint fetches mythic_rating and best_runs. Verified against a live character that best_runs can list two entries per dungeon (best-timed vs best-overall, same map_rating but different keystone_level) — deduped to the higher level per dungeon before writing. New endpoints: altmythicplus, altmythicplusdungeons, mythicdungeons, datascan/mythicdungeons. --- CLAUDE.md | 11 ++- ...icdungeon_profilealtmythicplus_and_more.py | 53 ++++++++++ apicore/models.py | 43 +++++++++ apicore/serializers.py | 33 +++++++ apicore/tasks.py | 96 +++++++++++++++++++ apicore/views.py | 91 ++++++++++++++++++ backend/urls.py | 4 + 7 files changed, 328 insertions(+), 3 deletions(-) create mode 100644 apicore/migrations/0015_datamythicdungeon_profilealtmythicplus_and_more.py diff --git a/CLAUDE.md b/CLAUDE.md index 23f06ec..7e05e2a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -98,6 +98,8 @@ conftest.py pytest env-var setup (pytest_configure hook) - `usermounts` / `userpets` — Collected mounts/pets per user - `altachievements` — `ProfileAltAchievement`: Achievement completions per alt - `altreputations` — `ProfileAltReputation`: Faction standing per alt +- `altmythicplus` — `ProfileAltMythicPlus`: Current-season M+ rating summary per alt +- `altmythicplusdungeons` — `ProfileAltMythicPlusDungeon`: Best run per dungeon per alt ### Data endpoints (static WoW data, synced via DataScan task) - `professions`, `professiontiers`, `professionrecipes`, `reagents`, `recipereagents` @@ -105,6 +107,7 @@ conftest.py pytest env-var setup (pytest_configure hook) - `mounts`, `pets` - `achievements` — `DataAchievement`: All WoW achievements (name, points, category) - `factions` — `DataFaction`: All WoW reputation factions +- `mythicdungeons` — `DataMythicDungeon`: All Mythic+ dungeons (current and historical) ### Custom endpoints - `POST /api/custom/bnetlogin/` — Battle.net OAuth2 callback; creates/updates user and syncs alts @@ -116,6 +119,7 @@ conftest.py pytest env-var setup (pytest_configure hook) - `POST /api/custom/datascan/pets/` — Triggers pet data scan only - `POST /api/custom/datascan/achievements/` — Triggers achievement data scan only - `POST /api/custom/datascan/factions/` — Triggers faction data scan only +- `POST /api/custom/datascan/mythicdungeons/` — Triggers Mythic+ dungeon catalog scan only ## Key data flows @@ -130,6 +134,7 @@ Dispatches two sets of tasks in parallel: 2. `/professions` → upserts `ProfileAltProfession` + `ProfileAltProfessionData` 3. `/equipment` → upserts `ProfileAltEquipment` + `DataEquipment` / `DataEquipmentVariant` 4. `/reputations` → upserts `ProfileAltReputation` per faction +5. `/mythic-keystone-profile` → upserts `ProfileAltMythicPlus` + `ProfileAltMythicPlusDungeon`. The main endpoint only lists season refs (no rating/runs) — the season id isn't flagged as "current" anywhere, so `max(season.id)` is used to pick it, then a second call to `/mythic-keystone-profile/season/{id}` fetches `mythic_rating` and `best_runs`. Blizzard can list two `best_runs` entries per dungeon (best-timed and best-overall, same `map_rating` but different `keystone_level`) — the sync keeps the higher level. **Per-user** (`scan_user_collection` × 1): Picks the highest-level, highest-ilvl alt per faction (Alliance + Horde) and fetches: @@ -138,7 +143,7 @@ Picks the highest-level, highest-ilvl alt per faction (Alliance + Horde) and fet - `/achievements` → upserts `ProfileAltAchievement` for that representative alt ### Data scan (`fullDataScan` Celery task) -Dispatches five independent subtasks: `scanProfessionData`, `scanMountData`, `scanPetData`, `scanAchievementData`, `scanFactionData`. Each can also be triggered individually via its own endpoint. +Dispatches six independent subtasks: `scanProfessionData`, `scanMountData`, `scanPetData`, `scanAchievementData`, `scanFactionData`, `scanMythicDungeonData`. Each can also be triggered individually via its own endpoint. The dungeon index requires `namespace=dynamic-eu` (not `static-eu` like other catalogs). ### Lua addon file `ProfileUser.perform_update` validates and stores a `FazzToolsScraper.lua` addon export. @@ -146,9 +151,9 @@ Dispatches five independent subtasks: `scanProfessionData`, `scanMountData`, `sc ## Database tables (all prefixed `ft_`) -**Data (static):** `ft_data_profession`, `ft_data_professiontier`, `ft_data_professionrecipe`, `ft_data_reagent`, `ft_data_recipereagent`, `ft_data_equipment`, `ft_data_equipmentvariant`, `ft_data_mount`, `ft_data_pet`, `ft_data_achievement`, `ft_data_faction` +**Data (static):** `ft_data_profession`, `ft_data_professiontier`, `ft_data_professionrecipe`, `ft_data_reagent`, `ft_data_recipereagent`, `ft_data_equipment`, `ft_data_equipmentvariant`, `ft_data_mount`, `ft_data_pet`, `ft_data_achievement`, `ft_data_faction`, `ft_data_mythicdungeon` -**Profile (user):** `ft_profile_user`, `ft_profile_alt`, `ft_profile_altprofession`, `ft_profile_altprofessiondata`, `ft_profile_altequipment`, `ft_profile_usermount`, `ft_profile_userpet`, `ft_profile_altachievement`, `ft_profile_altreputation` +**Profile (user):** `ft_profile_user`, `ft_profile_alt`, `ft_profile_altprofession`, `ft_profile_altprofessiondata`, `ft_profile_altequipment`, `ft_profile_usermount`, `ft_profile_userpet`, `ft_profile_altachievement`, `ft_profile_altreputation`, `ft_profile_altmythicplus`, `ft_profile_altmythicplusdungeon` ## Things to know diff --git a/apicore/migrations/0015_datamythicdungeon_profilealtmythicplus_and_more.py b/apicore/migrations/0015_datamythicdungeon_profilealtmythicplus_and_more.py new file mode 100644 index 0000000..dd8cdab --- /dev/null +++ b/apicore/migrations/0015_datamythicdungeon_profilealtmythicplus_and_more.py @@ -0,0 +1,53 @@ +# Generated by Django 5.2 on 2026-06-30 20:14 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('apicore', '0014_add_faction_category'), + ] + + operations = [ + migrations.CreateModel( + name='DataMythicDungeon', + fields=[ + ('dungeon_id', models.PositiveIntegerField(primary_key=True, serialize=False)), + ('dungeon_name', models.CharField(max_length=256)), + ], + options={ + 'db_table': 'ft_data_mythicdungeon', + }, + ), + migrations.CreateModel( + name='ProfileAltMythicPlus', + fields=[ + ('alt', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='apicore.profilealt')), + ('season_id', models.PositiveIntegerField()), + ('mythic_rating', models.FloatField(default=0)), + ('alt_mythicplus_expiry_date', models.DateTimeField()), + ], + options={ + 'db_table': 'ft_profile_altmythicplus', + }, + ), + migrations.CreateModel( + name='ProfileAltMythicPlusDungeon', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('keystone_level', models.PositiveSmallIntegerField(default=0)), + ('score', models.FloatField(default=0)), + ('completed_timestamp', models.DateTimeField(blank=True, null=True)), + ('is_completed_within_time', models.BooleanField(default=False)), + ('alt_mythicplusdungeon_expiry_date', models.DateTimeField()), + ('alt', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='apicore.profilealtmythicplus')), + ('dungeon', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='apicore.datamythicdungeon')), + ], + options={ + 'db_table': 'ft_profile_altmythicplusdungeon', + 'constraints': [models.UniqueConstraint(fields=('alt', 'dungeon'), name='unique_altmythicplusdungeon')], + }, + ), + ] diff --git a/apicore/models.py b/apicore/models.py index df741c1..573262a 100644 --- a/apicore/models.py +++ b/apicore/models.py @@ -369,3 +369,46 @@ class Meta: def __str__(self): return f"{self.alt} - {self.faction}" + + +class DataMythicDungeon(models.Model): + dungeon_id = models.PositiveIntegerField(primary_key=True) + dungeon_name = models.CharField(max_length=256) + + class Meta: + db_table = "ft_data_mythicdungeon" + + def __str__(self): + return f"{self.dungeon_id} - {self.dungeon_name}" + + +class ProfileAltMythicPlus(models.Model): + alt = models.OneToOneField(ProfileAlt, on_delete=models.CASCADE, primary_key=True) + season_id = models.PositiveIntegerField() + mythic_rating = models.FloatField(default=0) + alt_mythicplus_expiry_date = models.DateTimeField() + + class Meta: + db_table = "ft_profile_altmythicplus" + + def __str__(self): + return f"{self.alt.alt_name} - {self.alt.alt_realm}" + + +class ProfileAltMythicPlusDungeon(models.Model): + alt = models.ForeignKey(ProfileAltMythicPlus, on_delete=models.CASCADE) + dungeon = models.ForeignKey(DataMythicDungeon, on_delete=models.CASCADE) + keystone_level = models.PositiveSmallIntegerField(default=0) + score = models.FloatField(default=0) + completed_timestamp = models.DateTimeField(null=True, blank=True) + is_completed_within_time = models.BooleanField(default=False) + alt_mythicplusdungeon_expiry_date = models.DateTimeField() + + class Meta: + db_table = "ft_profile_altmythicplusdungeon" + constraints = [ + models.UniqueConstraint(fields=["alt", "dungeon"], name="unique_altmythicplusdungeon") + ] + + def __str__(self): + return f"{self.alt} - {self.dungeon}" diff --git a/apicore/serializers.py b/apicore/serializers.py index d0bdb1d..4cb0c27 100644 --- a/apicore/serializers.py +++ b/apicore/serializers.py @@ -6,6 +6,7 @@ DataEquipmentVariant, DataFaction, DataMount, + DataMythicDungeon, DataPet, DataProfession, DataProfessionRecipe, @@ -15,6 +16,8 @@ ProfileAlt, ProfileAltAchievement, ProfileAltEquipment, + ProfileAltMythicPlus, + ProfileAltMythicPlusDungeon, ProfileAltProfession, ProfileAltProfessionData, ProfileAltReputation, @@ -255,3 +258,33 @@ class Meta: "standing_type", "standing_value", ) + + +class DataMythicDungeonSerializer(serializers.ModelSerializer): + class Meta: + model = DataMythicDungeon + fields = ("dungeon_id", "dungeon_name") + + +class ProfileAltMythicPlusSerializer(serializers.ModelSerializer): + alt_name = serializers.ReadOnlyField(source="alt.alt_name") + + class Meta: + model = ProfileAltMythicPlus + fields = ("alt", "alt_name", "season_id", "mythic_rating") + + +class ProfileAltMythicPlusDungeonSerializer(serializers.ModelSerializer): + dungeon_name = serializers.ReadOnlyField(source="dungeon.dungeon_name") + + class Meta: + model = ProfileAltMythicPlusDungeon + fields = ( + "alt", + "dungeon", + "dungeon_name", + "keystone_level", + "score", + "completed_timestamp", + "is_completed_within_time", + ) diff --git a/apicore/tasks.py b/apicore/tasks.py index 0281fdb..1eeed6e 100644 --- a/apicore/tasks.py +++ b/apicore/tasks.py @@ -16,6 +16,7 @@ DataEquipmentVariant, DataFaction, DataMount, + DataMythicDungeon, DataPet, DataProfession, DataProfessionRecipe, @@ -25,6 +26,8 @@ ProfileAlt, ProfileAltAchievement, ProfileAltEquipment, + ProfileAltMythicPlus, + ProfileAltMythicPlusDungeon, ProfileAltProfession, ProfileAltProfessionData, ProfileAltReputation, @@ -39,6 +42,7 @@ _EU_API_BASE = "https://eu.api.blizzard.com" _PROFILE_PARAMS = {"namespace": "profile-eu", "locale": "en_US"} _STATIC_PARAMS = {"namespace": "static-eu", "locale": "en_US"} +_DYNAMIC_PARAMS = {"namespace": "dynamic-eu", "locale": "en_US"} _LOCALE_PARAMS = {"locale": "en_US"} _EQUIPMENT_SLOTS = [ @@ -178,6 +182,8 @@ def scan_single_alt(self, alt_id: int, user_id: str, token: str) -> None: elif key == "reputations": _sync_reputations(alt, resp.json()) + _sync_mythic_plus(alt, char_base, auth_headers) + logger.info("Completed alt scan: %s-%s", alt.alt_name, alt.alt_realm_slug) except Exception as exc: logger.error( @@ -427,6 +433,7 @@ def fullDataScan(client: str, secret: str) -> str: scanPetData.delay(client, secret) scanAchievementData.delay(client, secret) scanFactionData.delay(client, secret) + scanMythicDungeonData.delay(client, secret) return "Dispatched all data scans" @@ -492,6 +499,20 @@ def scanFactionData(client: str, secret: str) -> str: return "Done" +@shared_task +def scanMythicDungeonData(client: str, secret: str) -> str: + token = _fetch_token(client, secret) + auth_headers = {"Authorization": f"Bearer {token}"} + resp = _api_get( + f"{_EU_API_BASE}/data/wow/mythic-keystone/dungeon/index", _DYNAMIC_PARAMS, auth_headers + ) + if resp.status_code != 200: + logger.error("Blizzard API %s mythic-keystone/dungeon/index", resp.status_code) + return "Failed" + _sync_mythic_dungeon_data(resp.json()) + return "Done" + + def _sync_profession_data(index_data: dict, auth_headers: dict) -> None: for profession_ref in index_data.get("professions", []): resp = _api_get(profession_ref["key"]["href"], _STATIC_PARAMS, auth_headers) @@ -817,6 +838,70 @@ def _sync_reputations(alt: ProfileAlt, data: dict) -> None: logger.warning("Failed to sync reputation for %s: %s", alt.alt_name, exc) +def _sync_mythic_plus(alt: ProfileAlt, char_base: str, auth_headers: dict) -> None: + resp = _api_get(f"{char_base}/mythic-keystone-profile", _PROFILE_PARAMS, auth_headers) + if resp.status_code != 200: + return + seasons = resp.json().get("seasons", []) + if not seasons: + return + current_season_id = max(s["id"] for s in seasons) + + season_resp = _api_get( + f"{char_base}/mythic-keystone-profile/season/{current_season_id}", + _PROFILE_PARAMS, + auth_headers, + ) + if season_resp.status_code != 200: + logger.warning( + "Blizzard API %s mythic-keystone-profile/season/%s", + season_resp.status_code, + current_season_id, + ) + return + + data = season_resp.json() + expiry = timezone.now() + datetime.timedelta(days=30) + mp_record, _ = ProfileAltMythicPlus.objects.update_or_create( + alt=alt, + defaults={ + "season_id": data.get("season", {}).get("id", current_season_id), + "mythic_rating": data.get("mythic_rating", {}).get("rating", 0), + "alt_mythicplus_expiry_date": expiry, + }, + ) + + # Blizzard can list two entries per dungeon (best-timed and best-overall) sharing the + # same map_rating but different keystone_level — keep the highest level per dungeon. + best_run_by_dungeon: dict[int, dict] = {} + for run in data.get("best_runs", []): + dungeon_id = run.get("dungeon", {}).get("id") + if dungeon_id is None: + continue + existing = best_run_by_dungeon.get(dungeon_id) + if existing is None or run.get("keystone_level", 0) > existing.get("keystone_level", 0): + best_run_by_dungeon[dungeon_id] = run + + for dungeon_id, run in best_run_by_dungeon.items(): + try: + dungeon = DataMythicDungeon.objects.get(dungeon_id=dungeon_id) + except DataMythicDungeon.DoesNotExist: + continue + ts = run.get("completed_timestamp") + completed = timezone.datetime.fromtimestamp(ts / 1000, tz=datetime.UTC) if ts else None + ProfileAltMythicPlusDungeon.objects.update_or_create( + alt=mp_record, + dungeon=dungeon, + defaults={ + "keystone_level": run.get("keystone_level", 0), + "score": run.get("map_rating", {}).get("rating", 0), + "completed_timestamp": completed, + "is_completed_within_time": run.get("is_completed_within_time", False), + "alt_mythicplusdungeon_expiry_date": expiry, + }, + ) + + # --------------------------------------------------------------------------- # Static data sync helpers — achievements + factions # --------------------------------------------------------------------------- @@ -857,3 +942,14 @@ def _sync_faction_data(index_data: dict) -> None: ) except (KeyError, TypeError) as exc: logger.warning("Failed to parse faction %s: %s", faction_ref.get("id"), exc) + + +def _sync_mythic_dungeon_data(index_data: dict) -> None: + for dungeon_ref in index_data.get("dungeons", []): + try: + DataMythicDungeon.objects.get_or_create( + dungeon_id=dungeon_ref["id"], + defaults={"dungeon_name": dungeon_ref["name"]}, + ) + except (KeyError, TypeError) as exc: + logger.warning("Failed to parse dungeon %s: %s", dungeon_ref.get("id"), exc) diff --git a/apicore/views.py b/apicore/views.py index 48bd05e..c027021 100644 --- a/apicore/views.py +++ b/apicore/views.py @@ -26,6 +26,7 @@ DataEquipmentVariant, DataFaction, DataMount, + DataMythicDungeon, DataPet, DataProfession, DataProfessionRecipe, @@ -35,6 +36,8 @@ ProfileAlt, ProfileAltAchievement, ProfileAltEquipment, + ProfileAltMythicPlus, + ProfileAltMythicPlusDungeon, ProfileAltProfession, ProfileAltProfessionData, ProfileAltReputation, @@ -49,6 +52,7 @@ DataEquipmentVariantSerializer, DataFactionSerializer, DataMountSerializer, + DataMythicDungeonSerializer, DataPetSerializer, DataProfessionRecipeSerializer, DataProfessionSerializer, @@ -57,6 +61,8 @@ DataRecipeReagentSerializer, ProfileAltAchievementSerializer, ProfileAltEquipmentSerializer, + ProfileAltMythicPlusDungeonSerializer, + ProfileAltMythicPlusSerializer, ProfileAltProfessionDataSerializer, ProfileAltProfessionSerializer, ProfileAltReputationSerializer, @@ -71,6 +77,7 @@ scanAchievementData, scanFactionData, scanMountData, + scanMythicDungeonData, scanPetData, scanProfessionData, ) @@ -721,6 +728,11 @@ class DataFactionView(viewsets.ModelViewSet): queryset = DataFaction.objects.all() +class DataMythicDungeonView(viewsets.ModelViewSet): + serializer_class = DataMythicDungeonSerializer + queryset = DataMythicDungeon.objects.all() + + class ProfileAltAchievementView(viewsets.ModelViewSet): serializer_class = ProfileAltAchievementSerializer queryset = ProfileAltAchievement.objects.all() @@ -842,6 +854,76 @@ def list(self, request): return response.Response(serializer.data) +class ProfileAltMythicPlusView(viewsets.ModelViewSet): + serializer_class = ProfileAltMythicPlusSerializer + queryset = ProfileAltMythicPlus.objects.all() + + def list(self, request): + user_id = request.query_params.get("user") + alt_name = request.query_params.get("alt", "").title() + realm_slug = request.query_params.get("realm", "") + + if not user_id: + return response.Response([]) + if request.session.get("user_id") != user_id: + return response.Response([], status=403) + + if alt_name and realm_slug: + alt = ProfileAlt.objects.filter( + alt_name=alt_name, alt_realm_slug=realm_slug, user=user_id + ).first() + if not alt: + return response.Response([]) + qs = ProfileAltMythicPlus.objects.filter(alt=alt) + else: + alt_ids = ProfileAlt.objects.filter(user=user_id).values_list("alt_id", flat=True) + qs = ( + ProfileAltMythicPlus.objects.filter(alt__in=alt_ids) + .select_related("alt") + .order_by("-mythic_rating") + ) + + serializer = self.get_serializer(qs, many=True) + return response.Response(serializer.data) + + +class ProfileAltMythicPlusDungeonView(viewsets.ModelViewSet): + serializer_class = ProfileAltMythicPlusDungeonSerializer + queryset = ProfileAltMythicPlusDungeon.objects.all() + + def list(self, request): + user_id = request.query_params.get("user") + alt_name = request.query_params.get("alt", "").title() + realm_slug = request.query_params.get("realm", "") + + if not user_id: + return response.Response([]) + if request.session.get("user_id") != user_id: + return response.Response([], status=403) + + if alt_name and realm_slug: + alt = ProfileAlt.objects.filter( + alt_name=alt_name, alt_realm_slug=realm_slug, user=user_id + ).first() + if not alt: + return response.Response([]) + qs = ( + ProfileAltMythicPlusDungeon.objects.filter(alt__alt=alt) + .select_related("dungeon") + .order_by("-score") + ) + else: + alt_ids = ProfileAlt.objects.filter(user=user_id).values_list("alt_id", flat=True) + qs = ( + ProfileAltMythicPlusDungeon.objects.filter(alt__alt__in=alt_ids) + .select_related("alt", "dungeon") + .order_by("alt__alt__alt_name", "-score") + ) + + serializer = self.get_serializer(qs, many=True) + return response.Response(serializer.data) + + class ScanAlt(viewsets.ViewSet): def create(self, request): user_id = request.data.get("userid") @@ -911,3 +993,12 @@ class DataScanFactions(viewsets.ViewSet): def create(self, request): scanFactionData.delay(BLIZZ_CLIENT, BLIZZ_SECRET) return response.Response("Faction scan started") + + +class DataScanMythicDungeons(viewsets.ViewSet): + authentication_classes = [SessionAuthentication] + permission_classes = [IsAdminUser] + + def create(self, request): + scanMythicDungeonData.delay(BLIZZ_CLIENT, BLIZZ_SECRET) + return response.Response("Mythic dungeon scan started") diff --git a/backend/urls.py b/backend/urls.py index b4e3866..efa70b5 100644 --- a/backend/urls.py +++ b/backend/urls.py @@ -31,6 +31,8 @@ profile.register(r"altequipments", views.ProfileAltEquipmentView) profile.register(r"altachievements", views.ProfileAltAchievementView) profile.register(r"altreputations", views.ProfileAltReputationView) +profile.register(r"altmythicplus", views.ProfileAltMythicPlusView) +profile.register(r"altmythicplusdungeons", views.ProfileAltMythicPlusDungeonView) data = routers.DefaultRouter() data.register(r"professions", views.DataProfessionView) @@ -44,6 +46,7 @@ data.register(r"pets", views.DataPetView) data.register(r"achievements", views.DataAchievementView) data.register(r"factions", views.DataFactionView) +data.register(r"mythicdungeons", views.DataMythicDungeonView) custom = routers.DefaultRouter() custom.register(r"bnetlogin", views.BnetLogin, "bnetlogin") @@ -55,6 +58,7 @@ custom.register(r"datascan/pets", views.DataScanPets, "datascanpets") custom.register(r"datascan/achievements", views.DataScanAchievements, "datscanachievements") custom.register(r"datascan/factions", views.DataScanFactions, "datascanfactions") +custom.register(r"datascan/mythicdungeons", views.DataScanMythicDungeons, "datascanmythicdungeons") # custom.register(r'fileupload', views.FileUpload, 'fileupload') urlpatterns = [ From 5fc5355dda605a40199a076b300539bcbaf501d5 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 30 Jun 2026 21:59:41 +0100 Subject: [PATCH 13/18] Schedule fullDataScan weekly via Celery Beat (#45) Runs every Sunday 03:00 UTC alongside the existing daily purge_stale_profiles schedule. The admin-triggered datascan endpoints are kept for ad-hoc manual scans. --- CLAUDE.md | 4 ++++ backend/settings.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 7e05e2a..ccd2cfc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -145,6 +145,10 @@ Picks the highest-level, highest-ilvl alt per faction (Alliance + Horde) and fet ### Data scan (`fullDataScan` Celery task) Dispatches six independent subtasks: `scanProfessionData`, `scanMountData`, `scanPetData`, `scanAchievementData`, `scanFactionData`, `scanMythicDungeonData`. Each can also be triggered individually via its own endpoint. The dungeon index requires `namespace=dynamic-eu` (not `static-eu` like other catalogs). +### Scheduled tasks (`CELERY_BEAT_SCHEDULE` in `settings.py`) +- `purge_stale_profiles` — daily, deletes expired profile records. +- `fullDataScan` — weekly, Sunday 03:00 UTC, with `BLIZZ_CLIENT`/`BLIZZ_SECRET` baked into the schedule args at startup. The admin-triggered `/api/custom/datascan/` endpoints still work for ad-hoc/manual scans (e.g. testing a single category). + ### Lua addon file `ProfileUser.perform_update` validates and stores a `FazzToolsScraper.lua` addon export. `ProfileUserView.list` with `?page=header` returns the last-update timestamp; the file is parsed via `LuaParser` and cached on read (`userfile:{user_id}`), ready for future addon-only data (gold, currencies, lockouts) to consume — no page handler reads it yet. diff --git a/backend/settings.py b/backend/settings.py index 5fa0e97..43f4a1c 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -1,6 +1,7 @@ from pathlib import Path import environ +from celery.schedules import crontab env = environ.Env() environ.Env.read_env() @@ -111,6 +112,11 @@ "task": "apicore.tasks.purge_stale_profiles", "schedule": 86400, }, + "full-data-scan-weekly": { + "task": "apicore.tasks.fullDataScan", + "schedule": crontab(day_of_week="sunday", hour=3, minute=0), + "args": (env("BLIZZ_CLIENT"), env("BLIZZ_SECRET")), + }, } REST_FRAMEWORK = { From 47a4f791629cae7fc3b61168c16bcfc469ebe16e Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 30 Jun 2026 23:34:43 +0100 Subject: [PATCH 14/18] Serve gold and played time from cached addon file (#46) --- apicore/views.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/apicore/views.py b/apicore/views.py index c027021..b48bbd4 100644 --- a/apicore/views.py +++ b/apicore/views.py @@ -227,6 +227,21 @@ def list(self, request): data = LuaParser(lines).parse() cache.set(cache_key, data, timeout=None) + if page == "addon": + addon_alts = data.get("alts", {}) + result = [] + for alt in ProfileAlt.objects.filter(user=user_id): + addon_alt = addon_alts.get(f"{alt.alt_name}-{alt.alt_realm}", {}) + result.append( + { + "alt_id": alt.alt_id, + "gold": addon_alt.get("gold"), + "played_time_total": addon_alt.get("playedTimeTotal"), + "played_time_level": addon_alt.get("playedTimeLevel"), + } + ) + return response.Response(result) + return response.Response([]) From 6700fe3c9dcdaa591f4fc361959a877a373366c5 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 30 Jun 2026 23:58:34 +0100 Subject: [PATCH 15/18] Fix 500 on .lua file upload from closed-file write (#47) perform_update closed the uploaded file's stream via a context manager before handing it to the storage backend, which then failed trying to re-read it. Validate and normalise the content directly, then save it as a fresh ContentFile instead of reusing the original upload stream. --- apicore/views.py | 19 ++++++++----------- tests/test_views.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/apicore/views.py b/apicore/views.py index b48bbd4..1cca3d0 100644 --- a/apicore/views.py +++ b/apicore/views.py @@ -9,6 +9,7 @@ import environ import requests from django.core.cache import cache +from django.core.files.base import ContentFile from django.db import models from django.utils import timezone from requests.adapters import HTTPAdapter @@ -175,18 +176,14 @@ def perform_update(self, serializer): logger.warning("Rejected upload: file too large (%d bytes)", file.size) return - file.name = user_id + ".lua" - with file.open("r+") as f: - content = f.read().decode("utf-8") + content = file.read().decode("utf-8") - if "FazzToolsScraperDB" not in content[0:25]: - logger.warning("Rejected upload: invalid file header") - return + if "FazzToolsScraperDB" not in content[0:25]: + logger.warning("Rejected upload: invalid file header") + return - normalised = re.sub(r'(\r\n|\r|\n)(?=(?:[^"]*"[^"]*")*[^"]*$)', r"\n", content) - f.seek(0) - f.write(normalised.encode()) - f.truncate() + normalised = re.sub(r'(\r\n|\r|\n)(?=(?:[^"]*"[^"]*")*[^"]*$)', r"\n", content) + normalised_file = ContentFile(normalised.encode(), name=user_id + ".lua") user_obj = ProfileUser.objects.get(user_id=user_id) update_date = user_obj.user_last_update @@ -197,7 +194,7 @@ def perform_update(self, serializer): except OSError as exc: logger.warning("Could not remove old file: %s", exc) - serializer.save(user_id=user_id, user_file=file, user_last_update=update_date) + serializer.save(user_id=user_id, user_file=normalised_file, user_last_update=update_date) cache.delete(f"userfile:{user_id}") def list(self, request): diff --git a/tests/test_views.py b/tests/test_views.py index 7553615..0b21c33 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -6,9 +6,13 @@ """ import pytest +from django.core.files.uploadedfile import SimpleUploadedFile from django.test import Client +from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart +from django.utils import timezone from apicore.libs.expansion_order import tier_sort_key +from apicore.models import ProfileUser # --------------------------------------------------------------------------- # Pure helper: tier_sort_key @@ -124,6 +128,34 @@ def test_header_page_for_unknown_user_returns_empty(self): assert resp.status_code == 200 assert resp.json() == [] + def test_upload_does_not_500(self): + """Regression test: uploads used to crash perform_update with 'I/O operation + on closed file' because a `with file.open("r+")` block closed the upload + stream's underlying BytesIO before it was handed to the storage backend — + Django's InMemoryUploadedFile doesn't override File.close().""" + user_id = "u1" + ProfileUser.objects.create(user_id=user_id, user_file="", user_last_update=timezone.now()) + c = self._authed_client(user_id) + + body = "FazzToolsScraperDB = {\n}\n" + upload = SimpleUploadedFile( + "FazzToolsScraper.lua", body.encode(), content_type="text/plain" + ) + + resp = c.put( + f"/api/profile/users/{user_id}/", + data=encode_multipart( + BOUNDARY, + { + "user_id": user_id, + "user_file": upload, + "user_last_update": timezone.now().isoformat(), + }, + ), + content_type=MULTIPART_CONTENT, + ) + assert resp.status_code == 200 + # --------------------------------------------------------------------------- # ProfileAltView — IsSessionUser enforcement From 593e2cc4dcdd0f48beb1cabd1d8a8a1aa430bc4a Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 1 Jul 2026 00:02:51 +0100 Subject: [PATCH 16/18] Fix 500 reading uploaded addon file in text mode (#48) ProfileUserView.list opened the stored .lua file with mode "r" (text), so readlines() already returned str, and the subsequent .decode("utf-8") call crashed. This path was previously unreachable in practice (no page read the file before page=addon was added) so the bug went unnoticed. --- apicore/views.py | 2 +- tests/test_views.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/apicore/views.py b/apicore/views.py index 1cca3d0..47f8da8 100644 --- a/apicore/views.py +++ b/apicore/views.py @@ -219,7 +219,7 @@ def list(self, request): cache_key = f"userfile:{user_id}" data = cache.get(cache_key) if data is None: - with user_obj.user_file.open("r") as f: + with user_obj.user_file.open("rb") as f: lines = [line.decode("utf-8") for line in f.readlines()] data = LuaParser(lines).parse() cache.set(cache_key, data, timeout=None) diff --git a/tests/test_views.py b/tests/test_views.py index 0b21c33..8d10494 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -156,6 +156,45 @@ def test_upload_does_not_500(self): ) assert resp.status_code == 200 + def test_addon_page_reads_uploaded_file(self): + """Regression test: page=addon (and any non-header page reading the + uploaded file) used to crash with 'str' object has no attribute 'decode' + because the file was opened in text mode ("r") despite the parser + expecting bytes to decode itself.""" + user_id = "u1" + ProfileUser.objects.create(user_id=user_id, user_file="", user_last_update=timezone.now()) + c = self._authed_client(user_id) + + body = ( + "FazzToolsScraperDB = {\n" + "preamble\n" + '["alts"] = {\n' + '["Testchar-Realm"] = {\n' + '["gold"] = 12345,\n' + "},\n" + "},\n" + "}\n" + ) + upload = SimpleUploadedFile( + "FazzToolsScraper.lua", body.encode(), content_type="text/plain" + ) + c.put( + f"/api/profile/users/{user_id}/", + data=encode_multipart( + BOUNDARY, + { + "user_id": user_id, + "user_file": upload, + "user_last_update": timezone.now().isoformat(), + }, + ), + content_type=MULTIPART_CONTENT, + ) + + resp = c.get(f"/api/profile/users/?user={user_id}&page=addon") + assert resp.status_code == 200 + assert resp.json() == [] + # --------------------------------------------------------------------------- # ProfileAltView — IsSessionUser enforcement From 4b48b5a1cbe04ee16cf9f01f607c98be681cbdc8 Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 1 Jul 2026 00:10:26 +0100 Subject: [PATCH 17/18] Parse addon gold/played-time into a DB table on upload (#49) Replaces the page=addon read-through cache (which re-parsed the raw .lua file on every cache miss, with no expiry) with parse-on-upload into a new ProfileAltAddonData model, matching how every other Profile* model is populated. Adds the altaddondata endpoint and drops the now-dead generic file-read branch from ProfileUserView.list. --- CLAUDE.md | 6 +- .../migrations/0016_profilealtaddondata.py | 26 +++++++ apicore/models.py | 16 +++++ apicore/serializers.py | 9 +++ apicore/views.py | 71 +++++++++++-------- backend/urls.py | 1 + tests/test_views.py | 60 +++++++++++++--- 7 files changed, 150 insertions(+), 39 deletions(-) create mode 100644 apicore/migrations/0016_profilealtaddondata.py diff --git a/CLAUDE.md b/CLAUDE.md index ccd2cfc..10f5586 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -100,6 +100,7 @@ conftest.py pytest env-var setup (pytest_configure hook) - `altreputations` — `ProfileAltReputation`: Faction standing per alt - `altmythicplus` — `ProfileAltMythicPlus`: Current-season M+ rating summary per alt - `altmythicplusdungeons` — `ProfileAltMythicPlusDungeon`: Best run per dungeon per alt +- `altaddondata` — `ProfileAltAddonData`: Gold + played time per alt, parsed from the uploaded `.lua` file (addon-only, no Blizzard API equivalent) ### Data endpoints (static WoW data, synced via DataScan task) - `professions`, `professiontiers`, `professionrecipes`, `reagents`, `recipereagents` @@ -150,14 +151,13 @@ Dispatches six independent subtasks: `scanProfessionData`, `scanMountData`, `sca - `fullDataScan` — weekly, Sunday 03:00 UTC, with `BLIZZ_CLIENT`/`BLIZZ_SECRET` baked into the schedule args at startup. The admin-triggered `/api/custom/datascan/` endpoints still work for ad-hoc/manual scans (e.g. testing a single category). ### Lua addon file -`ProfileUser.perform_update` validates and stores a `FazzToolsScraper.lua` addon export. -`ProfileUserView.list` with `?page=header` returns the last-update timestamp; the file is parsed via `LuaParser` and cached on read (`userfile:{user_id}`), ready for future addon-only data (gold, currencies, lockouts) to consume — no page handler reads it yet. +`ProfileUser.perform_update` validates and stores a `FazzToolsScraper.lua` addon export, then parses it via `LuaParser` and upserts `ProfileAltAddonData` for each alt found in the file (matched by `f"{alt.alt_name}-{alt.alt_realm}"`, the display name + display realm key `core.lua` writes). This is parse-on-upload, not parse-on-read — there's no Celery sync task for this data since it only ever exists in the addon export, never the Blizzard API, so the upload itself is the sync point. A failed parse logs a warning and leaves the stored file/timestamp update intact rather than failing the upload. Currencies/lockouts/keystone/vault are captured by the addon but not yet parsed into a model — same pattern, not built. ## Database tables (all prefixed `ft_`) **Data (static):** `ft_data_profession`, `ft_data_professiontier`, `ft_data_professionrecipe`, `ft_data_reagent`, `ft_data_recipereagent`, `ft_data_equipment`, `ft_data_equipmentvariant`, `ft_data_mount`, `ft_data_pet`, `ft_data_achievement`, `ft_data_faction`, `ft_data_mythicdungeon` -**Profile (user):** `ft_profile_user`, `ft_profile_alt`, `ft_profile_altprofession`, `ft_profile_altprofessiondata`, `ft_profile_altequipment`, `ft_profile_usermount`, `ft_profile_userpet`, `ft_profile_altachievement`, `ft_profile_altreputation`, `ft_profile_altmythicplus`, `ft_profile_altmythicplusdungeon` +**Profile (user):** `ft_profile_user`, `ft_profile_alt`, `ft_profile_altprofession`, `ft_profile_altprofessiondata`, `ft_profile_altequipment`, `ft_profile_usermount`, `ft_profile_userpet`, `ft_profile_altachievement`, `ft_profile_altreputation`, `ft_profile_altmythicplus`, `ft_profile_altmythicplusdungeon`, `ft_profile_altaddondata` ## Things to know diff --git a/apicore/migrations/0016_profilealtaddondata.py b/apicore/migrations/0016_profilealtaddondata.py new file mode 100644 index 0000000..c208aa1 --- /dev/null +++ b/apicore/migrations/0016_profilealtaddondata.py @@ -0,0 +1,26 @@ +# Generated by Django 5.2 on 2026-06-30 23:06 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('apicore', '0015_datamythicdungeon_profilealtmythicplus_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='ProfileAltAddonData', + fields=[ + ('alt', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='apicore.profilealt')), + ('gold', models.PositiveBigIntegerField(default=0)), + ('played_time_total', models.PositiveIntegerField(default=0)), + ('played_time_level', models.PositiveIntegerField(default=0)), + ], + options={ + 'db_table': 'ft_profile_altaddondata', + }, + ), + ] diff --git a/apicore/models.py b/apicore/models.py index 573262a..ae47491 100644 --- a/apicore/models.py +++ b/apicore/models.py @@ -412,3 +412,19 @@ class Meta: def __str__(self): return f"{self.alt} - {self.dungeon}" + + +class ProfileAltAddonData(models.Model): + """Per-alt data that only exists in the uploaded addon export (no Blizzard API + equivalent), parsed and upserted from the .lua file on upload.""" + + alt = models.OneToOneField(ProfileAlt, on_delete=models.CASCADE, primary_key=True) + gold = models.PositiveBigIntegerField(default=0) + played_time_total = models.PositiveIntegerField(default=0) + played_time_level = models.PositiveIntegerField(default=0) + + class Meta: + db_table = "ft_profile_altaddondata" + + def __str__(self): + return f"{self.alt.alt_name} - {self.alt.alt_realm}" diff --git a/apicore/serializers.py b/apicore/serializers.py index 4cb0c27..59f8054 100644 --- a/apicore/serializers.py +++ b/apicore/serializers.py @@ -15,6 +15,7 @@ DataRecipeReagent, ProfileAlt, ProfileAltAchievement, + ProfileAltAddonData, ProfileAltEquipment, ProfileAltMythicPlus, ProfileAltMythicPlusDungeon, @@ -288,3 +289,11 @@ class Meta: "completed_timestamp", "is_completed_within_time", ) + + +class ProfileAltAddonDataSerializer(serializers.ModelSerializer): + alt_id = serializers.ReadOnlyField(source="alt.alt_id") + + class Meta: + model = ProfileAltAddonData + fields = ("alt_id", "gold", "played_time_total", "played_time_level") diff --git a/apicore/views.py b/apicore/views.py index 47f8da8..6729bc1 100644 --- a/apicore/views.py +++ b/apicore/views.py @@ -8,7 +8,6 @@ import environ import requests -from django.core.cache import cache from django.core.files.base import ContentFile from django.db import models from django.utils import timezone @@ -36,6 +35,7 @@ DataRecipeReagent, ProfileAlt, ProfileAltAchievement, + ProfileAltAddonData, ProfileAltEquipment, ProfileAltMythicPlus, ProfileAltMythicPlusDungeon, @@ -61,6 +61,7 @@ DataReagentSerializer, DataRecipeReagentSerializer, ProfileAltAchievementSerializer, + ProfileAltAddonDataSerializer, ProfileAltEquipmentSerializer, ProfileAltMythicPlusDungeonSerializer, ProfileAltMythicPlusSerializer, @@ -195,7 +196,29 @@ def perform_update(self, serializer): logger.warning("Could not remove old file: %s", exc) serializer.save(user_id=user_id, user_file=normalised_file, user_last_update=update_date) - cache.delete(f"userfile:{user_id}") + self._sync_addon_data(user_id, normalised) + + @staticmethod + def _sync_addon_data(user_id, normalised_content): + try: + parsed = LuaParser(normalised_content.splitlines(keepends=True)).parse() + except (ValueError, IndexError) as exc: + logger.warning("Could not parse addon file for %s: %s", user_id, exc) + return + + addon_alts = parsed.get("alts", {}) + for alt in ProfileAlt.objects.filter(user=user_id): + addon_alt = addon_alts.get(f"{alt.alt_name}-{alt.alt_realm}") + if addon_alt is None: + continue + ProfileAltAddonData.objects.update_or_create( + alt=alt, + defaults={ + "gold": addon_alt.get("gold") or 0, + "played_time_total": addon_alt.get("playedTimeTotal") or 0, + "played_time_level": addon_alt.get("playedTimeLevel") or 0, + }, + ) def list(self, request): user_id = request.query_params.get("user") @@ -213,32 +236,6 @@ def list(self, request): ts = time.mktime(user_obj.user_last_update.timetuple()) * 1000 return response.Response([ts]) - if not user_obj.user_file: - return response.Response([]) - - cache_key = f"userfile:{user_id}" - data = cache.get(cache_key) - if data is None: - with user_obj.user_file.open("rb") as f: - lines = [line.decode("utf-8") for line in f.readlines()] - data = LuaParser(lines).parse() - cache.set(cache_key, data, timeout=None) - - if page == "addon": - addon_alts = data.get("alts", {}) - result = [] - for alt in ProfileAlt.objects.filter(user=user_id): - addon_alt = addon_alts.get(f"{alt.alt_name}-{alt.alt_realm}", {}) - result.append( - { - "alt_id": alt.alt_id, - "gold": addon_alt.get("gold"), - "played_time_total": addon_alt.get("playedTimeTotal"), - "played_time_level": addon_alt.get("playedTimeLevel"), - } - ) - return response.Response(result) - return response.Response([]) @@ -936,6 +933,24 @@ def list(self, request): return response.Response(serializer.data) +class ProfileAltAddonDataView(viewsets.ModelViewSet): + serializer_class = ProfileAltAddonDataSerializer + queryset = ProfileAltAddonData.objects.all() + + def list(self, request): + user_id = request.query_params.get("user") + + if not user_id: + return response.Response([]) + if request.session.get("user_id") != user_id: + return response.Response([], status=403) + + alt_ids = ProfileAlt.objects.filter(user=user_id).values_list("alt_id", flat=True) + qs = ProfileAltAddonData.objects.filter(alt__in=alt_ids).select_related("alt") + serializer = self.get_serializer(qs, many=True) + return response.Response(serializer.data) + + class ScanAlt(viewsets.ViewSet): def create(self, request): user_id = request.data.get("userid") diff --git a/backend/urls.py b/backend/urls.py index efa70b5..4493e8c 100644 --- a/backend/urls.py +++ b/backend/urls.py @@ -33,6 +33,7 @@ profile.register(r"altreputations", views.ProfileAltReputationView) profile.register(r"altmythicplus", views.ProfileAltMythicPlusView) profile.register(r"altmythicplusdungeons", views.ProfileAltMythicPlusDungeonView) +profile.register(r"altaddondata", views.ProfileAltAddonDataView) data = routers.DefaultRouter() data.register(r"professions", views.DataProfessionView) diff --git a/tests/test_views.py b/tests/test_views.py index 8d10494..ed6c28a 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -12,7 +12,7 @@ from django.utils import timezone from apicore.libs.expansion_order import tier_sort_key -from apicore.models import ProfileUser +from apicore.models import ProfileAlt, ProfileAltAddonData, ProfileUser # --------------------------------------------------------------------------- # Pure helper: tier_sort_key @@ -156,13 +156,26 @@ def test_upload_does_not_500(self): ) assert resp.status_code == 200 - def test_addon_page_reads_uploaded_file(self): - """Regression test: page=addon (and any non-header page reading the - uploaded file) used to crash with 'str' object has no attribute 'decode' - because the file was opened in text mode ("r") despite the parser - expecting bytes to decode itself.""" + def test_upload_syncs_addon_data_for_matching_alts(self): + """Gold/played-time are parsed from the uploaded file and upserted into + ProfileAltAddonData at upload time, not re-parsed on every read.""" user_id = "u1" ProfileUser.objects.create(user_id=user_id, user_file="", user_last_update=timezone.now()) + ProfileAlt.objects.create( + alt_id=1, + alt_account_id=1, + alt_level=80, + alt_name="Testchar", + alt_realm="Realm", + alt_realm_id=1, + alt_realm_slug="realm", + alt_class=1, + alt_race=1, + alt_gender="Male", + alt_faction="Alliance", + alt_expiry_date=timezone.now(), + user_id=user_id, + ) c = self._authed_client(user_id) body = ( @@ -171,6 +184,8 @@ def test_addon_page_reads_uploaded_file(self): '["alts"] = {\n' '["Testchar-Realm"] = {\n' '["gold"] = 12345,\n' + '["playedTimeTotal"] = 6000,\n' + '["playedTimeLevel"] = 100,\n' "},\n" "},\n" "}\n" @@ -178,7 +193,7 @@ def test_addon_page_reads_uploaded_file(self): upload = SimpleUploadedFile( "FazzToolsScraper.lua", body.encode(), content_type="text/plain" ) - c.put( + resp = c.put( f"/api/profile/users/{user_id}/", data=encode_multipart( BOUNDARY, @@ -190,8 +205,37 @@ def test_addon_page_reads_uploaded_file(self): ), content_type=MULTIPART_CONTENT, ) + assert resp.status_code == 200 + + addon_data = ProfileAltAddonData.objects.get(alt_id=1) + assert addon_data.gold == 12345 + assert addon_data.played_time_total == 6000 + assert addon_data.played_time_level == 100 + + +# --------------------------------------------------------------------------- +# ProfileAltAddonDataView — IsSessionUser enforcement +# --------------------------------------------------------------------------- + - resp = c.get(f"/api/profile/users/?user={user_id}&page=addon") +@pytest.mark.django_db +class TestProfileAltAddonDataView: + def test_no_session_returns_403(self, client: Client): + resp = client.get("/api/profile/altaddondata/?user=u1") + assert resp.status_code == 403 + + def test_wrong_session_returns_403(self, client: Client): + session = client.session + session["user_id"] = "other" + session.save() + resp = client.get("/api/profile/altaddondata/?user=u1") + assert resp.status_code == 403 + + def test_correct_session_returns_200(self, client: Client): + session = client.session + session["user_id"] = "u1" + session.save() + resp = client.get("/api/profile/altaddondata/?user=u1") assert resp.status_code == 200 assert resp.json() == [] From 8072ed0a5f60ec3f1f957eb9d10780a73d1aeff5 Mon Sep 17 00:00:00 2001 From: Joe Farrelly Date: Wed, 1 Jul 2026 12:31:56 +0100 Subject: [PATCH 18/18] Remove Codecov integration --- .github/workflows/lint.yml | 6 ------ pyproject.toml | 2 +- readme.md | 1 - 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2eebd56..72b5e78 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -55,9 +55,3 @@ jobs: BLIZZ_SECRET: ci-secret BLIZZ_REDIRECT_URI: http://localhost:3000/redirect/ run: python -m pytest tests/ -v - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 - with: - files: coverage.xml - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index 08f1dc1..ac216b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ known-first-party = ["apicore"] [tool.pytest.ini_options] DJANGO_SETTINGS_MODULE = "backend.test_settings" python_files = ["tests/test_*.py"] -addopts = "--cov=apicore --cov-report=term-missing --cov-report=xml" +addopts = "--cov=apicore --cov-report=term-missing" [tool.coverage.run] omit = ["*/migrations/*", "*/libs/icon_mapping.py", "*/libs/mount_icons.py", "*/libs/race_mapping.py"] diff --git a/readme.md b/readme.md index f9dd5d0..f1fcd47 100644 --- a/readme.md +++ b/readme.md @@ -2,7 +2,6 @@ [![Deploy](https://github.com/joefarrelly/FazzToolsAPI/actions/workflows/deploy.yml/badge.svg)](https://github.com/joefarrelly/FazzToolsAPI/actions/workflows/deploy.yml) [![Lint and Test](https://github.com/joefarrelly/FazzToolsAPI/actions/workflows/lint.yml/badge.svg)](https://github.com/joefarrelly/FazzToolsAPI/actions/workflows/lint.yml) -[![codecov](https://codecov.io/gh/joefarrelly/FazzToolsAPI/graph/badge.svg)](https://codecov.io/gh/joefarrelly/FazzToolsAPI) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) **Suite:** [Backend](https://github.com/joefarrelly/FazzToolsAPI) · [Frontend](https://github.com/joefarrelly/FazzToolsFrontend) · [Addon](https://github.com/joefarrelly/FazzToolsScraper)