From 62c71b9db4b5fa8b95ac62013854ea4369df854e Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Tue, 16 Jun 2026 10:31:24 +0800 Subject: [PATCH 01/32] fix: stabilize douyin api task output and utf8 keywords --- api/login.py | 498 +++++++++++++++ api/main.py | 485 +++++++++----- api/routes.py | 700 +++++++++++++++++++++ api/tasks.py | 608 ++++++++++++++++++ api/ws.py | 68 ++ cmd_arg/arg.py | 14 + config/base_config.py | 17 +- douyin_scraper/__init__.py | 55 ++ douyin_scraper/config.py | 233 +++++++ douyin_scraper/core.py | 1155 ++++++++++++++++++++++++++++++++++ douyin_scraper/exceptions.py | 80 +++ douyin_scraper/models.py | 115 ++++ douyin_scraper/state.py | 456 ++++++++++++++ douyin_scraper/utils.py | 505 +++++++++++++++ tools/async_file_writer.py | 7 + 15 files changed, 4824 insertions(+), 172 deletions(-) create mode 100644 api/login.py create mode 100644 api/routes.py create mode 100644 api/tasks.py create mode 100644 api/ws.py create mode 100644 douyin_scraper/__init__.py create mode 100644 douyin_scraper/config.py create mode 100644 douyin_scraper/core.py create mode 100644 douyin_scraper/exceptions.py create mode 100644 douyin_scraper/models.py create mode 100644 douyin_scraper/state.py create mode 100644 douyin_scraper/utils.py diff --git a/api/login.py b/api/login.py new file mode 100644 index 000000000..f637c1859 --- /dev/null +++ b/api/login.py @@ -0,0 +1,498 @@ +""" +api.login — 抖音扫码登录路由 +============================== +实现二维码登录流程，供前端 Web UI 调用。 + +端点： + POST /login/qrcode/start — 启动浏览器，返回二维码图片（base64） + GET /login/qrcode/status/{sid} — 轮询登录状态 + POST /login/logout — 登出（清除 Cookie） + GET /login/status — 查询当前登录状态 + +设计要点： + - 使用 playwright.async_api 异步操作，兼容 FastAPI async 路由 + - session 超时 120 秒自动清理 + - 最多 1 个并发登录 session（避免资源竞争） + - Cookie 持久化到 /app/data/douyin_cookie.txt + - 全局内存 Cookie 缓存，供采集任务复用 +""" + +import asyncio +import base64 +import logging +import os +import time as _time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, Optional + +from fastapi import APIRouter, HTTPException + +logger = logging.getLogger("douyin_scraper.api.login") + +router = APIRouter(prefix="/login", tags=["login"]) + +# ═══════════════════════════════════════════════════════════════ +# 常量与持久化路径 +# ═══════════════════════════════════════════════════════════════ + +COOKIE_FILE = Path("/app/data/douyin_cookie.txt") +SESSION_TIMEOUT_SECONDS = 120 + +# Chromium 可执行路径（Docker 容器内由环境变量指定） +CHROMIUM_EXECUTABLE = os.environ.get( + "PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "" +) + +# ═══════════════════════════════════════════════════════════════ +# 全局状态 +# ═══════════════════════════════════════════════════════════════ + +# 内存中缓存的 Cookie 字符串（复用给采集任务） +_memory_cookie: str = "" + +# 当前活跃的登录 session（最多 1 个） +# 结构: {session_id: {"playwright": ..., "browser": ..., "context": ..., "page": ..., "expires_at": datetime, "status": str}} +_active_sessions: Dict[str, Dict[str, Any]] = {} + +# 全局锁，防止并发创建 session +_session_lock = asyncio.Lock() + + +# ═══════════════════════════════════════════════════════════════ +# 内部辅助函数 +# ═══════════════════════════════════════════════════════════════ + +def _load_cookie_from_file() -> str: + """从持久化文件读取 Cookie，失败返回空字符串。""" + try: + if COOKIE_FILE.exists(): + content = COOKIE_FILE.read_text(encoding="utf-8").strip() + if content: + return content + except OSError as exc: + logger.warning("读取 Cookie 文件失败: %s", exc) + return "" + + +def _save_cookie_to_file(cookie_str: str) -> None: + """将 Cookie 字符串写入持久化文件，自动创建父目录。""" + try: + COOKIE_FILE.parent.mkdir(parents=True, exist_ok=True) + COOKIE_FILE.write_text(cookie_str, encoding="utf-8") + logger.info("Cookie 已写入 %s (长度: %d)", COOKIE_FILE, len(cookie_str)) + except OSError as exc: + logger.error("写入 Cookie 文件失败: %s", exc) + + +def _sync_cookie_to_browser_data(cookie_str: str) -> None: + """ + ★ Fix 6: 将 Cookie 同步写入 browser_data 目录 ★ + 供 Playwright persistent context 使用，确保子进程采集任务 + 能够使用登录 session 中获取的 Cookie。 + 写入 Netscape 格式的 Cookie 文件，兼容 Playwright。 + """ + try: + browser_data_dir = Path("/app/browser_data/dy_user_data_dir") + browser_data_dir.mkdir(parents=True, exist_ok=True) + # 将 Cookie 以 Netscape 格式写入（Playwright 兼容） + cookie_path = browser_data_dir / "Cookies" + lines = ["# Netscape HTTP Cookie File\n"] + # 解析 key=value; key=value 格式为 Netscape 行 + for pair in cookie_str.split("; "): + if "=" not in pair: + continue + name, _, value = pair.partition("=") + if not name.strip(): + continue + # Netscape Cookie 格式：domain\tflag\tpath\tsecure\texpiration\tname\tvalue + # 使用 .douyin.com 作为 domain，使 Playwright 在访问抖音时生效 + line = f".douyin.com\tTRUE\t/\tFALSE\t{int(_time.time()) + 86400}\t{name.strip()}\t{value.strip()}\n" + lines.append(line) + cookie_path.write_text("".join(lines), encoding="utf-8") + logger.info("Cookie 已同步到 browser_data: %s (%d 条)", browser_data_dir, len(lines) - 1) + except Exception as exc: + logger.warning("写入 browser_data 失败（非阻塞）: %s", exc) + + +def _delete_cookie_file() -> None: + """删除持久化 Cookie 文件。""" + try: + if COOKIE_FILE.exists(): + COOKIE_FILE.unlink() + logger.info("Cookie 文件已删除: %s", COOKIE_FILE) + except OSError as exc: + logger.warning("删除 Cookie 文件失败: %s", exc) + + +async def _close_session(session_id: str) -> None: + """关闭并清理指定 session 占用的 Playwright 资源。""" + session = _active_sessions.pop(session_id, None) + if session is None: + return + try: + browser = session.get("browser") + if browser: + await browser.close() + pw = session.get("playwright") + if pw: + await pw.stop() + logger.info("[session=%s] 浏览器资源已释放", session_id) + except Exception as exc: + logger.warning("[session=%s] 释放资源时出错: %s", session_id, exc) + + +async def _session_watchdog(session_id: str, timeout: float) -> None: + """ + 后台看门狗：等待 timeout 秒后，若 session 仍存在则强制清理。 + 防止用户忘记扫码导致浏览器进程泄漏。 + """ + await asyncio.sleep(timeout) + if session_id in _active_sessions: + logger.info("[session=%s] 超时 %ds，自动清理", session_id, int(timeout)) + _active_sessions[session_id]["status"] = "expired" + await _close_session(session_id) + + +async def _extract_cookies_from_context(context: Any) -> str: + """ + 从 Playwright BrowserContext 中提取抖音相关 Cookie， + 返回 `key=value; key=value` 格式字符串。 + """ + cookies = await context.cookies( + urls=["https://www.douyin.com", "https://www.tiktok.com"] + ) + if not cookies: + # 回退：提取全部 cookies + cookies = await context.cookies() + cookie_str = "; ".join( + f"{c['name']}={c['value']}" for c in cookies if c.get("name") + ) + return cookie_str + + +async def _check_login_state(page: Any, context: Any) -> bool: + """ + 检测抖音是否已完成扫码登录。 + 检测优先级： + 1. localStorage.HasUserLogin === "1" + 2. Cookie: LOGIN_STATUS === "1" + """ + # 检查 localStorage + try: + local_storage: Dict[str, Any] = await page.evaluate( + "() => { try { return Object.fromEntries(Object.entries(window.localStorage)); } catch(e) { return {}; } }" + ) + if isinstance(local_storage, dict) and local_storage.get("HasUserLogin") == "1": + return True + except Exception as exc: + logger.debug("localStorage 检查失败: %s", exc) + + # 检查 Cookie + try: + cookies = await context.cookies(urls=["https://www.douyin.com"]) + for c in cookies: + if c.get("name") == "LOGIN_STATUS" and c.get("value") == "1": + return True + except Exception as exc: + logger.debug("Cookie 检查失败: %s", exc) + + return False + + +# ═══════════════════════════════════════════════════════════════ +# 路由端点 +# ═══════════════════════════════════════════════════════════════ + + +@router.post("/qrcode/start", summary="启动扫码登录，返回二维码图片") +async def start_qrcode_login() -> Dict[str, Any]: + """ + 启动 Playwright 无头浏览器，打开抖音并截取登录二维码。 + + 返回： + - session_id: 用于后续轮询的唯一标识 + - qrcode: base64 编码的二维码 PNG 图片（data URL 格式） + - expires_at: session 过期时间（ISO 8601） + """ + global _active_sessions + + async with _session_lock: + # 最多允许 1 个活跃 session + if _active_sessions: + old_sid = next(iter(_active_sessions)) + logger.info("已有活跃 session [%s]，先关闭再新建", old_sid) + await _close_session(old_sid) + + session_id = str(uuid.uuid4()) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TIMEOUT_SECONDS) + + logger.info("[session=%s] 启动扫码登录流程", session_id) + + try: + from playwright.async_api import async_playwright + + pw = await async_playwright().start() + + # 构建浏览器启动参数 + launch_kwargs: Dict[str, Any] = { + "headless": True, + "args": [ + "--no-sandbox", + "--disable-dev-shm-usage", + "--disable-gpu", + "--disable-setuid-sandbox", + "--no-first-run", + "--no-zygote", + "--disable-extensions", + ], + } + if CHROMIUM_EXECUTABLE: + launch_kwargs["executable_path"] = CHROMIUM_EXECUTABLE + logger.info("使用指定 Chromium: %s", CHROMIUM_EXECUTABLE) + + browser = await pw.chromium.launch(**launch_kwargs) + context = await browser.new_context( + viewport={"width": 1280, "height": 900}, + user_agent=( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ), + locale="zh-CN", + timezone_id="Asia/Shanghai", + ) + page = await context.new_page() + + # 导航到抖音主页 + logger.info("[session=%s] 打开 https://www.douyin.com", session_id) + await page.goto("https://www.douyin.com", wait_until="domcontentloaded", timeout=30000) + await asyncio.sleep(2) + + # 触发登录弹窗：等待弹窗自动出现，否则手动点击"登录"按钮 + dialog_selector = "xpath=//div[@id='login-panel-new']" + try: + await page.wait_for_selector(dialog_selector, timeout=10000) + logger.info("[session=%s] 登录弹窗已自动弹出", session_id) + except Exception: + logger.info("[session=%s] 登录弹窗未自动弹出，尝试手动点击", session_id) + try: + login_btn = page.locator("xpath=//p[text()='登录']") + await login_btn.click(timeout=5000) + await asyncio.sleep(1) + await page.wait_for_selector(dialog_selector, timeout=8000) + logger.info("[session=%s] 手动触发登录弹窗成功", session_id) + except Exception as click_exc: + logger.warning("[session=%s] 手动点击登录失败: %s，继续尝试截图", session_id, click_exc) + + # 等待并截取二维码 + qrcode_b64: str = "" + qrcode_selector = "xpath=//div[@id='animate_qrcode_container']//img" + try: + qrcode_locator = page.locator(qrcode_selector) + await qrcode_locator.wait_for(timeout=15000) + qrcode_element = await qrcode_locator.element_handle() + if qrcode_element: + screenshot_bytes = await qrcode_element.screenshot() + qrcode_b64 = base64.b64encode(screenshot_bytes).decode("utf-8") + logger.info("[session=%s] 二维码截取成功 (%d bytes)", session_id, len(screenshot_bytes)) + except Exception as qr_exc: + logger.warning( + "[session=%s] 未找到二维码元素 (%s)，降级为全页面截图", + session_id, qr_exc, + ) + try: + full_screenshot = await page.screenshot(full_page=False) + qrcode_b64 = base64.b64encode(full_screenshot).decode("utf-8") + logger.info("[session=%s] 全页面截图完成 (%d bytes)", session_id, len(full_screenshot)) + except Exception as ss_exc: + logger.error("[session=%s] 全页面截图也失败: %s", session_id, ss_exc) + await browser.close() + await pw.stop() + raise HTTPException( + status_code=500, + detail=f"无法获取二维码图片: {ss_exc}", + ) + + # 存储 session + _active_sessions[session_id] = { + "playwright": pw, + "browser": browser, + "context": context, + "page": page, + "expires_at": expires_at, + "status": "pending", + } + + # 启动看门狗 + asyncio.create_task(_session_watchdog(session_id, SESSION_TIMEOUT_SECONDS)) + + return { + "session_id": session_id, + "qrcode": f"data:image/png;base64,{qrcode_b64}", + "expires_at": expires_at.isoformat(), + "message": "请使用抖音 App 扫描二维码登录", + } + + except HTTPException: + raise + except Exception as exc: + logger.error("[session=%s] 启动登录流程失败: %s", session_id, exc, exc_info=True) + # 清理残留资源 + _active_sessions.pop(session_id, None) + raise HTTPException( + status_code=500, + detail=f"启动浏览器失败: {exc}", + ) + + +@router.get("/qrcode/status/{session_id}", summary="轮询扫码登录状态") +async def poll_qrcode_status(session_id: str) -> Dict[str, Any]: + """ + 轮询扫码登录状态。 + + 返回 status： + - pending: 等待扫码 + - scanned: 已扫码，等待确认 + - success: 登录成功（Cookie 已保存） + - expired: session 已过期 + - not_found: session 不存在 + """ + global _memory_cookie + + session = _active_sessions.get(session_id) + if session is None: + # 检查是否之前已成功（session 被清理） + return {"status": "not_found", "message": "session 不存在或已过期"} + + # 检查超时 + if datetime.now(timezone.utc) > session["expires_at"]: + session["status"] = "expired" + await _close_session(session_id) + return {"status": "expired", "message": "二维码已过期，请重新获取"} + + current_status = session.get("status", "pending") + if current_status in ("expired", "success"): + return {"status": current_status} + + page = session.get("page") + context = session.get("context") + + # 检测登录状态 + try: + logged_in = await _check_login_state(page, context) + except Exception as exc: + logger.warning("[session=%s] 检测登录状态失败: %s", session_id, exc) + logged_in = False + + if logged_in: + logger.info("[session=%s] 检测到登录成功！", session_id) + session["status"] = "success" + + # 提取 Cookie + try: + cookie_str = await _extract_cookies_from_context(context) + if cookie_str: + _memory_cookie = cookie_str + _save_cookie_to_file(cookie_str) + # ★ Fix 6: 同步 Cookie 到 browser_data 目录 ★ + _sync_cookie_to_browser_data(cookie_str) + logger.info("[session=%s] Cookie 已保存，长度: %d", session_id, len(cookie_str)) + else: + logger.warning("[session=%s] 登录成功但 Cookie 为空", session_id) + except Exception as exc: + logger.error("[session=%s] 提取 Cookie 失败: %s", session_id, exc) + + # 关闭浏览器 session + await _close_session(session_id) + + return { + "status": "success", + "message": "登录成功！Cookie 已保存", + "cookie_saved": bool(_memory_cookie), + } + + # 尝试检测"已扫码未确认"状态（抖音扫码后会出现确认页面） + try: + # 页面标题或 URL 变化可作为"已扫码"的信号 + current_url = page.url + if "confirm" in current_url or "auth" in current_url: + session["status"] = "scanned" + return {"status": "scanned", "message": "已扫码，请在手机端确认登录"} + except Exception: + pass + + return { + "status": "pending", + "message": "等待扫码...", + "expires_at": session["expires_at"].isoformat(), + } + + +@router.post("/logout", summary="登出（清除 Cookie）") +async def logout() -> Dict[str, Any]: + """ + 清除保存的登录 Cookie（持久化文件 + 内存缓存）。 + 同时关闭所有活跃的登录 session。 + """ + global _memory_cookie + + # 关闭所有活跃 session + for sid in list(_active_sessions.keys()): + await _close_session(sid) + + # 删除持久化 Cookie 文件 + _delete_cookie_file() + + # 清空内存缓存 + _memory_cookie = "" + + logger.info("已登出，Cookie 已清除") + return {"ok": True, "message": "已登出，Cookie 已清除"} + + +@router.get("/status", summary="查询当前登录状态") +async def get_login_status() -> Dict[str, Any]: + """ + 查询当前是否有有效的登录 Cookie。 + + 返回： + - logged_in: 是否已登录 + - cookie_preview: Cookie 的前 30 个字符（用于确认是哪个账号） + - source: Cookie 来源（file / memory / none） + """ + global _memory_cookie + + # 优先使用内存缓存 + cookie_str = _memory_cookie + + # 内存为空时尝试从文件加载 + if not cookie_str: + cookie_str = _load_cookie_from_file() + if cookie_str: + _memory_cookie = cookie_str # 同步到内存缓存 + source = "file" + else: + source = "none" + else: + source = "memory" + + logged_in = bool(cookie_str) + preview = cookie_str[:30] + "..." if len(cookie_str) > 30 else cookie_str + + return { + "logged_in": logged_in, + "cookie_preview": preview if logged_in else "", + "source": source, + "active_sessions": len(_active_sessions), + } + + +def get_memory_cookie() -> str: + """★ Fix 6: 获取内存中缓存的 Cookie（供采集任务复用）★""" + global _memory_cookie + # 内存为空时尝试从文件加载 + if not _memory_cookie: + _memory_cookie = _load_cookie_from_file() + return _memory_cookie diff --git a/api/main.py b/api/main.py index 82f166493..9a973be39 100644 --- a/api/main.py +++ b/api/main.py @@ -1,200 +1,355 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2025 relakkes@gmail.com -# -# This file is part of MediaCrawler project. -# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/main.py -# GitHub: https://github.com/NanmiCoder -# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 -# -# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： -# 1. 不得用于任何商业用途。 -# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 -# 3. 不得进行大规模爬取或对平台造成运营干扰。 -# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 -# 5. 不得用于任何非法或不当的用途。 -# -# 详细许可条款请参阅项目根目录下的LICENSE文件。 -# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 - """ -MediaCrawler WebUI API Server -Start command: uvicorn api.main:app --port 8080 --reload -Or: python -m api.main +douyin_scraper.api.main — FastAPI 应用入口 +============================================= +v6 新增：Web API 服务入口。 + +启动方式： + uvicorn api.main:app --host 0.0.0.0 --port 8000 + +或通过环境变量配置： + DY_API_HOST=0.0.0.0 DY_API_PORT=8000 uvicorn api.main:app + +我实际执行时踩过的坑： + - 直接在 HTTP handler 中跑采集 → 请求超时 + - 无任务隔离 → 并发互相覆盖状态 + - 无健康检查 → 运维无法判断服务是否正常 + - 日志无结构化 → 排查问题时翻几小时日志 """ + import asyncio +import logging import os +import platform +import shutil import sys -import subprocess -import uvicorn -from fastapi import FastAPI +import time +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any, Dict, Optional + +from fastapi import FastAPI, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware -from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse -from .routers import crawler_router, data_router, websocket_router +from douyin_scraper.utils import ( + check_command_exists, + check_disk_space, + check_port_in_use, + setup_ffmpeg, +) + +from .routes import router, set_task_manager +from .login import router as login_router +from .tasks import TaskManager +from .ws import ws_manager +from .schemas import LogEntry + +logger = logging.getLogger("douyin_scraper.api") + +# ═══════════════════════════════════════════════════════════════ +# 日志广播后台任务（把 crawler_manager 的日志队列推给 WebSocket 前端） +# ═══════════════════════════════════════════════════════════════ + +_log_broadcaster_task: Optional[asyncio.Task] = None + + +async def log_broadcaster(): + """ + 后台任务：从 crawler_manager 的日志队列读取 LogEntry， + 通过 ws_manager.broadcast() 推送给所有连接的 WebSocket 前端。 + 消息格式：{"type": "log", "data": {...}} + """ + from .services.crawler_manager import CrawlerManager + from .ws import ws_manager + + # 获取 crawler_manager 单例（在 services/__init__.py 中定义） + from .services import crawler_manager + + queue = crawler_manager.get_log_queue() + logger.info("[LogBroadcaster] 启动，等待日志消息...") + + while True: + try: + entry = await queue.get() + # 广播给所有连接的 WS 客户端 + msg = { + "type": "log", + "data": entry.model_dump() if hasattr(entry, "model_dump") else str(entry), + } + await ws_manager.broadcast(msg) + except asyncio.CancelledError: + logger.info("[LogBroadcaster] 被取消，退出") + break + except Exception as e: + logger.warning("[LogBroadcaster] 错误: %s", e) + await asyncio.sleep(0.1) + + +# ═══════════════════════════════════════════════════════════════ +# 全局配置 +# ═══════════════════════════════════════════════════════════════ + +API_HOST = os.environ.get("DY_API_HOST", "0.0.0.0") +API_PORT = int(os.environ.get("DY_API_PORT", "8000")) +WORKSPACE_DIR = os.environ.get("DY_WORKSPACE_DIR", "./workspaces") +CHROME_PORT = int(os.environ.get("DY_CHROME_PORT", "9222")) +LOG_LEVEL = os.environ.get("DY_LOG_LEVEL", "INFO") + + +# ═══════════════════════════════════════════════════════════════ +# 结构化日志格式 +# ═══════════════════════════════════════════════════════════════ + +class JSONFormatter(logging.Formatter): + """ + JSON Lines 日志格式化器。 + ★ 我实际执行时：纯文本日志无法被 ELK/Grafana 解析。★ + """ + + def format(self, record: logging.LogRecord) -> str: + import json + entry = { + "ts": self.formatTime(record, self.datefmt), + "level": record.levelname, + "logger": record.name, + "msg": record.getMessage(), + } + if record.exc_info and record.exc_info[1]: + entry["exception"] = str(record.exc_info[1]) + return json.dumps(entry, ensure_ascii=False) + + +def _setup_logging() -> None: + """配置结构化日志""" + log_dir = Path(WORKSPACE_DIR) / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + + # 控制台：人类可读格式 + console = logging.StreamHandler() + console.setFormatter(logging.Formatter( + "[%(asctime)s] %(levelname)s %(name)s: %(message)s" + )) + console.setLevel(getattr(logging, LOG_LEVEL, logging.INFO)) + + # 文件：JSON Lines 格式（自动轮转 100MB × 5） + from logging.handlers import RotatingFileHandler + file_handler = RotatingFileHandler( + str(log_dir / "api.jsonl"), + maxBytes=100 * 1024 * 1024, # 100MB + backupCount=5, + encoding="utf-8", + ) + file_handler.setFormatter(JSONFormatter(datefmt="%Y-%m-%dT%H:%M:%S")) + file_handler.setLevel(logging.DEBUG) + + # 配置根 logger + root = logging.getLogger() + root.setLevel(logging.DEBUG) + root.addHandler(console) + root.addHandler(file_handler) + + # 降低第三方库日志级别 + logging.getLogger("uvicorn").setLevel(logging.INFO) + logging.getLogger("httpx").setLevel(logging.WARNING) + + +# ═══════════════════════════════════════════════════════════════ +# 应用生命周期 +# ═══════════════════════════════════════════════════════════════ + +_task_manager_instance = None +_start_time: float = 0 +_ffmpeg_available: Optional[bool] = None + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """应用生命周期管理""" + global _task_manager_instance, _start_time + _start_time = time.time() + + # 启动时初始化 + _setup_logging() + logger.info("API 服务启动中...") + + _task_manager_instance = TaskManager(base_dir=WORKSPACE_DIR) + app.state.task_manager = _task_manager_instance + set_task_manager(_task_manager_instance) + + # 启动时清理过期任务 + _task_manager_instance.cleanup_old_tasks(max_age_hours=72) + + # 启动时检测 ffmpeg 可用性（仅一次，避免健康检查副作用） + global _ffmpeg_available + _ffmpeg_available = setup_ffmpeg() + + logger.info( + "API 服务就绪: host=%s port=%d workspace=%s", + API_HOST, API_PORT, WORKSPACE_DIR, + ) + + # 启动日志广播后台任务（把 crawler_manager 的日志推送给 WebSocket 前端） + global _log_broadcaster_task + _log_broadcaster_task = asyncio.create_task(log_broadcaster()) + logger.info("日志广播后台任务已启动") + + yield + + # 关闭时取消日志广播任务 + if _log_broadcaster_task and not _log_broadcaster_task.done(): + _log_broadcaster_task.cancel() + try: + await _log_broadcaster_task + except asyncio.CancelledError: + pass + logger.info("API 服务已关闭") + + +# ═══════════════════════════════════════════════════════════════ +# FastAPI 应用 +# ═══════════════════════════════════════════════════════════════ app = FastAPI( - title="MediaCrawler WebUI API", - description="API for controlling MediaCrawler from WebUI", - version="1.0.0" + title="抖音采集工具 API", + description=( + "抖音关键词批量采集工具的 RESTful API。" + "支持搜索采集、评论采集、文案提取、数据合并。" + "所有长时间操作异步执行，通过 task_id 查询进度。" + ), + version="6.0.0", + lifespan=lifespan, + docs_url="/docs", + redoc_url="/redoc", ) -# Get webui static files directory -WEBUI_DIR = os.path.join(os.path.dirname(__file__), "webui") - -# CORS configuration - allow frontend dev server access +# CORS（根据环境配置来源列表，allow_credentials=False 与通配符来源兼容） +_cors_origins = os.environ.get("DY_CORS_ORIGINS", "*").split(",") app.add_middleware( CORSMiddleware, - allow_origins=[ - "http://localhost:5173", # Vite dev server - "http://localhost:3000", # Backup port - "http://127.0.0.1:5173", - "http://127.0.0.1:3000", - ], - allow_credentials=True, + allow_origins=_cors_origins, + allow_credentials=False if "*" in _cors_origins else True, allow_methods=["*"], allow_headers=["*"], ) -# Register routers -app.include_router(crawler_router, prefix="/api") -app.include_router(data_router, prefix="/api") -app.include_router(websocket_router, prefix="/api") +# 注册路由 +app.include_router(router) +app.include_router(login_router) -@app.get("/") -async def serve_frontend(): - """Return frontend page""" - index_path = os.path.join(WEBUI_DIR, "index.html") - if os.path.exists(index_path): - return FileResponse(index_path) - return { - "message": "MediaCrawler WebUI API", - "version": "1.0.0", - "docs": "/docs", - "note": "WebUI not found, please build it first: cd webui && npm run build" - } +# ═══════════════════════════════════════════════════════════════ +# WebSocket 端点（直接注册在 app，不走 /scrape 前缀） +# ═══════════════════════════════════════════════════════════════ + +@app.websocket("/ws/tasks") +async def websocket_tasks(ws: WebSocket): + """WebSocket 端点：推送任务状态变更""" + await ws_manager.connect(ws) + try: + while True: + await ws.receive_text() + except WebSocketDisconnect: + ws_manager.disconnect(ws) + +# ═══════════════════════════════════════════════════════════════ +# 静态文件挂载（Web UI） +# ═══════════════════════════════════════════════════════════════ +# 挂在 /ui 而非 /，避免拦截 WebSocket (/ws/tasks) 和 API (/scrape/*) +from fastapi.staticfiles import StaticFiles + +_webui_dir = Path(__file__).parent / "webui" +if _webui_dir.exists() and (_webui_dir / "index.html").exists(): + app.mount("/ui", StaticFiles(directory=str(_webui_dir), html=True), name="webui") -@app.get("/api/health") -async def health_check(): - return {"status": "ok"} +# ═══════════════════════════════════════════════════════════════ +# 健康检查 +# ═══════════════════════════════════════════════════════════════ +@app.get("/health", summary="健康检查", tags=["system"]) +async def health_check() -> Dict[str, Any]: + """ + 健康检查端点。 -@app.get("/api/env/check") -async def check_environment(): - """Check if MediaCrawler environment is configured correctly""" + 返回： + - 服务运行时间 + - Chrome CDP 端口状态 + - 磁盘空间 + - 依赖版本 + - 任务统计 + """ + uptime = time.time() - _start_time if _start_time else 0 + + # Chrome CDP 端口检查 + cdp_ok = check_port_in_use(CHROME_PORT) + + # 磁盘空间 + workspace_path = Path(WORKSPACE_DIR) + disk_ok = True + free_gb = 0.0 try: - # Run uv run main.py --help command to check environment - if sys.platform == "win32": - loop = asyncio.get_running_loop() - process = await loop.run_in_executor( - None, - lambda: subprocess.run( - ["uv", "run", "main.py", "--help"], - capture_output=True, - timeout=30.0, - cwd="." - ) - ) - stdout, stderr = process.stdout, process.stderr # bytes - else: - process = await asyncio.create_subprocess_exec( - "uv", "run", "main.py", "--help", - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd="." # Project root directory - ) - stdout, stderr = await asyncio.wait_for( - process.communicate(), - timeout=30.0 # 30 seconds timeout - ) - if process.returncode == 0: - return { - "success": True, - "message": "MediaCrawler environment configured correctly", - "output": stdout.decode("utf-8", errors="ignore")[:500] # Truncate to first 500 characters - } - else: - error_msg = stderr.decode("utf-8", errors="ignore") or stdout.decode("utf-8", errors="ignore") - return { - "success": False, - "message": "Environment check failed", - "error": error_msg[:500] - } - except asyncio.TimeoutError: - return { - "success": False, - "message": "Environment check timeout", - "error": "Command execution exceeded 30 seconds" - } - except FileNotFoundError: - return { - "success": False, - "message": "uv command not found", - "error": "Please ensure uv is installed and configured in system PATH" - } - except Exception as e: - return { - "success": False, - "message": "Environment check error", - "error": f"{type(e).__name__}: {str(e) or 'Unknown'}" - } + usage = shutil.disk_usage(str(workspace_path)) + free_gb = usage.free / (1024**3) + disk_ok = free_gb > 1.0 + except OSError: + pass + # 依赖检查（使用启动时缓存的结果，避免副作用） + ffmpeg_ok = _ffmpeg_available if _ffmpeg_available is not None else check_command_exists("ffmpeg") + python_ver = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" -@app.get("/api/config/platforms") -async def get_platforms(): - """Get list of supported platforms""" - return { - "platforms": [ - {"value": "xhs", "label": "Xiaohongshu", "icon": "book-open"}, - {"value": "dy", "label": "Douyin", "icon": "music"}, - {"value": "ks", "label": "Kuaishou", "icon": "video"}, - {"value": "bili", "label": "Bilibili", "icon": "tv"}, - {"value": "wb", "label": "Weibo", "icon": "message-circle"}, - {"value": "tieba", "label": "Baidu Tieba", "icon": "messages-square"}, - {"value": "zhihu", "label": "Zhihu", "icon": "help-circle"}, - ] + # 任务统计 + task_stats = {} + if _task_manager_instance: + task_stats = _task_manager_instance.get_stats() + + health = { + "status": "healthy" if (disk_ok and cdp_ok) else "degraded", + "uptime_seconds": round(uptime, 1), + "checks": { + "chrome_cdp": { + "port": CHROME_PORT, + "status": "ok" if cdp_ok else "not_running", + }, + "disk": { + "free_gb": round(free_gb, 2), + "status": "ok" if disk_ok else "low", + }, + "ffmpeg": { + "status": "ok" if ffmpeg_ok else "missing", + }, + }, + "system": { + "python": python_ver, + "platform": platform.platform(), + "workspace": str(workspace_path.resolve()), + }, + "tasks": task_stats, } + return health + -@app.get("/api/config/options") -async def get_config_options(): - """Get all configuration options""" +@app.get("/", summary="根路径", tags=["system"]) +async def root() -> Dict[str, str]: + """API 根路径，返回基本信息""" return { - "login_types": [ - {"value": "qrcode", "label": "QR Code Login"}, - {"value": "cookie", "label": "Cookie Login"}, - ], - "crawler_types": [ - {"value": "search", "label": "Search Mode"}, - {"value": "detail", "label": "Detail Mode"}, - {"value": "creator", "label": "Creator Mode"}, - ], - "save_options": [ - {"value": "jsonl", "label": "JSONL File"}, - {"value": "json", "label": "JSON File"}, - {"value": "csv", "label": "CSV File"}, - {"value": "excel", "label": "Excel File"}, - {"value": "sqlite", "label": "SQLite Database"}, - {"value": "db", "label": "MySQL Database"}, - {"value": "mongodb", "label": "MongoDB Database"}, - ], + "name": "抖音采集工具 API", + "version": "6.0.0", + "docs": "/docs", + "health": "/health", } -# Mount static resources - must be placed after all routes -if os.path.exists(WEBUI_DIR): - assets_dir = os.path.join(WEBUI_DIR, "assets") - if os.path.exists(assets_dir): - app.mount("/assets", StaticFiles(directory=assets_dir), name="assets") - # Mount logos directory - logos_dir = os.path.join(WEBUI_DIR, "logos") - if os.path.exists(logos_dir): - app.mount("/logos", StaticFiles(directory=logos_dir), name="logos") - # Mount other static files (e.g., vite.svg) - app.mount("/static", StaticFiles(directory=WEBUI_DIR), name="webui-static") - +# ═══════════════════════════════════════════════════════════════ +# 直接运行入口 +# ═══════════════════════════════════════════════════════════════ if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=8080) + import uvicorn + uvicorn.run( + "api.main:app", + host=API_HOST, + port=API_PORT, + reload=os.environ.get("DY_RELOAD", "0") == "1", + log_level=LOG_LEVEL.lower(), + ) diff --git a/api/routes.py b/api/routes.py new file mode 100644 index 000000000..4b70479ef --- /dev/null +++ b/api/routes.py @@ -0,0 +1,700 @@ +""" +douyin_scraper.api.routes — API 路由 +====================================== +v6 新增：FastAPI 路由层，对 DouyinScraper 模块的 HTTP 封装。 + +API 设计决策： + 1. 所有长时间操作返回 task_id，客户端轮询状态 + 2. 每个任务独立 workspace，互不干扰 + 3. 结果文件通过 /scrape/result/{task_id} 下载 + 4. 错误响应包含 exit_code 分类（1=可重试, 2=不可重试, 3=致命） + +我实际执行时踩过的坑： + - HTTP handler 中直接运行采集 → 请求超时 + - 没有任务隔离 → 并发请求互相干扰 + - 错误只返回 500 → 客户端无法区分可重试和不可重试错误 + - 结果文件路径硬编码 → 部署后找不到文件 +""" + +import logging +import os +import shutil +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, HTTPException, Query, WebSocket, WebSocketDisconnect +from fastapi.responses import FileResponse, JSONResponse +from pydantic import BaseModel, Field, field_validator +from typing import Literal + +from douyin_scraper import DouyinScraper, ScraperConfig +from douyin_scraper.exceptions import ( + ConfigError, + FatalError, + NonRetryableError, + RetryableError, + ScraperError, +) +from douyin_scraper.utils import ( + check_disk_space, + check_port_in_use, + check_command_exists, + setup_ffmpeg, +) + +from .tasks import TaskManager +from .utils import validate_path_in_workspace +from .ws import ws_manager + +logger = logging.getLogger("douyin_scraper.api") + +router = APIRouter(prefix="/scrape", tags=["scrape"]) + +# 全局任务管理器（由 main.py 注入） +_task_manager: Optional[TaskManager] = None + + +def set_task_manager(tm: TaskManager) -> None: + global _task_manager + _task_manager = tm + + +def get_task_manager() -> TaskManager: + if _task_manager is None: + raise RuntimeError("TaskManager 未初始化") + return _task_manager + + +# ═══════════════════════════════════════════════════════════════ +# 请求模型 +# ═══════════════════════════════════════════════════════════════ + +class SearchRequest(BaseModel): + """搜索采集请求""" + keywords: List[str] = Field(..., description="搜索关键词列表") + max_count: int = Field(20, description="每个关键词最大采集数", ge=1, le=200) + project_dir: Optional[str] = Field(None, description="工作目录（默认自动创建）") + + @field_validator("keywords") + @classmethod + def validate_keywords(cls, v: List[str]) -> List[str]: + if not v: + raise ValueError("keywords 不能为空") + if len(v) > 50: + raise ValueError("keywords 最多 50 个") + for kw in v: + if len(kw) > 200: + raise ValueError(f"关键词过长: {kw[:50]}...") + return v + + +class CommentsRequest(BaseModel): + """评论采集请求""" + video_jsonl: Optional[str] = Field(None, description="视频 JSONL 路径") + project_dir: Optional[str] = Field(None, description="工作目录") + + +class ScriptsRequest(BaseModel): + """文案提取请求""" + video_jsonl: Optional[str] = Field(None, description="视频 JSONL 路径") + model: Literal["tiny", "base", "small", "medium", "large"] = Field( + "small", description="Whisper 模型大小: tiny/base/small/medium/large" + ) + project_dir: Optional[str] = Field(None, description="工作目录") + + +class MergeRequest(BaseModel): + """合并数据请求""" + video_jsonl: Optional[str] = Field(None, description="视频 JSONL 路径") + comments_jsonl: Optional[str] = Field(None, description="评论 JSONL 路径") + scripts_jsonl: Optional[str] = Field(None, description="文案 JSONL 路径") + output_csv: Optional[str] = Field(None, description="输出 CSV 路径") + project_dir: Optional[str] = Field(None, description="工作目录") + + +class ResetRequest(BaseModel): + """重置步骤请求""" + step: str = Field(..., description="要重置的步骤名称") + clear_dedupe: bool = Field(False, description="是否同时清除去重索引") + project_dir: Optional[str] = Field(None, description="工作目录") + + @field_validator("step") + @classmethod + def validate_step(cls, v: str) -> str: + valid_steps = { + "clone_repo", "setup_env", "config_douyin", + "run_search", "fetch_comments", "install_ffmpeg", + "install_whisper", "run_extract", "merge_csv", + } + if v not in valid_steps: + raise ValueError( + f"无效步骤名: {v}，有效步骤: {', '.join(sorted(valid_steps))}" + ) + return v + + +class RunAllRequest(BaseModel): + """一键运行请求""" + keywords: List[str] = Field(..., description="搜索关键词列表") + max_count: int = Field(20, description="每个关键词最大采集数") + steps: Optional[List[str]] = Field(None, description="指定步骤（默认全部）") + project_dir: Optional[str] = Field(None, description="工作目录") + + +# ═══════════════════════════════════════════════════════════════ +# 辅助函数 +# ═══════════════════════════════════════════════════════════════ + +def _make_scraper(project_dir: Optional[str], workspace: str) -> DouyinScraper: + """ + 创建 DouyinScraper 实例。 + + ★ 我实际执行时：多个请求共享同一个 scraper 实例， + 状态互相覆盖 → 每个任务独立实例。★ + + Docker 环境检测：如果 /app/main.py 存在（Docker 容器）， + 使用 /app/ 作为 project_dir（MediaCrawler 项目根）， + workspace 作为数据输出目录。 + """ + if project_dir is None: + # Docker 检测：/app/main.py 存在 → 容器环境 + if Path("/app/main.py").exists(): + project_dir = "/app/" + else: + project_dir = workspace + + # 每个任务使用独立的 state_dir，避免任务间状态污染 + # workspace 格式: /app/workspaces/ → state_dir = /app/workspaces//state + ws_path = Path(workspace) + task_id = ws_path.name # 从 workspace 路径中提取 task_id + state_dir_name = f"workspaces/{task_id}/state" + + config_dict: Dict[str, Any] = { + "project_dir": project_dir, + "state_dir_name": state_dir_name, + "enable_cdp_mode": False, # Docker 中不用 CDP，用 headless playwright + } + return DouyinScraper(config_dict) + + +def _error_response(e: Exception) -> HTTPException: + """将异常转换为 HTTP 响应""" + if isinstance(e, ScraperError): + status_map = {1: 503, 2: 400, 3: 500} # 可重试/不可重试/致命 + status_code = status_map.get(e.exit_code, 500) + return HTTPException( + status_code=status_code, + detail={ + "error": str(e), + "step": e.step, + "exit_code": e.exit_code, + "details": e.details, + }, + ) + # 非 ScraperError：不暴露内部信息给 API 调用者 + logger.error("未预期异常: %s", e, exc_info=True) + return HTTPException( + status_code=500, + detail={"error": "内部错误，请查看日志"}, + ) + + +# ═══════════════════════════════════════════════════════════════ +# API 端点 +# ═══════════════════════════════════════════════════════════════ + +@router.post("/search", summary="触发搜索采集") +async def search(req: SearchRequest) -> Dict[str, Any]: + """ + 触发搜索采集任务（异步执行）。 + + 返回 task_id，使用 GET /scrape/status/{task_id} 查询进度。 + """ + tm = get_task_manager() + task = tm.create_task("search", params=req.model_dump()) + + def _do_search() -> Dict[str, Any]: + logger.info("API search request task_id=%s keywords=%r", task.task_id, req.keywords) + scraper = _make_scraper(req.project_dir, task.workspace) + output = scraper.search(keywords=req.keywords, max_count=req.max_count) + return {"video_jsonl": str(output), "status": scraper.get_status()} + + tm.submit(task, _do_search) + return {"task_id": task.task_id, "status": "submitted", "type": "search"} + + +@router.post("/comments", summary="触发评论采集") +async def fetch_comments(req: CommentsRequest) -> Dict[str, Any]: + """ + 触发评论采集任务（异步执行）。 + 需要先完成搜索采集，或提供 video_jsonl 路径。 + """ + tm = get_task_manager() + task = tm.create_task("comments", params=req.model_dump()) + + def _do_comments() -> Dict[str, Any]: + scraper = _make_scraper(req.project_dir, task.workspace) + video_path = Path(req.video_jsonl) if req.video_jsonl else None + if video_path: + video_path = validate_path_in_workspace( + req.video_jsonl, Path(task.workspace) + ) + output = scraper.fetch_comments(video_jsonl=video_path) + return {"comments_jsonl": str(output), "status": scraper.get_status()} + + tm.submit(task, _do_comments) + return {"task_id": task.task_id, "status": "submitted", "type": "comments"} + + +@router.post("/scripts", summary="触发言案提取") +async def extract_scripts(req: ScriptsRequest) -> Dict[str, Any]: + """ + 触发视频文案提取任务（异步执行）。 + 需要先完成搜索采集，或提供 video_jsonl 路径。 + """ + tm = get_task_manager() + task = tm.create_task("scripts", params=req.model_dump()) + + def _do_scripts() -> Dict[str, Any]: + scraper = _make_scraper(req.project_dir, task.workspace) + video_path = Path(req.video_jsonl) if req.video_jsonl else None + if video_path: + video_path = validate_path_in_workspace( + req.video_jsonl, Path(task.workspace) + ) + output = scraper.extract_scripts( + video_jsonl=video_path, model=req.model + ) + return {"scripts_jsonl": str(output), "status": scraper.get_status()} + + tm.submit(task, _do_scripts) + return {"task_id": task.task_id, "status": "submitted", "type": "scripts"} + + +@router.post("/merge", summary="触发数据合并") +async def merge(req: MergeRequest) -> Dict[str, Any]: + """ + 触发数据合并任务（异步执行）。 + 合并视频、评论、文案数据生成标准 CSV。 + """ + tm = get_task_manager() + task = tm.create_task("merge", params=req.model_dump()) + + def _do_merge() -> Dict[str, Any]: + scraper = _make_scraper(req.project_dir, task.workspace) + workspace = Path(task.workspace) + v_path = Path(req.video_jsonl) if req.video_jsonl else None + c_path = Path(req.comments_jsonl) if req.comments_jsonl else None + s_path = Path(req.scripts_jsonl) if req.scripts_jsonl else None + o_path = Path(req.output_csv) if req.output_csv else None + # 路径遍历防护：验证所有用户提供的路径都在 workspace 内 + if v_path: + v_path = validate_path_in_workspace(req.video_jsonl, workspace) + if c_path: + c_path = validate_path_in_workspace(req.comments_jsonl, workspace) + if s_path: + s_path = validate_path_in_workspace(req.scripts_jsonl, workspace) + if o_path: + o_path = validate_path_in_workspace(req.output_csv, workspace) + output = scraper.merge( + video_jsonl=v_path, + comments_jsonl=c_path, + scripts_jsonl=s_path, + output_csv=o_path, + ) + return {"csv_path": str(output), "status": scraper.get_status()} + + tm.submit(task, _do_merge) + return {"task_id": task.task_id, "status": "submitted", "type": "merge"} + + +@router.post("/run-all", summary="一键执行全部步骤") +async def run_all(req: RunAllRequest) -> Dict[str, Any]: + """ + 一键执行全部采集步骤（搜索→评论→文案→合并）。 + """ + tm = get_task_manager() + task = tm.create_task("run_all", params=req.model_dump()) + + def _do_run_all() -> Dict[str, Any]: + config_dict: Dict[str, Any] = { + "project_dir": req.project_dir or task.workspace, + "keywords": req.keywords, + "max_videos_per_keyword": req.max_count, + } + scraper = DouyinScraper(config_dict) + return scraper.run_all(steps=req.steps) + + tm.submit(task, _do_run_all) + return {"task_id": task.task_id, "status": "submitted", "type": "run_all"} + + +@router.get("/status/{task_id}", summary="查询任务状态") +async def get_status(task_id: str) -> Dict[str, Any]: + """查询异步任务的状态""" + tm = get_task_manager() + task = tm.get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail=f"任务不存在: {task_id}") + return task.to_dict() + + +@router.get("/result/{task_id}", summary="下载结果文件") +async def get_result(task_id: str): + """ + 下载任务的结果文件（CSV 或 JSONL）。 + + ★ 我实际执行时：结果文件散落在各处，用户找不到。 + v6：通过 task_id 自动定位结果文件。★ + """ + tm = get_task_manager() + task = tm.get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail=f"任务不存在: {task_id}") + if task.status != "completed": + raise HTTPException( + status_code=400, + detail=f"任务未完成，当前状态: {task.status}", + ) + + result_path = tm.get_result_path(task_id) + if not result_path: + raise HTTPException(status_code=404, detail="结果文件不存在") + + if result_path.is_file(): + # 确定媒体类型 + media_type = "application/octet-stream" + if result_path.suffix == ".csv": + media_type = "text/csv" + elif result_path.suffix == ".jsonl": + media_type = "application/jsonl" + + filename = result_path.name + return FileResponse( + path=str(result_path), + media_type=media_type, + filename=filename, + ) + elif result_path.is_dir(): + # 返回目录下所有文件列表 + files = [f.name for f in result_path.rglob("*") if f.is_file()] + return JSONResponse(content={"task_id": task_id, "files": files}) + + raise HTTPException(status_code=404, detail="结果路径无效") + + +@router.post("/reset", summary="重置步骤状态") +async def reset_step(req: ResetRequest) -> Dict[str, Any]: + """ + 重置某步骤的状态为 pending。 + + ★ 我实际执行时：步骤失败后无法重新执行，只能手动删除状态文件。 + v6：通过 API 重置，可选清除去重索引。★ + """ + try: + scraper = _make_scraper(req.project_dir, "./workspace_default") + scraper.reset_step(req.step, clear_dedupe=req.clear_dedupe) + return {"status": "reset", "step": req.step, "clear_dedupe": req.clear_dedupe} + except ScraperError as e: + raise _error_response(e) + except Exception as e: + raise _error_response(e) + + +@router.get("/tasks", summary="列出所有任务") +async def list_tasks( + task_type: Optional[str] = Query(None, description="按类型过滤"), + status: Optional[str] = Query(None, description="按状态过滤"), + limit: int = Query(50, ge=1, le=200, description="返回数量上限"), + offset: int = Query(0, ge=0, description="偏移量"), +) -> Dict[str, Any]: + """列出任务（支持分页）""" + tm = get_task_manager() + all_tasks = tm.list_tasks(task_type=task_type, status=status, limit=10000) + total = len(all_tasks) + # 应用 offset 和 limit + paginated = all_tasks[offset : offset + limit] + return { + "tasks": [t.to_dict() for t in paginated], + "total": total, + "offset": offset, + "limit": limit, + "stats": tm.get_stats(), + } + + +@router.delete("/tasks/{task_id}", summary="删除任务记录") +async def delete_task(task_id: str) -> Dict[str, str]: + """删除任务记录""" + tm = get_task_manager() + if tm.delete_task(task_id): + return {"status": "deleted", "task_id": task_id} + raise HTTPException(status_code=404, detail=f"任务不存在: {task_id}") + + +@router.post("/cleanup", summary="清理过期任务") +async def cleanup_tasks( + max_age_hours: int = Query(72, ge=1, description="保留最近 N 小时的任务"), +) -> Dict[str, Any]: + """清理超过指定时间的已完成/失败任务""" + tm = get_task_manager() + removed = tm.cleanup_old_tasks(max_age_hours=max_age_hours) + return {"removed": removed, "remaining": len(tm.list_tasks())} + + +# ═══════════════════════════════════════════════════════════════ +# 数据管理 API +# ═══════════════════════════════════════════════════════════════ + +import csv +import io +import json as _json + +MAX_EXPORT_ROWS = 200 +MAX_EXPORT_BYTES = 2 * 1024 * 1024 # 2MB + + +def _find_result_files(workspace: Path) -> List[Path]: + """在 workspace 中查找 CSV/JSONL 结果文件（最多 2 层深度）""" + results: List[Path] = [] + for pattern in ("*.csv", "*.jsonl"): + for p in workspace.rglob(pattern): + if len(p.relative_to(workspace).parts) <= 2 and p.is_file(): + results.append(p) + return results + + +def _count_file_rows(path: Path) -> int: + """统计文件行数（不含空行），出错返回 0""" + try: + count = 0 + with open(path, "r", encoding="utf-8", errors="replace") as f: + for line in f: + if line.strip(): + count += 1 + return count + except OSError: + return 0 + + +def _read_jsonl_rows(path: Path, limit: int = MAX_EXPORT_ROWS) -> List[Dict[str, Any]]: + """读取 JSONL 文件，返回字典列表""" + rows: List[Dict[str, Any]] = [] + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + rows.append(_json.loads(line)) + except _json.JSONDecodeError: + continue + if len(rows) >= limit: + break + except OSError: + pass + return rows + + +def _read_csv_rows(path: Path, limit: int = MAX_EXPORT_ROWS) -> List[Dict[str, Any]]: + """读取 CSV 文件，返回字典列表""" + rows: List[Dict[str, Any]] = [] + try: + with open(path, "r", encoding="utf-8-sig", errors="replace", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + rows.append(dict(row)) + if len(rows) >= limit: + break + except OSError: + pass + return rows + + +def _normalize_row(row: Dict[str, Any]) -> Dict[str, Any]: + """规范化一行数据，确保关键字段存在""" + return { + "video_id": str(row.get("video_id", row.get("id", ""))), + "platform": str(row.get("platform", "douyin")), + "script_text": str(row.get("script_text", row.get("script", row.get("text", "")))), + "likes": str(row.get("likes", row.get("like_count", ""))), + "favorites": str(row.get("favorites", row.get("collect_count", ""))), + "shares": str(row.get("shares", row.get("share_count", ""))), + "comments": str(row.get("comments", row.get("comment_count", ""))), + } + + +@router.get("/data/list", summary="列出可导出的数据文件") +async def list_data_files() -> Dict[str, Any]: + """ + 扫描所有已完成任务的 workspace，返回可导出的数据文件列表。 + 每条记录包含：task_id, task_type, file_name, file_size, row_count, created_at + """ + tm = get_task_manager() + all_tasks = tm.list_tasks(status="completed", limit=10000) + + items: List[Dict[str, Any]] = [] + for task in all_tasks: + workspace = Path(task.workspace) + if not workspace.exists(): + continue + files = _find_result_files(workspace) + for fpath in files: + try: + stat = fpath.stat() + items.append({ + "task_id": task.task_id, + "task_type": task.task_type, + "file_name": fpath.name, + "file_path": str(fpath.relative_to(workspace)), + "file_size": stat.st_size, + "row_count": _count_file_rows(fpath), + "created_at": task.completed_at or task.created_at, + "keywords": task.params.get("keywords", []), + }) + except OSError: + continue + + # 按完成时间降序 + items.sort(key=lambda x: x["created_at"], reverse=True) + return {"items": items, "total": len(items)} + + +@router.get("/data/preview/{task_id}", summary="预览任务结果数据（前 20 行）") +async def preview_data(task_id: str) -> Dict[str, Any]: + """返回任务结果文件的前 20 行数据""" + tm = get_task_manager() + task = tm.get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail=f"任务不存在: {task_id}") + if task.status != "completed": + raise HTTPException(status_code=400, detail=f"任务未完成，当前状态: {task.status}") + + workspace = Path(task.workspace) + files = _find_result_files(workspace) + if not files: + raise HTTPException(status_code=404, detail="未找到结果文件") + + # 优先 CSV，其次 JSONL + target = next((f for f in files if f.suffix == ".csv"), files[0]) + + if target.suffix == ".csv": + rows = _read_csv_rows(target, limit=20) + else: + raw_rows = _read_jsonl_rows(target, limit=20) + rows = [_normalize_row(r) for r in raw_rows] + + return { + "task_id": task_id, + "file_name": target.name, + "rows": rows, + "total_rows": _count_file_rows(target), + } + + +class ExportRequest(BaseModel): + """数据导出请求""" + task_ids: List[str] = Field(..., description="任务 ID 列表（支持多选）") + format: Literal["csv", "txt"] = Field("csv", description="导出格式") + limit: int = Field(MAX_EXPORT_ROWS, ge=1, le=MAX_EXPORT_ROWS, description="最大行数上限") + + @field_validator("task_ids") + @classmethod + def validate_task_ids(cls, v: List[str]) -> List[str]: + if not v: + raise ValueError("task_ids 不能为空") + if len(v) > 50: + raise ValueError("一次最多导出 50 个任务") + return v + + +@router.post("/data/export", summary="导出数据（CSV 或 TXT）") +async def export_data(req: ExportRequest): + """ + 批量导出多个任务的结果数据。 + + CSV 格式：列 video_id, platform, script_text, likes, favorites, shares, comments（| 分隔多值） + TXT 格式：每行一条，字段用 || 分隔：video_id||script_text||likes||favorites||shares||comments + 上限 200 行、2MB、UTF-8 + """ + tm = get_task_manager() + + # 收集所有行 + all_rows: List[Dict[str, Any]] = [] + for task_id in req.task_ids: + task = tm.get_task(task_id) + if not task or task.status != "completed": + continue + workspace = Path(task.workspace) + files = _find_result_files(workspace) + if not files: + continue + target = next((f for f in files if f.suffix == ".csv"), files[0]) + if target.suffix == ".csv": + rows = _read_csv_rows(target, limit=req.limit) + else: + raw_rows = _read_jsonl_rows(target, limit=req.limit) + rows = [_normalize_row(r) for r in raw_rows] + all_rows.extend(rows) + if len(all_rows) >= req.limit: + all_rows = all_rows[:req.limit] + break + + if not all_rows: + raise HTTPException(status_code=404, detail="未找到可导出的数据（任务未完成或无结果文件）") + + if req.format == "csv": + # CSV 格式：标准 CSV，comments 字段多值用 | 分隔 + output = io.StringIO() + fieldnames = ["video_id", "platform", "script_text", "likes", "favorites", "shares", "comments"] + writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore", lineterminator="\n") + writer.writeheader() + size = 0 + written = 0 + for row in all_rows: + norm = _normalize_row(row) + writer.writerow(norm) + written += 1 + size = output.tell() + if size >= MAX_EXPORT_BYTES: + break + content = output.getvalue().encode("utf-8") + media_type = "text/csv; charset=utf-8" + filename = f"export_{len(req.task_ids)}tasks_{written}rows.csv" + + else: # txt + # TXT 格式：每行 video_id||script_text||likes||favorites||shares||comments + lines: List[str] = [] + size = 0 + for row in all_rows: + norm = _normalize_row(row) + parts = [ + norm["video_id"], + norm["script_text"], + norm["likes"], + norm["favorites"], + norm["shares"], + norm["comments"], + ] + line = "||".join(parts) + "\n" + line_bytes = line.encode("utf-8") + if size + len(line_bytes) > MAX_EXPORT_BYTES: + break + lines.append(line) + size += len(line_bytes) + content = "".join(lines).encode("utf-8") + media_type = "text/plain; charset=utf-8" + filename = f"export_{len(req.task_ids)}tasks_{len(lines)}rows.txt" + + from fastapi.responses import Response + return Response( + content=content, + media_type=media_type, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +# WebSocket 路由已移至 main.py（路径：/ws/tasks）， +# 避免 /scrape 前缀导致路径不匹配。 diff --git a/api/tasks.py b/api/tasks.py new file mode 100644 index 000000000..58565cc2e --- /dev/null +++ b/api/tasks.py @@ -0,0 +1,608 @@ +""" +douyin_scraper.api.tasks — 异步任务管理 +========================================= +v6 新增：轻量级异步任务调度器。 + +我实际执行时踩过的坑： + - 采集任务动辄运行数小时，HTTP 请求会超时 + → 必须后台执行，返回 task_id 供查询 + - 多个任务同时操作同一个工作目录 + → 每个任务独立工作目录，互不干扰 + - 任务进程崩溃后无状态记录 + → 任务状态持久化到 JSON，重启后可恢复 + - 内存中存储所有任务导致重启丢失 + → 可选 SQLite 持久化（默认 JSON 文件） + +设计决策： + - 使用 threading 而非 asyncio，因为 DouyinScraper 内部全是同步 I/O + - 任务状态三态：pending → running → completed/failed + - 每个任务创建独立的 DouyinScraper 实例 + - 任务结果文件通过 task_id 关联 +""" + +import atexit +import json +import logging +import os +import shutil +import signal +import tempfile +import threading +import time as _time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +from fastapi import HTTPException + +from .utils import validate_no_symlink +from .ws import ws_manager + +logger = logging.getLogger("douyin_scraper.api") + +_CST = timezone(timedelta(hours=8)) + + +def _now_iso() -> str: + return datetime.now(_CST).isoformat() + + +def _atomic_write_json(filepath: Path, data: dict) -> None: + """ + 原子写入 JSON 文件：先写临时文件，再 os.replace 替换原文件。 + 防止写入过程中崩溃导致文件损坏。 + + Args: + filepath: 目标 JSON 文件路径 + data: 要写入的字典数据 + """ + filepath = Path(filepath) + filepath.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_path = tempfile.mkstemp( + dir=str(filepath.parent), + prefix=filepath.stem + ".tmp", + suffix=".json", + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + os.replace(tmp_path, str(filepath)) + except Exception: + # 清理临时文件 + if os.path.exists(tmp_path): + os.unlink(tmp_path) + raise + + +class TaskInfo: + """单个任务的信息""" + + def __init__( + self, + task_id: str, + task_type: str, + workspace: str, + params: Optional[dict] = None, + ) -> None: + self.task_id = task_id + self.task_type = task_type + self.workspace = workspace + self.params = params or {} + self.status: str = "pending" # pending / running / completed / failed + self.created_at: str = _now_iso() + self.started_at: Optional[str] = None + self.completed_at: Optional[str] = None + self.error: Optional[str] = None + self.exit_code: int = 0 + self.result: Optional[Dict[str, Any]] = None + self.progress: str = "" + + def to_dict(self) -> Dict[str, Any]: + return { + "task_id": self.task_id, + "task_type": self.task_type, + "workspace": self.workspace, + "params": self.params, + "status": self.status, + "created_at": self.created_at, + "started_at": self.started_at, + "completed_at": self.completed_at, + "error": self.error, + "exit_code": self.exit_code, + "result": self.result, + "progress": self.progress, + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "TaskInfo": + task = cls( + task_id=data["task_id"], + task_type=data["task_type"], + workspace=data["workspace"], + params=data.get("params"), + ) + task.status = data.get("status", "pending") + task.created_at = data.get("created_at", _now_iso()) + task.started_at = data.get("started_at") + task.completed_at = data.get("completed_at") + task.error = data.get("error") + task.exit_code = data.get("exit_code", 0) + task.result = data.get("result") + task.progress = data.get("progress", "") + return task + + +class TaskManager: + """ + 异步任务管理器。 + + ★ 我实际执行时：直接在 HTTP handler 中运行采集，导致： + 1. 请求超时（采集耗时远超 HTTP 超时） + 2. 无法并发（同一进程只能做一个采集任务） + 3. 进程崩溃全部丢失 + v6：后台线程 + 任务注册表 + 状态持久化。 + """ + + def __init__(self, base_dir: Optional[str] = None) -> None: + self._base_dir = Path(base_dir or os.environ.get( + "DY_WORKSPACE_DIR", "./workspaces" + )) + self._base_dir.mkdir(parents=True, exist_ok=True) + + # 内存任务注册表 + self._tasks: Dict[str, TaskInfo] = {} + self._lock = threading.Lock() + + # 优雅关闭：收到 SIGTERM 后等待任务完成的超时时间（秒） + self.GRACEFUL_SHUTDOWN_TIMEOUT: int = 60 + # 关闭信号标志（信号处理器设置，工作线程/轮询循环检查） + self._shutdown_flag = threading.Event() + + # 持久化文件 + self._state_file = self._base_dir / "tasks_registry.json" + + # 从持久化文件恢复 + self._load_registry() + + # atexit 注册标记（避免重复注册） + self._atexit_registered: bool = False + + # 启动时清理残留的 running 任务（上次进程异常退出遗留） + self.cleanup_stale_tasks() + + def _register_shutdown_handlers(self) -> None: + """ + 注册优雅关闭处理器（替代纯 atexit 方案）。 + + 工作流： + 1. Docker/Podman 发送 SIGTERM → 我们的 handler 收到 + 2. 设置 _shutdown_flag，通知所有轮询循环停止 + 3. 等待运行中任务完成（最多 GRACEFUL_SHUTDOWN_TIMEOUT 秒） + 4. 超时后仍未结束的标记为 failed (exit_code=4, "被中断") + 5. atexit 兜底：进程崩溃/非正常退出时仍然会标记残留任务 + """ + if self._atexit_registered: + return + self._atexit_registered = True + + def _do_graceful_shutdown(source: str) -> None: + """核心优雅关闭逻辑，信号处理器和 atexit 兜底共用""" + # 防止重复执行（SIGTERM + atexit 可能同时触发） + if self._shutdown_flag.is_set(): + return + self._shutdown_flag.set() + + running_count = sum( + 1 for t in self._tasks.values() if t.status == "running" + ) + if running_count == 0: + logger.info("[%s] 没有运行中任务，直接退出", source) + return + + logger.warning( + "[%s] 优雅关闭开始：等待 %d 个运行中任务完成（最多 %ds）...", + source, running_count, self.GRACEFUL_SHUTDOWN_TIMEOUT, + ) + + deadline = _time.monotonic() + self.GRACEFUL_SHUTDOWN_TIMEOUT + while _time.monotonic() < deadline: + with self._lock: + still_running = [ + t for t in self._tasks.values() + if t.status == "running" + ] + if not still_running: + break + _time.sleep(0.5) + + # 检查结果 + with self._lock: + killed = 0 + finished = 0 + for task in self._tasks.values(): + if task.status == "running": + task.status = "failed" + task.completed_at = _now_iso() + task.error = "任务被中断：服务正在关闭（容器重启/进程退出）" + task.exit_code = 4 # 被外部中断（区别于 3=致命错误） + killed += 1 + elif task.status == "completed": + finished += 1 + if killed > 0: + self._save_registry() + logger.info( + "[%s] 优雅关闭完成 — %d 完成, %d 被中断", + source, finished, killed, + ) + + # ── SIGTERM 处理器（Docker stop / docker-compose restart）── + def _sigterm_handler(signum: int, frame: Any) -> None: + _do_graceful_shutdown("SIGTERM") + # 恢复默认行为让进程自然退出 + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGTERM) + + # ── SIGINT 处理器（Ctrl+C）── + def _sigint_handler(signum: int, frame: Any) -> None: + _do_graceful_shutdown("SIGINT") + signal.signal(signal.SIGINT, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGINT) + + signal.signal(signal.SIGTERM, _sigterm_handler) + signal.signal(signal.SIGINT, _sigint_handler) + logger.debug("已注册 SIGTERM/SIGINT 优雅关闭处理器") + + # ── atexit 兜底：进程崩溃/非正常退出时仍然会触发 ── + def _atexit_safety_net() -> None: + if not self._shutdown_flag.is_set(): + _do_graceful_shutdown("atexit（异常退出）") + + atexit.register(_atexit_safety_net) + + def cleanup_stale_tasks(self) -> int: + """ + 清理残留的 running 任务（进程异常退出后重启时调用）。 + 将超过 1 小时仍在 running 的任务标记为 failed。 + """ + cutoff = datetime.now(_CST) - timedelta(hours=1) + stale_count = 0 + with self._lock: + for task in self._tasks.values(): + if task.status != "running": + continue + started = task.started_at or task.created_at + try: + started_dt = datetime.fromisoformat(started) + if started_dt < cutoff: + task.status = "failed" + task.completed_at = _now_iso() + task.error = "任务超时，可能因进程异常退出" + task.exit_code = 3 + stale_count += 1 + except (ValueError, TypeError): + # 无法解析时间，也标记为 failed + task.status = "failed" + task.completed_at = _now_iso() + task.error = "任务状态异常，时间无法解析" + task.exit_code = 3 + stale_count += 1 + if stale_count > 0: + self._save_registry() + if stale_count: + logger.warning("清理 %d 个残留 running 任务", stale_count) + return stale_count + + def _load_registry(self) -> None: + """从 JSON 文件恢复任务注册表""" + if not self._state_file.exists(): + return + try: + with open(self._state_file, "r", encoding="utf-8") as f: + data = json.load(f) + for task_data in data.get("tasks", []): + task = TaskInfo.from_dict(task_data) + self._tasks[task.task_id] = task + logger.info("恢复 %d 个任务记录", len(self._tasks)) + except (json.JSONDecodeError, OSError) as e: + logger.warning("任务注册表损坏，跳过恢复: %s", e) + + def _save_registry(self) -> None: + """持久化任务注册表到 JSON（原子写入）""" + self._base_dir.mkdir(parents=True, exist_ok=True) + data = { + "tasks": [t.to_dict() for t in self._tasks.values()], + "updated_at": _now_iso(), + } + try: + _atomic_write_json(self._state_file, data) + except (OSError, PermissionError): + logger.warning("任务注册表写入失败") + + def create_task( + self, + task_type: str, + params: Optional[dict] = None, + ) -> TaskInfo: + """ + 创建新任务，返回 TaskInfo。 + + Args: + task_type: search / comments / scripts / merge / run_all + params: 任务参数（keywords, model, 等） + """ + task_id = uuid.uuid4().hex[:12] + # 防止 UUID4 碰撞：最多重试 3 次 + for _ in range(3): + with self._lock: + if task_id not in self._tasks: + break + task_id = uuid.uuid4().hex[:12] + else: + raise RuntimeError("task_id 碰撞 3 次，无法生成唯一 ID") + workspace = str(self._base_dir / task_id) + + task = TaskInfo( + task_id=task_id, + task_type=task_type, + workspace=workspace, + params=params, + ) + + with self._lock: + self._tasks[task_id] = task + self._save_registry() + + logger.info("创建任务: %s (%s)", task_id, task_type) + return task + + def submit( + self, + task: TaskInfo, + func: Callable[..., Any], + *args: Any, + **kwargs: Any, + ) -> None: + """ + 提交任务到后台线程执行。 + + ★ 设计决策：使用 threading 而非 asyncio ★ + 因为 DouyinScraper 内部全是同步 I/O（subprocess、文件写入、 + faster-whisper），在 async 中调用需要 run_in_executor， + 不如直接用线程简单。 + + 线程安全：所有对 TaskInfo 属性的修改都在 self._lock 下进行。 + """ + def _worker() -> None: + with self._lock: + task.status = "running" + task.started_at = _now_iso() + self._save_registry() + + self._broadcast("task_started", task) + + try: + result = func(*args, **kwargs) + with self._lock: + task.status = "completed" + task.completed_at = _now_iso() + task.result = result if isinstance(result, dict) else {"output": str(result)} + self._save_registry() + logger.info("任务完成: %s", task.task_id) + self._broadcast("task_completed", task) + except Exception as e: + with self._lock: + task.status = "failed" + task.completed_at = _now_iso() + task.error = str(e)[:500] + + # 分类退出码 + from douyin_scraper.utils import classify_error + task.exit_code = classify_error(e) + self._save_registry() + logger.error( + "任务失败: %s (exit_code=%d): %s", + task.task_id, task.exit_code, task.error, + ) + self._broadcast("task_failed", task) + + thread = threading.Thread( + target=_worker, + name=f"task-{task.task_id}", + daemon=False, + ) + thread.start() + # 注册优雅关闭处理器：收到 SIGTERM 后等待任务完成 + self._register_shutdown_handlers() + + def get_task(self, task_id: str) -> Optional[TaskInfo]: + """查询任务状态""" + return self._tasks.get(task_id) + + def list_tasks( + self, + task_type: Optional[str] = None, + status: Optional[str] = None, + limit: int = 50, + ) -> List[TaskInfo]: + """列出任务""" + tasks = list(self._tasks.values()) + + if task_type: + tasks = [t for t in tasks if t.task_type == task_type] + if status: + tasks = [t for t in tasks if t.status == status] + + # 按创建时间降序 + tasks.sort(key=lambda t: t.created_at, reverse=True) + return tasks[:limit] + + def delete_task(self, task_id: str) -> bool: + """删除任务记录及对应 workspace 目录""" + with self._lock: + if task_id in self._tasks: + task = self._tasks[task_id] + # 先清理磁盘 workspace + workspace_path = Path(task.workspace) + if workspace_path.exists(): + try: + shutil.rmtree(str(workspace_path), ignore_errors=True) + logger.info("已清理 workspace: %s", workspace_path) + except Exception as e: + logger.warning("清理 workspace 失败: %s: %s", workspace_path, e) + del self._tasks[task_id] + self._save_registry() + return True + return False + + def cleanup_old_tasks(self, max_age_hours: int = 72) -> int: + """ + 清理超过 max_age_hours 的已完成/失败任务。 + ★ 我实际执行时：任务注册表无限增长，占用磁盘。 + """ + cutoff = datetime.now(_CST) - timedelta(hours=max_age_hours) + removed = 0 + with self._lock: + to_remove = [] + for tid, task in self._tasks.items(): + if task.status in ("completed", "failed"): + completed = task.completed_at or task.created_at + try: + completed_dt = datetime.fromisoformat(completed) + if completed_dt < cutoff: + to_remove.append(tid) + except (ValueError, TypeError): + pass + for tid in to_remove: + del self._tasks[tid] + removed += 1 + if removed: + self._save_registry() + logger.info("清理 %d 个过期任务", removed) + return removed + + def get_result_path(self, task_id: str) -> Optional[Path]: + """获取任务的结果文件路径（已验证符号链接安全）""" + task = self.get_task(task_id) + if not task or task.status != "completed": + return None + + workspace = Path(task.workspace) + + # ★ Fix 5: 优先路径 — workspace/outputs/search_result.jsonl ★ + preferred = workspace / "outputs" / "search_result.jsonl" + if preferred.exists(): + try: + validate_no_symlink(preferred) + return preferred + except HTTPException: + logger.warning("结果路径含符号链接，拒绝访问: %s", preferred) + + # 次选：workspace/outputs/ 下任意 jsonl（按修改时间倒序） + outputs_dir = workspace / "outputs" + if outputs_dir.exists(): + jsonl_files = sorted( + outputs_dir.rglob("*.jsonl"), + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + for p in jsonl_files: + try: + validate_no_symlink(p) + return p + except HTTPException: + continue + + # 根据任务类型查找结果文件 + result_path: Optional[Path] = None + if task.task_type == "merge": + # CSV 文件 — 只搜索前 2 层深度 + csv_files = [ + p for p in workspace.rglob("*.csv") + if len(p.relative_to(workspace).parts) <= 2 + ] + if csv_files: + result_path = csv_files[0] + elif task.task_type in ("search", "comments", "scripts", "run_all"): + # JSONL 文件 — 只搜索前 2 层深度（旧版兼容兜底） + jsonl_files = [ + p for p in workspace.rglob("*.jsonl") + if len(p.relative_to(workspace).parts) <= 2 + ] + if jsonl_files: + # 优先返回最新文件（忽略已被删除的） + valid_files = [] + for p in jsonl_files: + try: + valid_files.append((p, p.stat().st_mtime)) + except FileNotFoundError: + continue + if valid_files: + result_path = sorted(valid_files, key=lambda x: x[1])[-1][0] + + # 兜底：返回 workspace 目录 + if result_path is None and workspace.exists(): + result_path = workspace + + if result_path is not None: + # 符号链接安全检查 + try: + validate_no_symlink(result_path) + except HTTPException: + logger.warning("结果路径含符号链接，拒绝访问: %s", result_path) + return None + + return result_path + + def get_stats(self) -> Dict[str, Any]: + """获取任务统计（修复：枚举成员转字符串）""" + stats: Dict[str, int] = { + "total": len(self._tasks), + "pending": 0, + "running": 0, + "completed": 0, + "failed": 0, + } + for task in self._tasks.values(): + # task.status 是 TaskStatus 枚举成员，需要取 .value 得到字符串 + status_str = task.status.value if hasattr(task.status, 'value') else str(task.status) + if status_str in stats: + stats[status_str] += 1 + return stats + + def update_progress(self, task_id: str, progress: str) -> None: + """ + 更新任务进度字符串。 + + Args: + task_id: 任务 ID + progress: 进度描述文本 + """ + with self._lock: + task = self._tasks.get(task_id) + if task: + task.progress = progress + self._save_registry() + if task: + self._broadcast("task_progress", task) + + def _broadcast(self, event_type: str, task: TaskInfo) -> None: + """ + 通过 WebSocket 广播任务状态变更。 + + Args: + event_type: 事件类型 (task_started/task_progress/task_completed/task_failed) + task: 任务信息对象 + """ + message = { + "type": event_type, + "task_id": task.task_id, + "status": task.status, + "progress": task.progress, + "timestamp": _now_iso(), + } + if event_type == "task_completed" and task.result: + message["result"] = task.result + if event_type == "task_failed" and task.error: + message["error"] = task.error + ws_manager.broadcast_sync(message) diff --git a/api/ws.py b/api/ws.py new file mode 100644 index 000000000..e43cd69bf --- /dev/null +++ b/api/ws.py @@ -0,0 +1,68 @@ +""" +douyin_scraper.api.ws — WebSocket 连接管理器 +============================================== +Phase 2 新增：管理所有 /ws/tasks 客户端连接，支持广播消息。 +""" + +import asyncio +import json +import logging +from typing import Set + +from fastapi import WebSocket + +logger = logging.getLogger("douyin_scraper.api") + + +class WSManager: + """管理所有 WebSocket 连接，支持广播消息""" + + def __init__(self): + self._connections: Set[WebSocket] = set() + + async def connect(self, ws: WebSocket): + await ws.accept() + self._connections.add(ws) + logger.info("WebSocket 连接建立, 当前连接数: %d", len(self._connections)) + + def disconnect(self, ws: WebSocket): + self._connections.discard(ws) + logger.info("WebSocket 连接断开, 当前连接数: %d", len(self._connections)) + + async def broadcast(self, message: dict): + """向所有连接广播消息（异步版本，在事件循环中调用）""" + if not self._connections: + return + payload = json.dumps(message, ensure_ascii=False) + disconnected = set() + for ws in self._connections: + try: + await ws.send_text(payload) + except Exception: + disconnected.add(ws) + # 清理已断开的连接 + self._connections -= disconnected + + def broadcast_sync(self, message: dict): + """向所有连接广播消息（同步版本，从工作线程中调用）""" + if not self._connections: + return + payload = json.dumps(message, ensure_ascii=False) + # 在工作线程中创建新事件循环发送消息 + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(self.broadcast(message)) + finally: + loop.close() + except Exception as e: + logger.warning("WebSocket 同步广播失败: %s", e) + + @property + def connection_count(self) -> int: + return len(self._connections) + + +# 全局单例 +ws_manager = WSManager() diff --git a/cmd_arg/arg.py b/cmd_arg/arg.py index 074dd135f..5ce397d8e 100644 --- a/cmd_arg/arg.py +++ b/cmd_arg/arg.py @@ -299,6 +299,14 @@ def main( rich_help_panel="Storage Configuration", ), ] = config.SAVE_DATA_PATH, + output_dir: Annotated[ + str, + typer.Option( + "--output_dir", + help="Output directory for this task's results (will be used as SAVE_DATA_PATH)", + rich_help_panel="Storage Configuration", + ), + ] = "", enable_ip_proxy: Annotated[ str, typer.Option( @@ -361,6 +369,10 @@ def main( config.CRAWLER_MAX_NOTES_COUNT = crawler_max_notes_count config.MAX_CONCURRENCY_NUM = max_concurrency_num config.SAVE_DATA_PATH = save_data_path + # ★ Fix 2: --output_dir 覆盖 SAVE_DATA_PATH（优先级高于 --save_data_path）★ + if output_dir: + config.SAVE_DATA_PATH = output_dir + config.TASK_OUTPUT_DIR = output_dir config.ENABLE_IP_PROXY = enable_ip_proxy_value config.IP_PROXY_POOL_COUNT = ip_proxy_pool_count config.IP_PROXY_PROVIDER_NAME = ip_proxy_provider_name @@ -413,6 +425,8 @@ def main( cookies=config.COOKIES, specified_id=specified_id, creator_id=creator_id, + output_dir=config.SAVE_DATA_PATH, + task_output_dir=config.TASK_OUTPUT_DIR, ) command = typer.main.get_command(app) diff --git a/config/base_config.py b/config/base_config.py index 28a852e08..381cadc9a 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -18,17 +18,17 @@ # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 # Basic configuration -PLATFORM = "xhs" # Platform, xhs | dy | ks | bili | wb | tieba | zhihu +PLATFORM = "dy" # Platform, xhs | dy | ks | bili | wb | tieba | zhihu # 是否使用海外版小红书 (rednote.com) # 开启后 API 走 webapi.rednote.com，cookie 域使用 .rednote.com XHS_INTERNATIONAL = False -KEYWORDS = "编程副业,编程兼职" # Keyword search configuration, separated by English commas +KEYWORDS = "科目三,靠边停车,直线行驶" # Keyword search configuration, separated by English commas LOGIN_TYPE = "qrcode" # qrcode or phone or cookie COOKIES = "" CRAWLER_TYPE = ( - "search" # Crawling type, search (keyword search) | detail (post details) | creator (creator homepage data) + "detail" # Crawling type, search (keyword search) | detail (post details) | creator (creator homepage data) ) # Whether to enable IP proxy ENABLE_IP_PROXY = False @@ -56,7 +56,7 @@ # 是否启用 CDP 模式 - 使用用户本地的 Chrome/Edge 浏览器进行爬取，具有更好的反检测能力 # 开启后，会自动检测并启动用户的 Chrome/Edge 浏览器，通过 CDP 协议进行控制 # 该方式使用真实浏览器环境，包括用户的扩展、Cookie 和设置，大幅降低被风控检测的风险 -ENABLE_CDP_MODE = True +ENABLE_CDP_MODE = False # CDP 调试端口，用于与浏览器通信 # 如果端口被占用，系统会自动尝试下一个可用端口 @@ -66,7 +66,7 @@ # 如果为空，系统会自动检测 Chrome/Edge 的安装路径 # Windows 示例: "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" # macOS 示例: "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" -CUSTOM_BROWSER_PATH = "" +CUSTOM_BROWSER_PATH = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" # 是否在 CDP 模式下启用无头模式 # 注意：即使设置为 True，某些反检测功能在无头模式下可能无法正常工作 @@ -99,7 +99,7 @@ START_PAGE = 1 # Control the number of crawled videos/posts -CRAWLER_MAX_NOTES_COUNT = 15 +CRAWLER_MAX_NOTES_COUNT = 20 # Controlling the number of concurrent crawlers MAX_CONCURRENCY_NUM = 1 @@ -111,7 +111,7 @@ ENABLE_GET_COMMENTS = True # Control the number of crawled first-level comments (single video/post) -CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = 10 +CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = 200 # Whether to enable the mode of crawling second-level comments. By default, crawling of second-level comments is not enabled. # If the old version of the project uses db, you need to refer to schema/tables.sql line 287 to add table fields. @@ -140,6 +140,9 @@ # 警告：禁用 SSL 验证将使所有流量暴露于中间人攻击风险，请勿在生产环境中开启。 DISABLE_SSL_VERIFY = False +# Task output directory — when set, AsyncFileWriter writes flat stable-name files +TASK_OUTPUT_DIR: str = "" + from .bilibili_config import * from .xhs_config import * from .dy_config import * diff --git a/douyin_scraper/__init__.py b/douyin_scraper/__init__.py new file mode 100644 index 000000000..28a800fc4 --- /dev/null +++ b/douyin_scraper/__init__.py @@ -0,0 +1,55 @@ +""" +douyin_scraper — 抖音关键词批量采集工具（生产级 Python 模块） +========================================================= +版本: v5.0 | 日期: 2026-06-05 +本文档对应的提示词版本为 v5。 + +从 v4 脚本集合重构为可安装的 Python 模块。 + +自我反思 — v4 仍然存在的 5 个问题： + 1. 步骤间数据传递依赖 handoff.md 人工查看，未自动化 + → v5: DouyinScraper 类内部通过 self._paths 字典自动管理路径 + 2. fallback 目录硬编码为 C:/temp/dy_fallback + → v5: 通过 config.fallback_dir 配置，默认跨平台自动选择 + 3. 没有支持增量采集或并发采集 + → v5: 架构预留了增量接口，search/fetch_comments/extract_scripts + 均支持 skip_existing 参数 + 4. 日志轮转策略在 common_utils 中实现了但 extract_scripts 等脚本 + 没有调用 setup_log_rotation + → v5: DouyinScraper.__init__() 自动调用，所有方法共享 + 5. 测试用例依赖模块级全局变量（PROJECT_DIR, FALLBACK_DIR）， + 无法并行运行 + → v5: 所有状态通过实例属性管理，测试时注入临时目录 + +核心改进： + - DouyinScraper 类：统一入口，状态内部管理 + - 配置支持：字典 / .env / YAML + - 自定义异常：带 step / exit_code / details 属性 + - 跨平台：全部使用 pathlib，无 shell 特有命令 + - 可测试：所有外部依赖可 mock + - logging 替代 print + - 类型注解 + mypy --strict 兼容 +""" + +from douyin_scraper.core import DouyinScraper +from douyin_scraper.exceptions import ( + RetryableError, + NonRetryableError, + ConfigError, + FatalError, +) +from douyin_scraper.models import Video, Comment, Script +from douyin_scraper.config import ScraperConfig + +__version__ = "5.0.0" +__all__ = [ + "DouyinScraper", + "RetryableError", + "NonRetryableError", + "ConfigError", + "FatalError", + "Video", + "Comment", + "Script", + "ScraperConfig", +] diff --git a/douyin_scraper/config.py b/douyin_scraper/config.py new file mode 100644 index 000000000..4ddc66138 --- /dev/null +++ b/douyin_scraper/config.py @@ -0,0 +1,233 @@ +""" +douyin_scraper.config — 配置管理 +================================= +v5 新增：统一配置管理，支持字典 / .env 文件 / YAML。 +我实际执行时：配置散落在 base_config.py、命令行参数、环境变量中， +无法集中管理和验证。 +""" + +import os +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +from douyin_scraper.exceptions import ConfigError + +# 跨平台默认 fallback 目录 +if sys.platform == "win32": + _DEFAULT_FALLBACK = "C:/temp/dy_fallback" +else: + _DEFAULT_FALLBACK = "/tmp/dy_fallback" + + +class RetryConfig: + """重试配置""" + max_attempts: int = 3 + base_delay: float = 5.0 + backoff_factor: float = 3.0 + + def __init__(self, **kwargs: Any) -> None: + for k, v in kwargs.items(): + if hasattr(self, k): + setattr(self, k, v) + + +class ScraperConfig: + """ + 采集工具配置。 + + 支持三种初始化方式： + 1. 字典：ScraperConfig({"project_dir": "/path", "keywords": ["xxx"]}) + 2. .env 文件路径：ScraperConfig("/path/to/.env") + 3. 默认值：ScraperConfig() + + 我实际执行时：配置散落在各处，新机器部署时容易遗漏。 + v5：集中管理 + 验证 + 默认值。 + """ + + def __init__(self, config: Optional[Union[str, Path, dict]] = None) -> None: + # 默认值 + self.project_dir: Path = Path.cwd() / "douyin_scraper_workspace" + self.keywords: List[str] = [] + self.max_videos_per_keyword: int = 20 + self.enable_comments: bool = True + self.enable_cdp_mode: bool = True + self.chrome_debugging_port: int = 9222 + self.ffmpeg_backend: str = "imageio-ffmpeg" # or "system" + self.retry: RetryConfig = RetryConfig() + self.log_level: str = "INFO" + self.fallback_dir: Path = Path(_DEFAULT_FALLBACK) + self.state_dir_name: str = "state" + self.whisper_model: str = "small" + self.keep_videos: bool = False + self.max_workers: int = 1 + + # 加载配置 + if config is not None: + self._load(config) + + # 环境变量覆盖（DOUYIN_SCRAPER_ 前缀） + self._load_env_vars() + + def _load(self, config: Union[str, Path, dict]) -> None: + """加载配置源""" + if isinstance(config, dict): + self._apply_dict(config) + elif isinstance(config, (str, Path)): + path = Path(config) + if not path.exists(): + raise ConfigError( + f"配置文件不存在: {path}", + step="config", + details={"path": str(path)}, + ) + if path.suffix == ".env": + self._load_dotenv(path) + elif path.suffix in (".yaml", ".yml"): + self._load_yaml(path) + else: + raise ConfigError( + f"不支持的配置文件格式: {path.suffix}", + step="config", + details={"path": str(path)}, + ) + + def _apply_dict(self, data: dict) -> None: + """应用字典配置""" + if "project_dir" in data: + self.project_dir = Path(data["project_dir"]) + if "keywords" in data: + self.keywords = list(data["keywords"]) + if "max_videos_per_keyword" in data: + try: + self.max_videos_per_keyword = int(data["max_videos_per_keyword"]) + except (ValueError, TypeError) as e: + raise ConfigError( + f"max_videos_per_keyword 必须是整数: {data.get('max_videos_per_keyword')}", + step="config", + ) from e + if "enable_comments" in data: + self.enable_comments = bool(data["enable_comments"]) + if "enable_cdp_mode" in data: + self.enable_cdp_mode = bool(data["enable_cdp_mode"]) + if "chrome_debugging_port" in data: + try: + self.chrome_debugging_port = int(data["chrome_debugging_port"]) + except (ValueError, TypeError) as e: + raise ConfigError( + f"chrome_debugging_port 必须是整数: {data.get('chrome_debugging_port')}", + step="config", + ) from e + if "ffmpeg_backend" in data: + self.ffmpeg_backend = str(data["ffmpeg_backend"]) + if "log_level" in data: + self.log_level = str(data["log_level"]) + if "fallback_dir" in data: + self.fallback_dir = Path(data["fallback_dir"]) + if "whisper_model" in data: + self.whisper_model = str(data["whisper_model"]) + if "keep_videos" in data: + self.keep_videos = bool(data["keep_videos"]) + if "max_workers" in data: + try: + self.max_workers = int(data["max_workers"]) + except (ValueError, TypeError) as e: + raise ConfigError( + f"max_workers 必须是整数: {data.get('max_workers')}", + step="config", + ) from e + if "state_dir_name" in data: + self.state_dir_name = str(data["state_dir_name"]) + if "retry" in data: + self.retry = RetryConfig(**data["retry"]) + + def _load_dotenv(self, path: Path) -> None: + """ + 加载 .env 文件。 + 格式：KEY=VALUE，支持 DOUYIN_SCRAPER_ 前缀。 + """ + data: Dict[str, Any] = {} + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + key, _, value = line.partition("=") + key = key.strip().lower() + value = value.strip().strip('"').strip("'") + data[key] = value + self._apply_dict(data) + + def _load_yaml(self, path: Path) -> None: + """加载 YAML 配置文件（需要 PyYAML）""" + try: + import yaml # type: ignore[import-untyped] + except ImportError: + raise ConfigError( + "PyYAML 未安装，请执行: pip install pyyaml", + step="config", + ) + with open(path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) or {} + self._apply_dict(data) + + def _load_env_vars(self) -> None: + """从环境变量加载配置（DOUYIN_SCRAPER_ 前缀）""" + prefix = "DOUYIN_SCRAPER_" + mapping: Dict[str, str] = {} + for key, value in os.environ.items(): + if key.startswith(prefix): + clean_key = key[len(prefix):].lower() + mapping[clean_key] = value + + if mapping: + self._apply_dict(mapping) + + @property + def state_dir(self) -> Path: + return self.project_dir / self.state_dir_name + + @property + def data_dir(self) -> Path: + return self.project_dir / "data" / "douyin" + + @property + def jsonl_dir(self) -> Path: + return self.data_dir / "jsonl" + + @property + def video_dir(self) -> Path: + return self.data_dir / "downloaded_videos" + + def validate(self) -> None: + """ + 验证配置完整性。 + 我实际执行时：配置错误到运行时才发现（如没有关键词就执行搜索）。 + """ + errors: List[str] = [] + if self.project_dir is None or not str(self.project_dir).strip(): + errors.append("project_dir 未设置") + if self.ffmpeg_backend not in ("imageio-ffmpeg", "system"): + errors.append(f"不支持的 ffmpeg_backend: {self.ffmpeg_backend}") + if self.chrome_debugging_port < 1 or self.chrome_debugging_port > 65535: + errors.append(f"无效端口: {self.chrome_debugging_port}") + if self.retry.max_attempts < 1: + errors.append("retry.max_attempts 必须 >= 1") + if self.max_videos_per_keyword < 1: + errors.append(f"max_videos_per_keyword 必须 >= 1, 当前: {self.max_videos_per_keyword}") + if not isinstance(self.keywords, list): + errors.append(f"keywords 必须是列表, 当前类型: {type(self.keywords).__name__}") + elif self.keywords: + empty_keywords = [k for k in self.keywords if not k.strip()] + if empty_keywords: + errors.append(f"keywords 包含 {len(empty_keywords)} 个空字符串") + if self.whisper_model not in ("tiny", "base", "small", "medium", "large"): + errors.append(f"不支持的 whisper_model: {self.whisper_model}") + if errors: + raise ConfigError( + "配置验证失败: " + "; ".join(errors), + step="config", + details={"errors": errors}, + ) diff --git a/douyin_scraper/core.py b/douyin_scraper/core.py new file mode 100644 index 000000000..d741af226 --- /dev/null +++ b/douyin_scraper/core.py @@ -0,0 +1,1155 @@ +""" +douyin_scraper.core — 核心采集引擎 +=================================== +v5 核心：DouyinScraper 类，统一入口，状态内部管理。 + +对外 API： + - DouyinScraper(config) → 初始化 + - .setup() → 环境准备 + - .search() → 搜索采集 + - .fetch_comments() → 评论采集 + - .extract_scripts() → 文案提取 + - .merge() → 合并数据 + - .run_all() → 一键执行 + - .get_status() → 状态查询 + - .reset_step() → 重置步骤 + +v4 → v5 关键变更： + - 全局变量 → 实例属性 + - print → logging + - 独立脚本 → 类方法 + - 手动管理路径 → self._paths 自动管理 + - 无配置验证 → ScraperConfig.validate() +""" + +import csv +import json +import logging +import os +import random +import shutil +import subprocess +import sys +import tempfile +import threading +import time +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Set, Union + +from douyin_scraper.config import ScraperConfig +from douyin_scraper.exceptions import ( + ConfigError, + FatalError, + NonRetryableError, + RetryableError, + ScraperError, +) +from douyin_scraper.models import Comment, Script, Video +from douyin_scraper.state import StateManager +from douyin_scraper.utils import ( + EXIT_FATAL, + EXIT_NON_RETRYABLE, + EXIT_RETRYABLE, + append_record, + check_disk_space_enforced, + check_port_in_use, + classify_error, + ensure_dir_writable, + kill_process_on_port, + parse_count, + retry, + retry_with_degradation, + safe_write_line, + setup_env_vars, + setup_ffmpeg, + setup_log_rotation, +) + +logger = logging.getLogger("douyin_scraper") + + +class DouyinScraper: + """ + 抖音关键词批量采集工具 — 主入口类。 + + 用法: + scraper = DouyinScraper({"project_dir": "G:/MediaCrawler"}) + scraper.setup() + scraper.search(keywords=["短视频运营"]) + scraper.fetch_comments() + scraper.extract_scripts() + scraper.merge() + + 或者一键运行: + scraper = DouyinScraper() + result = scraper.run_all() + """ + + # 步骤名称常量 + STEP_CLONE = "clone_repo" + STEP_SETUP_ENV = "setup_env" + STEP_CONFIG = "config_douyin" + STEP_SEARCH = "run_search" + STEP_COMMENTS = "fetch_comments" + STEP_FFMPEG = "install_ffmpeg" + STEP_WHISPER = "install_whisper" + STEP_EXTRACT = "run_extract" + STEP_MERGE = "merge_csv" + + ALL_STEPS = [ + STEP_CLONE, + STEP_SETUP_ENV, + STEP_CONFIG, + STEP_SEARCH, + STEP_COMMENTS, + STEP_FFMPEG, + STEP_WHISPER, + STEP_EXTRACT, + STEP_MERGE, + ] + + def __init__( + self, + config: Optional[Union[str, Path, dict]] = None, + ) -> None: + # 加载配置 + self._config = ScraperConfig(config) + # 不在这里 validate，因为 keywords 等参数是后续通过 search() 传入的 + # validate 将在 search() 方法中调用 + + # 初始化状态管理 + self._state = StateManager(self._config.state_dir) + + # 确保关键目录存在 + self._config.project_dir.mkdir(parents=True, exist_ok=True) + + # 内部路径管理（★ 替代 v4 的 handoff.md ★） + self._paths: Dict[str, Path] = {} + + # 可中断等待事件（★ 替代 time.sleep 实现可取消等待 ★） + self._cancel_event = threading.Event() + + # 日志轮转（★ v5: 自动调用 ★） + log_path = self._config.state_dir / "execution_log.jsonl" + setup_log_rotation(log_path) + + # 配置 logging 输出到控制台 + if not any( + isinstance(h, logging.StreamHandler) and not isinstance(h, logging.FileHandler) + for h in logger.handlers + ): + console = logging.StreamHandler() + console.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) + console.setLevel(getattr(logging, self._config.log_level, logging.INFO)) + logger.addHandler(console) + + logger.info("DouyinScraper 初始化完成, project_dir=%s", self._config.project_dir) + + def __del__(self) -> None: + """析构时确保去重索引已刷盘""" + try: + if hasattr(self, '_state') and self._state is not None: + self._state.flush_dedupe() + except Exception: + pass + + def cancel(self) -> None: + """取消正在执行的操作，中断等待中的 sleep。""" + self._cancel_event.set() + logger.info("收到取消请求，正在中断等待…") + + @property + def is_cancelled(self) -> bool: + """是否已被取消""" + return self._cancel_event.is_set() + + @property + def config(self) -> ScraperConfig: + return self._config + + @property + def state(self) -> StateManager: + return self._state + + # ═══════════════════════════════════════════════════════════════ + # 公共 API + # ═══════════════════════════════════════════════════════════════ + + def setup(self) -> bool: + """ + 环境准备：克隆仓库、安装依赖、检查 Chrome CDP。 + 幂等：重复调用不会重复操作。 + failed 状态的步骤会重新尝试（修复逻辑反转：原来 failed 被跳过）。 + """ + steps = [self.STEP_CLONE, self.STEP_SETUP_ENV, self.STEP_CONFIG] + for step_name in steps: + status = self._state.get_step_status(step_name) + # completed → 跳过; in_progress → 重试; failed → 重试; pending → 执行 + if status == "completed": + logger.info("[%s] 步骤已完成，跳过", step_name) + continue + self._state.mark_step_started(step_name) + try: + if step_name == self.STEP_CLONE: + self._do_clone() + elif step_name == self.STEP_SETUP_ENV: + self._do_setup_env() + elif step_name == self.STEP_CONFIG: + self._do_config() + self._state.mark_step_completed(step_name) + except ScraperError as e: + self._state.mark_step_failed( + step_name, error_summary=str(e), exit_code=e.exit_code + ) + return False + except Exception as e: + exit_code = classify_error(e) + self._state.mark_step_failed( + step_name, error_summary=str(e)[:200], exit_code=exit_code + ) + return False + return True + + def search( + self, + keywords: Optional[List[str]] = None, + max_count: Optional[int] = None, + ) -> Path: + """ + 执行搜索采集，返回视频元数据 JSONL 路径。 + + Args: + keywords: 搜索关键词列表（默认使用配置中的 keywords） + max_count: 每个关键词最大采集数（默认使用配置值） + """ + step = self.STEP_SEARCH + self._require_step_ready(step) + + if keywords: + self._config.keywords = keywords + if max_count: + self._config.max_videos_per_keyword = max_count + + # ★ Fix 1: validate 在 keywords/max_count 设置之后再校验 ★ + self._config.validate() + + logger.info("DouyinScraper.search keywords=%r max_count=%s", + self._config.keywords, self._config.max_videos_per_keyword) + + self._state.mark_step_started(step) + try: + output_path = self._do_search() + self._state.mark_step_completed( + step, detail=f"output={output_path}" + ) + self._paths["video_jsonl"] = output_path + return output_path + except Exception as e: + exit_code = classify_error(e) + self._state.mark_step_failed(step, str(e)[:200], exit_code) + raise + + def fetch_comments( + self, + video_jsonl: Optional[Path] = None, + ) -> Path: + """ + 对已有视频采集评论，返回评论 JSONL 路径。 + """ + step = self.STEP_COMMENTS + self._require_step_ready(step) + + input_path = video_jsonl or self._paths.get("video_jsonl") + if not input_path or not input_path.exists(): + raise NonRetryableError( + f"视频 JSONL 不存在: {input_path}", + step=step, + ) + + self._state.mark_step_started(step) + try: + output_path = self._do_fetch_comments(input_path) + self._state.mark_step_completed( + step, detail=f"output={output_path}" + ) + self._paths["comments_jsonl"] = output_path + return output_path + except Exception as e: + exit_code = classify_error(e) + self._state.mark_step_failed(step, str(e)[:200], exit_code) + raise + + def extract_scripts( + self, + video_jsonl: Optional[Path] = None, + model: str = "small", + ) -> Path: + """ + 提取文案，返回带文案的 JSONL 路径。 + """ + step = self.STEP_EXTRACT + self._require_step_ready(step) + + input_path = video_jsonl or self._paths.get("video_jsonl") + if not input_path or not input_path.exists(): + raise NonRetryableError( + f"视频 JSONL 不存在: {input_path}", + step=step, + ) + + # 前置依赖检查 + setup_env_vars() + if not setup_ffmpeg(): + raise NonRetryableError("ffmpeg 不可用", step=step) + + self._state.mark_step_started(step) + try: + output_path = self._do_extract_scripts(input_path, model) + self._state.mark_step_completed( + step, detail=f"output={output_path}" + ) + self._paths["scripts_jsonl"] = output_path + return output_path + except Exception as e: + exit_code = classify_error(e) + self._state.mark_step_failed(step, str(e)[:200], exit_code) + raise + + def merge( + self, + video_jsonl: Optional[Path] = None, + comments_jsonl: Optional[Path] = None, + scripts_jsonl: Optional[Path] = None, + output_csv: Optional[Path] = None, + ) -> Path: + """ + 合并数据生成 CSV，返回 CSV 路径。 + """ + step = self.STEP_MERGE + self._require_step_ready(step) + + v_path = video_jsonl or self._paths.get("video_jsonl") + c_path = comments_jsonl or self._paths.get("comments_jsonl") + s_path = scripts_jsonl or self._paths.get("scripts_jsonl") + + self._state.mark_step_started(step) + try: + csv_path = self._do_merge(v_path, c_path, s_path, output_csv) + self._state.mark_step_completed( + step, detail=f"output={csv_path}" + ) + return csv_path + except Exception as e: + exit_code = classify_error(e) + self._state.mark_step_failed(step, str(e)[:200], exit_code) + raise + + def run_all( + self, + steps: Optional[List[str]] = None, + ) -> Dict[str, Any]: + """ + 按顺序执行全部或指定步骤，返回结果摘要。 + """ + target_steps = steps or self.ALL_STEPS + results: Dict[str, Any] = {} + + for step_name in target_steps: + try: + if step_name == self.STEP_CLONE: + ok = self.setup() # setup 包含前3步 + elif step_name == self.STEP_SEARCH: + path = self.search() + results["video_jsonl"] = str(path) + elif step_name == self.STEP_COMMENTS: + path = self.fetch_comments() + results["comments_jsonl"] = str(path) + elif step_name == self.STEP_EXTRACT: + path = self.extract_scripts() + results["scripts_jsonl"] = str(path) + elif step_name == self.STEP_MERGE: + path = self.merge() + results["csv_path"] = str(path) + elif step_name == self.STEP_FFMPEG: + setup_ffmpeg() + elif step_name == self.STEP_WHISPER: + self._do_install_whisper() + elif step_name == self.STEP_CONFIG: + pass # 已在 setup 中处理 + elif step_name == self.STEP_SETUP_ENV: + pass # 已在 setup 中处理 + except ScraperError as e: + results["error"] = str(e) + results["error_step"] = step_name + results["exit_code"] = e.exit_code + break + except Exception as e: + results["error"] = str(e) + results["error_step"] = step_name + results["exit_code"] = classify_error(e) + break + + results["status"] = self._state.get_all_status() + return results + + def get_status(self) -> Dict[str, Any]: + """获取当前任务状态""" + return self._state.get_all_status() + + def reset_step(self, step: str, clear_dedupe: bool = False) -> None: + """重置某步骤状态""" + self._state.reset_step(step, clear_dedupe=clear_dedupe) + + # ═══════════════════════════════════════════════════════════════ + # 内部实现 + # ═══════════════════════════════════════════════════════════════ + + def _require_step_ready(self, step: str) -> None: + """检查步骤是否可执行，否则抛异常""" + if not self._state.check_step_ready(step): + status = self._state.get_step_status(step) + if status == "completed": + raise NonRetryableError( + f"步骤 {step} 已完成，跳过（使用 reset_step 重跑）", + step=step, + ) + if status == "failed": + info = self._state.get_step_info(step) + logger.warning( + "[%s] 上次失败 (%s)，自动重置后重试", + step, info.get("error_summary", "未知"), + ) + self._state.reset_step(step, clear_dedupe=False) + + def _do_clone(self) -> None: + """克隆 MediaCrawler 仓库""" + project_dir = self._config.project_dir + if (project_dir / ".git").exists(): + logger.info("仓库已存在: %s", project_dir) + return + # 如果目录存在但没有 .git，备份 + if project_dir.exists() and not (project_dir / ".git").exists(): + backup = project_dir.parent / f"MediaCrawler_backup_{int(time.time())}" + logger.warning("发现残留目录，备份到: %s", backup) + shutil.move(str(project_dir), str(backup)) + + subprocess.run( + ["git", "clone", "https://github.com/NanmiCoder/MediaCrawler.git", + str(project_dir)], + check=True, timeout=300, + ) + + def _do_setup_env(self) -> None: + """安装依赖""" + project_dir = self._config.project_dir + venv_python = self._get_venv_python() + + if venv_python.exists(): + result = subprocess.run( + [str(venv_python), "-c", "import httpx; print('OK')"], + capture_output=True, text=True, + ) + if result.returncode == 0: + logger.info("依赖已安装") + return + + # 尝试 uv sync + try: + subprocess.run( + ["uv", "sync"], cwd=str(project_dir), + check=True, timeout=300, + ) + except (subprocess.CalledProcessError, FileNotFoundError): + logger.warning("uv sync 失败，尝试 pip fallback") + if not venv_python.exists(): + subprocess.run( + [sys.executable, "-m", "venv", str(project_dir / ".venv")], + check=True, + ) + subprocess.run( + [str(venv_python), "-m", "pip", "install", "-r", + str(project_dir / "requirements.txt")], + check=False, + ) + + setup_env_vars() + + def _do_config(self) -> None: + """配置抖音搜索参数""" + config_path = self._config.project_dir / "config" / "base_config.py" + if not config_path.exists(): + raise NonRetryableError( + f"配置文件不存在: {config_path}", + step=self.STEP_CONFIG, + ) + logger.info("配置文件: %s (请手动确认)", config_path) + + def _do_search(self) -> Path: + """执行搜索采集""" + venv_python = self._get_venv_python() + project_dir = self._config.project_dir + + # ★ Fix 3: 计算 task output_dir — 在 state_dir 的父目录下创建 outputs/ ★ + state_parent = self._config.project_dir / Path(self._config.state_dir_name).parent + output_dir = state_parent / "outputs" + output_dir.mkdir(parents=True, exist_ok=True) + + jsonl_dir = project_dir / "data" / "douyin" / "jsonl" + + # ★ 子进程同一天写入同一个文件（search_contents_YYYY-MM-DD.jsonl 追加模式）。 + # 为防止新旧任务数据混在一起，先记下当前文件行数，跑完只取增量。 + global_output = None + existing_count = 0 + for f in jsonl_dir.glob("search_contents_*.jsonl"): + if "_with_scripts" in f.name or "_comments" in f.name: + continue + global_output = f + with open(f, "r", encoding="utf-8") as fh: + existing_count = sum(1 for _ in fh) + break # 只处理最新文件 + if existing_count: + logger.info("搜索前全局文件已有 %d 条数据（将被过滤，只取本次增量）", existing_count) + + # 检查 CDP 端口 — Docker 中可能没有预先启动的 Chrome + cdp_ready = check_port_in_use(self._config.chrome_debugging_port) + if not cdp_ready: + if self._config.enable_cdp_mode: + raise RetryableError( + f"Chrome CDP 端口 {self._config.chrome_debugging_port} 未就绪，" + "请先启动 Chrome: chrome.exe --remote-debugging-port=9222", + step=self.STEP_SEARCH, + ) + else: + logger.info("CDP 模式已禁用，使用 headless 浏览器") + + # 构建命令行参数 + keywords_str = ",".join(self._config.keywords) + # Cookie 优先级：1) 持久化文件 2) 环境变量 3) qrcode（Docker 中会失败） + cookie_file = Path("/app/data/douyin_cookie.txt") + douyin_cookie = "" + if cookie_file.exists(): + try: + _file_cookie = cookie_file.read_text(encoding="utf-8").strip() + if _file_cookie: + douyin_cookie = _file_cookie + logger.info("使用持久化 Cookie 文件登录，Cookie 长度: %d", len(douyin_cookie)) + except OSError as _e: + logger.warning("读取 Cookie 文件失败: %s", _e) + if not douyin_cookie and os.environ.get("DOUYIN_COOKIE"): + douyin_cookie = os.environ.get("DOUYIN_COOKIE", "") + logger.info("使用环境变量 DOUYIN_COOKIE 登录，Cookie 长度: %d", len(douyin_cookie)) + + if douyin_cookie: + login_type = "cookie" + logger.info("Cookie 登录模式，Cookie 长度: %d", len(douyin_cookie)) + else: + login_type = "qrcode" + logger.warning( + "未找到 Cookie，将使用 qrcode 登录" + "（Docker headless 模式中可能失败，请先通过 POST /login/qrcode/start 扫码登录）" + ) + cmd = [ + str(venv_python), str(project_dir / "main.py"), + "--platform", "dy", + "--lt", login_type, + "--type", "search", + "--keywords", keywords_str, + "--crawler_max_notes_count", str(self._config.max_videos_per_keyword), + "--save_data_option", "jsonl", + "--headless", "true", + "--get_comment", "false", + "--output_dir", str(output_dir), + ] + if douyin_cookie: + cmd.extend(["--cookies", douyin_cookie]) + logger.info("运行搜索子进程: %s (cwd=%s)", " ".join(cmd), project_dir) + + # ★ Fix 7: 构建子进程环境变量，禁用登录状态持久化以避免任务间互相干扰 ★ + env = os.environ.copy() + env["PYTHONUTF8"] = "1" + env["PYTHONIOENCODING"] = "utf-8" + env["LANG"] = "C.UTF-8" + env["LC_ALL"] = "C.UTF-8" + env["MEDIACRAWLER_SAVE_LOGIN_STATE"] = "false" + + logger.info("subprocess cmd keywords=%r", keywords_str) + + # 运行 MediaCrawler + subprocess.run( + cmd, + cwd=str(project_dir), check=True, timeout=1800, + env=env, + ) + + # ★ Fix 4: 优先直接使用 output_dir 下的稳定文件名 ★ + target = output_dir / "search_result.jsonl" + if target.exists() and target.stat().st_size > 0: + logger.info("搜索子进程完成，输出: %s (size=%d)", target, target.stat().st_size) + return target + + # 兼容：搜索 output_dir 下任意 jsonl + output_jsonls = list(output_dir.rglob("*.jsonl")) + if output_jsonls: + latest = sorted(output_jsonls, key=lambda p: p.stat().st_mtime)[-1] + if latest != target: + shutil.copy2(str(latest), str(target)) + logger.info("搜索子进程完成，输出: %s (来自 %s)", target, latest) + return target + + # Fallback：全局目录（极不安全，仅当 output_dir 完全未生效时） + logger.warning("WARNING: fallback to global data/douyin/jsonl latest file, this is unsafe for concurrent tasks") + if jsonl_dir.exists(): + files = [ + f for f in jsonl_dir.glob("search_contents_*.jsonl") + if "_with_scripts" not in f.name and "_comments" not in f.name + ] + if files: + latest = sorted(files)[-1] + # 从全局文件中提取增量数据，写入任务专属 workspace + new_lines = self._extract_new_lines(latest, existing_count) + state_parent.mkdir(parents=True, exist_ok=True) + task_output = state_parent / "search_result.jsonl" + with open(task_output, "w", encoding="utf-8") as outf: + for line in new_lines: + outf.write(line) + logger.info("搜索子进程完成（fallback），本次采集 %d 条，输出: %s", len(new_lines), task_output) + return task_output + + raise NonRetryableError("搜索采集未产出结果文件", step=self.STEP_SEARCH) + + def _extract_new_lines(self, file_path: Path, skip_count: int) -> List[str]: + """读取文件，跳过前 skip_count 行，返回剩余行""" + lines: List[str] = [] + with open(file_path, "r", encoding="utf-8") as f: + for i, line in enumerate(f): + if i < skip_count: + continue + if line.strip(): + lines.append(line) + return lines + + def _do_fetch_comments(self, input_path: Path) -> Path: + """ + 对已有视频采集评论。 + v5: 内置评论采集逻辑，不再依赖外部脚本。 + """ + output_dir = self._config.jsonl_dir + actual_dir = ensure_dir_writable(output_dir, self._config.fallback_dir) + output_path = actual_dir / "search_comments.jsonl" + + # 读取视频列表 + videos = self._load_jsonl(input_path) + video_ids = [str(v.get("aweme_id", "")) for v in videos if v.get("aweme_id")] + + # 断点续传 + done_ids = self._state.load_completed_ids_from_jsonl(output_path, "aweme_id") + + logger.info("视频 %d 个, 已完成评论 %d 个", len(video_ids), len(done_ids)) + + # 这里只是占位：实际评论采集需要 Chrome CDP 交互 + # MediaCrawler 的 crawl_comments_v2.py 负责实际采集 + # v5 通过子进程调用 + venv_python = self._get_venv_python() + comments_script = self._config.project_dir / "crawl_comments_v2.py" + + if comments_script.exists(): + logger.info("调用评论采集子进程: %s", comments_script) + try: + subprocess.run( + [str(venv_python), str(comments_script)], + cwd=str(self._config.project_dir), + check=True, timeout=3600, + ) + except subprocess.CalledProcessError as exc: + logger.error( + "评论采集子进程失败 (exit %d): %s", + exc.returncode, exc, + ) + raise NonRetryableError( + f"评论采集子进程失败 (exit {exc.returncode}): {exc}", + step="fetch_comments", + ) from exc + + return output_path + + def _do_extract_scripts(self, input_path: Path, model: str) -> Path: + """ + 提取文案：下载视频 → ASR 转写 → 写入 JSONL。 + v5: 内置提取逻辑，不再依赖 extract_scripts_v2.py。 + """ + # 准备目录 + video_dir = ensure_dir_writable( + self._config.video_dir, self._config.fallback_dir + ) + output_dir = ensure_dir_writable( + input_path.parent, self._config.fallback_dir + ) + output_path = output_dir / f"{input_path.stem}_with_scripts.jsonl" + + # 磁盘空间预检 + check_disk_space_enforced(video_dir, min_gb=2.0) + + # 读取输入 + records = self._load_jsonl(input_path) + valid = [r for r in records if r.get("video_download_url")] + logger.info("有视频链接的: %d 条", len(valid)) + + # 断点续传 + done_ids = self._state.load_completed_ids_from_jsonl(output_path, "aweme_id") + + # 加载 Whisper 模型 + try: + from faster_whisper import WhisperModel + whisper_model = WhisperModel(model, device="cpu", compute_type="int8") + except ImportError: + raise NonRetryableError( + "faster-whisper 未安装", step=self.STEP_EXTRACT + ) + except MemoryError: + raise FatalError("内存不足，无法加载 Whisper 模型", step=self.STEP_EXTRACT) + + success_count = 0 + fail_count = 0 + + max_workers = getattr(self._config, "max_workers", 1) + # 确保 max_workers 合法 + if not isinstance(max_workers, int) or max_workers < 1: + max_workers = 1 + + if max_workers <= 1: + # 串行模式（保持原有行为） + for i, rec in enumerate(valid, 1): + if self._cancel_event.is_set(): + logger.warning("文案提取被取消") + break + + aweme_id = str(rec.get("aweme_id", "")) + if not aweme_id or aweme_id == "None": + aweme_id = self._state.generate_pseudo_id(i, rec, prefix="video") + rec["aweme_id"] = aweme_id + + if aweme_id in done_ids or self._state.is_duplicate("script_extract", aweme_id): + continue + + logger.info("[%d/%d] %s", i, len(valid), aweme_id) + + # 每 10 条检查磁盘 + if i % 10 == 0: + try: + check_disk_space_enforced(video_dir, min_gb=0.5) + except FatalError as e: + self._state.mark_step_failed( + self.STEP_EXTRACT, str(e), EXIT_FATAL + ) + raise + + # 下载 + video_file = video_dir / f"{aweme_id}.mp4" + try: + video_size = video_file.stat().st_size + except FileNotFoundError: + video_size = 0 + if not (video_file.exists() and video_size > 10000): + try: + ok = self._download_video( + rec.get("video_download_url", ""), str(video_file) + ) + except Exception as e: + logger.error("下载重试耗尽: %s: %s", aweme_id, e) + ok = False + + if not ok: + rec["script_text"] = "" + rec["script_status"] = "download_failed" + append_record(output_path, rec, self._config.fallback_dir) + self._state.mark_written("script_extract", aweme_id) + fail_count += 1 + continue + + # 转写 + script_text = self._transcribe_video(str(video_file), whisper_model) + if script_text: + rec["script_text"] = script_text + rec["script_status"] = "success" + success_count += 1 + else: + rec["script_text"] = "" + rec["script_status"] = "asr_failed" + fail_count += 1 + + # ★ 立即写入 ★ + append_record(output_path, rec, self._config.fallback_dir) + self._state.mark_written("script_extract", aweme_id) + + # 删除视频节省磁盘 + if not self._config.keep_videos and video_file.exists(): + video_file.unlink() + else: + # 并发模式 + def _process_one(idx: int, rec: dict) -> Dict[str, Any]: + """处理单条视频，返回结果字典""" + aweme_id = str(rec.get("aweme_id", "")) + if not aweme_id or aweme_id == "None": + aweme_id = self._state.generate_pseudo_id(idx, rec, prefix="video") + rec["aweme_id"] = aweme_id + + if aweme_id in done_ids or self._state.is_duplicate("script_extract", aweme_id): + return {"action": "skip"} + + logger.info("[%d/%d] %s", idx, len(valid), aweme_id) + + # 下载 + video_file = video_dir / f"{aweme_id}.mp4" + try: + video_size = video_file.stat().st_size + except FileNotFoundError: + video_size = 0 + if not (video_file.exists() and video_size > 10000): + try: + ok = self._download_video( + rec.get("video_download_url", ""), str(video_file) + ) + except Exception as e: + logger.error("下载重试耗尽: %s: %s", aweme_id, e) + ok = False + + if not ok: + return {"action": "fail", "aweme_id": aweme_id, "status": "download_failed"} + + # 转写 + script_text = self._transcribe_video(str(video_file), whisper_model) + if script_text: + rec["script_text"] = script_text + rec["script_status"] = "success" + else: + rec["script_text"] = "" + rec["script_status"] = "asr_failed" + + # 立即写入（线程安全：append_record 内部有原子写入） + append_record(output_path, rec, self._config.fallback_dir) + self._state.mark_written("script_extract", aweme_id) + + # 删除视频节省磁盘 + if not self._config.keep_videos and video_file.exists(): + try: + video_file.unlink() + except OSError: + pass + + if rec["script_status"] == "success": + return {"action": "success", "aweme_id": aweme_id} + else: + return {"action": "fail", "aweme_id": aweme_id, "status": "asr_failed"} + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = {} + for i, rec in enumerate(valid, 1): + if self._cancel_event.is_set(): + logger.warning("文案提取被取消，不再提交新任务") + break + # 磁盘检查（仅主线程） + if i % 10 == 0: + try: + check_disk_space_enforced(video_dir, min_gb=0.5) + except FatalError as e: + self._state.mark_step_failed( + self.STEP_EXTRACT, str(e), EXIT_FATAL + ) + raise + future = executor.submit(_process_one, i, rec) + futures[future] = i + + for future in as_completed(futures): + if self._cancel_event.is_set(): + break + try: + result = future.result() + if result["action"] == "success": + success_count += 1 + elif result["action"] == "fail": + fail_count += 1 + if result.get("status") == "download_failed": + # 写入下载失败记录 + for rec in valid: + if str(rec.get("aweme_id", "")) == result["aweme_id"]: + rec["script_text"] = "" + rec["script_status"] = "download_failed" + append_record(output_path, rec, self._config.fallback_dir) + self._state.mark_written("script_extract", result["aweme_id"]) + break + except Exception as e: + logger.error("并发处理异常: %s", e) + fail_count += 1 + + logger.info( + "文案提取完成: 成功 %d, 失败 %d, 输出 %s", + success_count, fail_count, output_path, + ) + # 刷新去重索引到磁盘 + self._state.flush_dedupe() + return output_path + + def _do_merge( + self, + video_path: Optional[Path], + comments_path: Optional[Path], + scripts_path: Optional[Path], + output_csv: Optional[Path], + ) -> Path: + """合并三份数据生成标准 CSV""" + if not video_path or not video_path.exists(): + raise NonRetryableError( + f"视频数据不存在: {video_path}", step=self.STEP_MERGE + ) + + # 加载数据 + videos = self._load_jsonl(video_path) + comments = self._load_jsonl(comments_path) if comments_path and comments_path.exists() else [] + scripts = self._load_jsonl(scripts_path) if scripts_path and scripts_path.exists() else [] + + logger.info( + "数据源: 视频 %d | 评论 %d | 文案 %d", + len(videos), len(comments), len(scripts), + ) + + # 去重 + videos = self._deduplicate(videos, "aweme_id") + scripts = self._deduplicate(scripts, "aweme_id") + + # 构建索引 + scripts_map: Dict[str, dict] = { + str(r.get("aweme_id", "")): r + for r in scripts if r.get("aweme_id") + } + comments_map: Dict[str, List[str]] = defaultdict(list) + for c in comments: + aid = str(c.get("aweme_id", "")) + text = c.get("content") or c.get("text") or "" + if text: + comments_map[aid].append(text) + + # 合并 + rows = [] + for video in videos: + aid = str(video.get("aweme_id", "")) + if not aid or aid == "None": + continue + + script_rec = scripts_map.get(aid, {}) + script_text = "" + if script_rec.get("script_status") == "success": + script_text = script_rec.get("script_text", "") + + video_comments = comments_map.get(aid, []) + comments_str = "|".join(c.replace("|", "/") for c in video_comments) + + likes = parse_count(video.get("liked_count", 0)) + favorites = parse_count(video.get("collected_count", 0)) + shares = parse_count(video.get("share_count", 0)) + + rows.append({ + "video_id": aid, + "platform": "douyin", + "script_text": script_text, + "likes": likes, + "favorites": favorites, + "shares": shares, + "comments": comments_str, + }) + + # 校验：超限抛异常而非静默截断 + MAX_CSV_ROWS = 200 + if len(rows) > MAX_CSV_ROWS: + raise NonRetryableError( + f"合并数据行数 {len(rows)} 超过上限 {MAX_CSV_ROWS}，" + f"请缩小采集范围或联系管理员调整上限", + step=self.STEP_MERGE, + details={"row_count": len(rows), "max_rows": MAX_CSV_ROWS}, + ) + + # 输出 + if not output_csv: + output_csv = self._config.data_dir / "douyin_koubo_data.csv" + + actual_dir = ensure_dir_writable( + output_csv.parent, self._config.fallback_dir + ) + if actual_dir != output_csv.parent: + output_csv = actual_dir / "douyin_koubo_data.csv" + + check_disk_space_enforced(actual_dir, min_gb=0.1) + + fieldnames = [ + "video_id", "platform", "script_text", + "likes", "favorites", "shares", "comments", + ] + # 原子写入：先写临时文件，再 os.replace 替换 + output_csv.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_path = tempfile.mkstemp( + dir=str(output_csv.parent), + prefix=output_csv.stem + ".tmp", + suffix=".csv", + ) + try: + with os.fdopen(fd, "w", encoding="utf-8-sig", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames, quoting=csv.QUOTE_ALL) + writer.writeheader() + writer.writerows(rows) + os.replace(tmp_path, str(output_csv)) + except Exception: + # 清理临时文件 + if os.path.exists(tmp_path): + os.unlink(tmp_path) + raise + + file_size = output_csv.stat().st_size + logger.info( + "CSV: %s (%.1fKB, %d 行)", + output_csv, file_size / 1024, len(rows), + ) + return output_csv + + # ═══════════════════════════════════════════════════════════════ + # 辅助方法 + # ═══════════════════════════════════════════════════════════════ + + def _get_venv_python(self) -> Path: + """获取 venv 中的 Python 路径，不存在时回退到当前 Python""" + project_dir = self._config.project_dir + if sys.platform == "win32": + venv_python = project_dir / ".venv" / "Scripts" / "python.exe" + else: + venv_python = project_dir / ".venv" / "bin" / "python" + + if venv_python.exists(): + return venv_python + + logger.info("venv 不存在 (%s)，使用当前 Python: %s", venv_python, sys.executable) + return Path(sys.executable) + + @staticmethod + def _load_jsonl(filepath: Path) -> list: + """加载 JSONL 文件""" + if not filepath.exists(): + return [] + records = [] + with open(filepath, "r", encoding="utf-8") as f: + for line_num, line in enumerate(f, 1): + if not line.strip(): + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError as e: + logger.warning( + "JSONL 损坏行已跳过: %s 第 %d 行: %s", + filepath, line_num, str(e)[:100], + ) + return records + + def _deduplicate(self, records: list, key: str = "aweme_id") -> list: + """ + 按 key 去重，优先保留 script_status=success。 + ★ 幂等键缺失时生成确定性伪 ID ★ + """ + seen: Dict[str, dict] = {} + for i, rec in enumerate(records): + aid = str(rec.get(key, "")) + if not aid or aid == "None": + aid = self._state.generate_pseudo_id(i + 1, rec, prefix="video") + rec[key] = aid + + if aid not in seen: + seen[aid] = rec + else: + if (rec.get("script_status") == "success" and + seen[aid].get("script_status") != "success"): + seen[aid] = rec + return list(seen.values()) + + @retry(max_retries=3, base_delay=5, backoff_factor=3, + retryable_exceptions=(OSError, ConnectionError, TimeoutError)) + def _download_video(self, url: str, save_path: str, timeout: int = 120) -> bool: + """下载视频到本地""" + import httpx + + with httpx.Client(follow_redirects=True, timeout=timeout) as client: + resp = client.get(url, headers={ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Referer": "https://www.douyin.com/", + }) + + if resp.status_code == 429: + wait_time = 60 + random.uniform(0, 30) + logger.warning("HTTP 429 风控，等待 %.1fs（可被 cancel 中断）", wait_time) + interrupted = self._cancel_event.wait(timeout=wait_time) + if interrupted: + raise NonRetryableError( + "操作已被取消", step=self.STEP_EXTRACT + ) + raise ConnectionError("HTTP 429 rate limited") + + if resp.status_code >= 500: + raise ConnectionError(f"HTTP {resp.status_code}") + + if resp.status_code == 200 and len(resp.content) > 10000: + with open(save_path, "wb") as f: + f.write(resp.content) + return True + + # 小文件可能是防爬页面或空响应，重试 + if resp.status_code == 200 and len(resp.content) <= 10000: + logger.warning( + "下载返回小文件 (%d bytes)，可能是防爬页面，重试: %s", + len(resp.content), url[:80], + ) + raise ConnectionError( + f"下载文件过小: {len(resp.content)} bytes (可能防爬)" + ) + + logger.warning( + "下载失败: status=%d size=%d", resp.status_code, len(resp.content) + ) + return False + + @staticmethod + def _transcribe_video(video_path: str, model: Any, max_retries: int = 2) -> str: + """用 faster-whisper 转写视频中的语音""" + for attempt in range(1, max_retries + 1): + try: + segments, info = model.transcribe( + video_path, + language="zh", + beam_size=5, + vad_filter=True, + vad_parameters=dict(min_silence_duration_ms=500), + ) + texts = [seg.text.strip() for seg in segments] + result = "".join(texts) + if result: + return result + logger.warning("转写结果为空 (attempt %d)", attempt) + except Exception as e: + logger.error("转写失败 (attempt %d): %s", attempt, e) + if attempt < max_retries: + delay = 5 * attempt + random.uniform(0, 1) + time.sleep(delay) + return "" + + def _do_install_whisper(self) -> None: + """安装 faster-whisper""" + venv_python = self._get_venv_python() + result = subprocess.run( + [str(venv_python), "-c", "import faster_whisper; print('OK')"], + capture_output=True, text=True, + ) + if result.returncode == 0: + logger.info("faster-whisper 已安装") + return + + subprocess.run( + [str(venv_python), "-m", "pip", "install", + "faster-whisper", "imageio-ffmpeg", "httpx"], + check=True, timeout=300, + ) diff --git a/douyin_scraper/exceptions.py b/douyin_scraper/exceptions.py new file mode 100644 index 000000000..40bd92a44 --- /dev/null +++ b/douyin_scraper/exceptions.py @@ -0,0 +1,80 @@ +""" +douyin_scraper.exceptions — 自定义异常 +====================================== +v5 新增：所有异常包含 step / exit_code / details 属性。 +我实际执行时：异常只有消息字符串，无法判断来自哪个步骤、 +是否可重试，导致 run_all.py 中的错误分类只能靠字符串匹配。 +""" + +from typing import Optional + + +class ScraperError(Exception): + """采集工具基础异常""" + + def __init__( + self, + message: str, + step: str = "", + exit_code: int = 1, + details: Optional[dict] = None, + ) -> None: + super().__init__(message) + self.step = step + self.exit_code = exit_code + self.details = details or {} + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + f"step={self.step!r}, exit_code={self.exit_code}, " + f"message={str(self)!r})" + ) + + +class RetryableError(ScraperError): + """可重试错误：网络超时、HTTP 5xx、临时文件锁""" + + def __init__( + self, + message: str = "", + step: str = "", + details: Optional[dict] = None, + ) -> None: + super().__init__(message, step=step, exit_code=1, details=details) + + +class NonRetryableError(ScraperError): + """不可重试错误：配置错误、权限不足、Python 版本不对""" + + def __init__( + self, + message: str = "", + step: str = "", + details: Optional[dict] = None, + ) -> None: + super().__init__(message, step=step, exit_code=2, details=details) + + +class ConfigError(ScraperError): + """配置错误：缺少必需配置项、配置文件格式错误""" + + def __init__( + self, + message: str = "", + step: str = "", + details: Optional[dict] = None, + ) -> None: + super().__init__(message, step=step, exit_code=2, details=details) + + +class FatalError(ScraperError): + """致命错误：磁盘满、内存不足""" + + def __init__( + self, + message: str = "", + step: str = "", + details: Optional[dict] = None, + ) -> None: + super().__init__(message, step=step, exit_code=3, details=details) diff --git a/douyin_scraper/models.py b/douyin_scraper/models.py new file mode 100644 index 000000000..d39540dd7 --- /dev/null +++ b/douyin_scraper/models.py @@ -0,0 +1,115 @@ +""" +douyin_scraper.models — 数据模型 +================================= +v5 新增：使用 dataclass 定义数据结构，替代 v4 中的原始字典。 +我实际执行时：字典字段名拼写错误（content vs text）导致评论丢失， +数据模型可以提前发现这类错误。 +""" + +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class Video: + """视频元数据""" + aweme_id: str + title: str = "" + author: str = "" + video_download_url: str = "" + liked_count: int = 0 + collected_count: int = 0 + share_count: int = 0 + comment_count: int = 0 + create_time: str = "" + desc: str = "" + + def to_dict(self) -> dict: + return { + "aweme_id": self.aweme_id, + "title": self.title, + "author": self.author, + "video_download_url": self.video_download_url, + "liked_count": self.liked_count, + "collected_count": self.collected_count, + "share_count": self.share_count, + "comment_count": self.comment_count, + "create_time": self.create_time, + "desc": self.desc, + } + + @classmethod + def from_dict(cls, data: dict) -> "Video": + return cls( + aweme_id=str(data.get("aweme_id", "")), + title=str(data.get("title", "")), + author=str(data.get("author", data.get("nickname", ""))), + video_download_url=str(data.get("video_download_url", "")), + liked_count=int(data.get("liked_count", 0) or 0), + collected_count=int(data.get("collected_count", 0) or 0), + share_count=int(data.get("share_count", 0) or 0), + comment_count=int(data.get("comment_count", 0) or 0), + create_time=str(data.get("create_time", "")), + desc=str(data.get("desc", "")), + ) + + +@dataclass +class Comment: + """ + 评论数据。 + 我实际执行时：评论字段名有时是 content 有时是 text， + from_dict 兼容两种。 + """ + aweme_id: str = "" + content: str = "" + comment_id: str = "" + nickname: str = "" + create_time: str = "" + + def to_dict(self) -> dict: + return { + "aweme_id": self.aweme_id, + "content": self.content, + "comment_id": self.comment_id, + "nickname": self.nickname, + "create_time": self.create_time, + } + + @classmethod + def from_dict(cls, data: dict) -> "Comment": + # ★ 兼容两种字段名 ★ 我实际执行时踩的坑 + text = data.get("content") or data.get("text") or "" + return cls( + aweme_id=str(data.get("aweme_id", "")), + content=str(text), + comment_id=str(data.get("comment_id", data.get("cid", ""))), + nickname=str(data.get("nickname", "")), + create_time=str(data.get("create_time", "")), + ) + + +@dataclass +class Script: + """文案提取结果""" + aweme_id: str + script_text: str = "" + script_status: str = "" # success / download_failed / asr_failed + model_name: str = "" + + def to_dict(self) -> dict: + return { + "aweme_id": self.aweme_id, + "script_text": self.script_text, + "script_status": self.script_status, + "model_name": self.model_name, + } + + @classmethod + def from_dict(cls, data: dict) -> "Script": + return cls( + aweme_id=str(data.get("aweme_id", "")), + script_text=str(data.get("script_text", "")), + script_status=str(data.get("script_status", "")), + model_name=str(data.get("model_name", "")), + ) diff --git a/douyin_scraper/state.py b/douyin_scraper/state.py new file mode 100644 index 000000000..6b0926af9 --- /dev/null +++ b/douyin_scraper/state.py @@ -0,0 +1,456 @@ +""" +douyin_scraper.state — 状态管理 +================================= +v5 重构：从 v4 的 common_utils 中拆出状态管理逻辑。 +三态转换：pending → in_progress → completed / failed + +我实际执行时踩过的坑： + - 状态文件损坏导致全部步骤跳过 → 损坏时重置为空 + - 步骤开始前没有状态标记 → 崩溃后不知道是否已经开始 + - 步骤失败后状态不记录 → 重新运行时盲目重试 + - 重置步骤时忘了清除去重索引 → 数据被跳过 + - 全局变量存储状态路径 → 无法并行测试 + - 非原子写入导致状态文件损坏 → 原子写入（临时文件+os.replace） + - 每次标记去重都读写文件导致 I/O 风暴 → 内存缓存+批量写入 +""" + +import hashlib +import json +import logging +import os +import tempfile +import threading +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, Optional, Set + +logger = logging.getLogger("douyin_scraper") + +# 中国时区 +_CST = timezone(timedelta(hours=8)) + + +def _now_iso() -> str: + return datetime.now(_CST).isoformat() + + +def atomic_write_json(filepath: Path, data: dict) -> None: + """ + 原子写入 JSON 文件：先写临时文件，再 os.replace 替换原文件。 + 防止写入过程中崩溃导致文件损坏。 + + Args: + filepath: 目标 JSON 文件路径 + data: 要写入的字典数据 + """ + filepath = Path(filepath) + filepath.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_path = tempfile.mkstemp( + dir=str(filepath.parent), + prefix=filepath.stem + ".tmp", + suffix=".json", + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + os.replace(tmp_path, str(filepath)) + except Exception: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + raise + + +class StateManager: + """ + 状态管理器。 + + 管理三个状态文件： + - task_state.json: 步骤完成状态 + - dedupe_index.json: 去重索引 + - execution_log.jsonl: 结构化日志 + + v5 改进：不再使用模块级全局变量存储路径， + 通过实例属性管理，方便测试时注入临时目录。 + + 安全改进： + - 原子写入：所有 JSON 文件通过 atomic_write_json 写入 + - 去重缓存：内存缓存 + 脏标记 + 批量写入，减少 I/O + """ + + def __init__(self, state_dir: Path) -> None: + self.state_dir = state_dir + self.state_dir.mkdir(parents=True, exist_ok=True) + self._task_state_path = state_dir / "task_state.json" + self._dedupe_path = state_dir / "dedupe_index.json" + self._log_path = state_dir / "execution_log.jsonl" + + # 线程锁：保护所有 task_state 的 read-modify-write 操作 + self._state_lock = threading.Lock() + + # task_state 内存缓存：避免每次 get_step_status 都读磁盘 + self._task_state_cache: Optional[dict] = None + + # 去重索引内存缓存：index_name -> Set[item_id] + self._dedupe_cache: Dict[str, Set[str]] = {} + # 脏标记：记录哪些 index_name 被修改过但尚未写盘 + self._dedupe_dirty: Set[str] = set() + # 是否已从文件加载过缓存 + self._dedupe_loaded: bool = False + + def __del__(self) -> None: + """析构时确保脏去重索引写盘""" + try: + self.flush_dedupe() + except Exception: + pass + + def _ensure_dedupe_loaded(self) -> None: + """懒加载去重索引到内存缓存""" + if self._dedupe_loaded: + return + self._dedupe_loaded = True + data = self._load_all_dedupe_from_disk() + for index_name, ids in data.items(): + self._dedupe_cache[index_name] = set(ids) + + # ═══════════════════════════════════════════════════════════════ + # task_state.json 管理 + # ═══════════════════════════════════════════════════════════════ + + def load_task_state(self) -> dict: + """加载任务状态文件（从磁盘加载到缓存）""" + with self._state_lock: + if self._task_state_path.exists(): + try: + with open(self._task_state_path, "r", encoding="utf-8") as f: + data = json.load(f) + self._task_state_cache = data + return data + except (json.JSONDecodeError, OSError): + logger.warning("task_state.json 损坏，重置为空") + data = {"steps": {}, "metadata": {}} + self._task_state_cache = data + return data + data = {"steps": {}, "metadata": {}} + self._task_state_cache = data + return data + + def _get_cached_state(self) -> dict: + """获取缓存的状态，若缓存为空则从磁盘加载（调用方必须已持有 _state_lock）""" + if self._task_state_cache is not None: + return self._task_state_cache + # 直接加载，不经过 load_task_state（避免嵌套获取 _state_lock 导致死锁） + if self._task_state_path.exists(): + try: + with open(self._task_state_path, "r", encoding="utf-8") as f: + data = json.load(f) + self._task_state_cache = data + return data + except (json.JSONDecodeError, OSError): + logger.warning("task_state.json 损坏，重置为空") + data = {"steps": {}, "metadata": {}} + self._task_state_cache = data + return data + data = {"steps": {}, "metadata": {}} + self._task_state_cache = data + return data + + def save_task_state(self, state: dict) -> None: + """保存任务状态文件（原子写入 + 更新缓存）""" + with self._state_lock: + self.state_dir.mkdir(parents=True, exist_ok=True) + atomic_write_json(self._task_state_path, state) + self._task_state_cache = state + + def get_step_status(self, step_name: str) -> str: + """获取步骤状态：从内存缓存读取，无磁盘 I/O""" + with self._state_lock: + state = self._get_cached_state() + return state.get("steps", {}).get(step_name, {}).get("status", "pending") + + def get_step_info(self, step_name: str) -> dict: + """获取步骤的详细信息（从缓存读取）""" + with self._state_lock: + state = self._get_cached_state() + return state.get("steps", {}).get(step_name, {}) + + def mark_step_started(self, step_name: str) -> None: + """标记步骤开始（in_progress），线程安全""" + with self._state_lock: + state = self._get_cached_state() + state.setdefault("steps", {})[step_name] = { + "status": "in_progress", + "started_at": _now_iso(), + } + self.state_dir.mkdir(parents=True, exist_ok=True) + atomic_write_json(self._task_state_path, state) + self._task_state_cache = state + logger.info("[%s] 步骤开始", step_name) + + def mark_step_completed(self, step_name: str, detail: str = "") -> None: + """标记步骤完成，线程安全""" + with self._state_lock: + state = self._get_cached_state() + entry: Dict[str, Any] = { + "status": "completed", + "completed_at": _now_iso(), + } + if detail: + entry["detail"] = detail + state.setdefault("steps", {})[step_name] = entry + self.state_dir.mkdir(parents=True, exist_ok=True) + atomic_write_json(self._task_state_path, state) + self._task_state_cache = state + logger.info("[%s] 步骤完成: %s", step_name, detail) + + def mark_step_failed( + self, + step_name: str, + error_summary: str = "", + exit_code: int = 1, + ) -> None: + """ + 标记步骤失败，线程安全。 + + Args: + step_name: 步骤名称 + error_summary: 错误摘要（前 200 字符） + exit_code: 退出码（1=可重试, 2=不可重试, 3=致命） + """ + with self._state_lock: + state = self._get_cached_state() + state.setdefault("steps", {})[step_name] = { + "status": "failed", + "failed_at": _now_iso(), + "error_summary": error_summary[:200], + "exit_code": exit_code, + } + self.state_dir.mkdir(parents=True, exist_ok=True) + atomic_write_json(self._task_state_path, state) + self._task_state_cache = state + logger.error( + "[%s] 步骤失败 (exit_code=%d): %s", + step_name, exit_code, error_summary[:200], + ) + + def is_step_completed(self, step_name: str) -> bool: + return self.get_step_status(step_name) == "completed" + + def is_step_failed(self, step_name: str) -> bool: + return self.get_step_status(step_name) == "failed" + + def check_step_ready(self, step_name: str) -> bool: + """ + 检查步骤是否可以执行。 + completed → 不执行，failed → 不执行（需手动修复）。 + """ + status = self.get_step_status(step_name) + if status == "completed": + logger.info("[%s] 步骤已完成，跳过", step_name) + return False + if status == "in_progress": + logger.warning("[%s] 上次可能中断，继续执行", step_name) + return True + if status == "failed": + logger.warning("[%s] 上次失败，需修复后 reset_step()", step_name) + return False + return True # pending + + def reset_step(self, step_name: str, clear_dedupe: bool = False) -> None: + """ + 重置步骤状态为 pending，线程安全。 + """ + with self._state_lock: + state = self._get_cached_state() + if step_name in state.get("steps", {}): + del state["steps"][step_name] + self.state_dir.mkdir(parents=True, exist_ok=True) + atomic_write_json(self._task_state_path, state) + self._task_state_cache = state + + if clear_dedupe: + self._ensure_dedupe_loaded() + if step_name in self._dedupe_cache: + del self._dedupe_cache[step_name] + self._dedupe_dirty.discard(step_name) + # 同步写盘 + data = {k: sorted(list(v)) for k, v in self._dedupe_cache.items()} + self._save_all_dedupe_to_disk(data) + + logger.info("[%s] 步骤状态已重置 (clear_dedupe=%s)", step_name, clear_dedupe) + + def reset_all(self) -> None: + """重置所有状态，线程安全""" + with self._state_lock: + for path in [self._task_state_path, self._dedupe_path]: + if path.exists(): + path.unlink() + # 清空内存缓存 + self._task_state_cache = None + self._dedupe_cache.clear() + self._dedupe_dirty.clear() + self._dedupe_loaded = False + logger.info("所有状态文件已重置") + + # ═══════════════════════════════════════════════════════════════ + # dedupe_index.json 管理（内存缓存 + 批量写入） + # ═══════════════════════════════════════════════════════════════ + + def _load_all_dedupe_from_disk(self) -> dict: + """从磁盘读取去重索引 JSON""" + if self._dedupe_path.exists(): + try: + with open(self._dedupe_path, "r", encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, OSError) as e: + if isinstance(e, OSError): + logger.error("去重索引加载失败: %s, 视为无已处理记录", e) + return {} + return {} + + def _save_all_dedupe_to_disk(self, data: dict) -> None: + """将去重索引写入磁盘（原子写入）""" + atomic_write_json(self._dedupe_path, data) + + def _load_all_dedupe(self) -> dict: + """加载去重索引（兼容旧接口，从内存缓存返回）""" + self._ensure_dedupe_loaded() + return {k: sorted(list(v)) for k, v in self._dedupe_cache.items()} + + def _save_all_dedupe(self, data: dict) -> None: + """保存去重索引（兼容旧接口，同步内存缓存并写盘）""" + for index_name, ids in data.items(): + self._dedupe_cache[index_name] = set(ids) + self._dedupe_dirty.clear() + self._save_all_dedupe_to_disk(data) + + def is_duplicate(self, index_name: str, item_id: str) -> bool: + """检查是否已写入（从内存缓存读取，无 I/O）""" + self._ensure_dedupe_loaded() + return str(item_id) in self._dedupe_cache.get(index_name, set()) + + def is_written(self, index_name: str, item_id: str) -> bool: + """is_duplicate 的别名，语义更清晰""" + return self.is_duplicate(index_name, item_id) + + def mark_written(self, index_name: str, item_id: str) -> None: + """ + 标记已写入（只修改内存缓存，标记脏，不立即写盘）。 + 调用 flush_dedupe() 批量写盘。 + """ + self._ensure_dedupe_loaded() + item_id_str = str(item_id) + if index_name not in self._dedupe_cache: + self._dedupe_cache[index_name] = set() + self._dedupe_cache[index_name].add(item_id_str) + self._dedupe_dirty.add(index_name) + + def flush_dedupe(self) -> None: + """ + 将脏的去重索引批量写入磁盘（原子写入）。 + 通常在步骤完成或对象析构时调用。 + """ + if not self._dedupe_dirty: + return + self._ensure_dedupe_loaded() + data = {k: sorted(list(v)) for k, v in self._dedupe_cache.items()} + try: + self._save_all_dedupe_to_disk(data) + self._dedupe_dirty.clear() + logger.debug("去重索引已刷新到磁盘") + except Exception as e: + logger.error("去重索引刷新失败: %s", e) + raise + + def load_completed_ids_from_jsonl( + self, filepath: Path, id_field: str = "aweme_id" + ) -> Set[str]: + """ + 从已有 JSONL 输出文件中读取已完成的 ID 集合。 + ★ 断点续传的核心 ★ + + 我实际执行时：脚本崩溃后，唯一的恢复方式就是读取输出文件。 + v5：空文件、损坏行的容错处理。 + """ + done: Set[str] = set() + if not filepath.exists(): + return done + try: + with open(filepath, "r", encoding="utf-8") as f: + for line_num, line in enumerate(f, 1): + if not line.strip(): + continue + try: + d = json.loads(line) + aid = str(d.get(id_field, "")) + if aid: + done.add(aid) + except json.JSONDecodeError: + logger.warning( + "JSONL 损坏行已跳过: %s 行号 %d", + filepath, line_num, + ) + continue + except OSError: + pass + return done + + @staticmethod + def generate_pseudo_id( + line_num: int, record: dict, prefix: str = "unknown" + ) -> str: + """ + 当幂等键缺失时，生成确定性伪 ID。 + + 我实际执行时：有少数记录的 aweme_id 为空，导致去重失败和数据丢失。 + v5：空幂等键 → 生成 `unknown_<行号>_<内容哈希前8位>` 的确定性 ID。 + """ + content = json.dumps(record, sort_keys=True, ensure_ascii=False) + hash_hex = hashlib.md5(content.encode("utf-8")).hexdigest()[:8] + pseudo_id = f"{prefix}_{line_num}_{hash_hex}" + logger.warning("幂等键缺失，生成伪 ID: %s", pseudo_id) + return pseudo_id + + # ═══════════════════════════════════════════════════════════════ + # execution_log.jsonl + # ═══════════════════════════════════════════════════════════════ + + def write_log( + self, level: str, step: str, action: str, msg: str, **kwargs: Any + ) -> None: + """写入结构化 JSON Lines 日志""" + entry = { + "ts": _now_iso(), + "level": level, + "step": step, + "action": action, + "msg": msg, + **kwargs, + } + line = json.dumps(entry, ensure_ascii=False) + # 使用 os.open 绕过 Windows asyncio 问题 + import os as _os + flags = _os.O_WRONLY | _os.O_APPEND | _os.O_CREAT + try: + fd = _os.open(str(self._log_path), flags, 0o644) + try: + _os.write(fd, (line + "\n").encode("utf-8")) + finally: + _os.close(fd) + except OSError as e: + # JSONL 文件写入失败时，降级到 Python logging（写 stderr） + _log = logging.getLogger(__name__) + _log.warning("JSONL 日志写入失败: %s", e) + + def get_all_status(self) -> Dict[str, Any]: + """获取所有步骤状态摘要""" + state = self.load_task_state() + result: Dict[str, Any] = {} + for step_name, info in state.get("steps", {}).items(): + result[step_name] = { + "status": info.get("status", "pending"), + "detail": info.get("detail", ""), + "error_summary": info.get("error_summary", ""), + "exit_code": info.get("exit_code", 0), + } + return result diff --git a/douyin_scraper/utils.py b/douyin_scraper/utils.py new file mode 100644 index 000000000..f2716b03e --- /dev/null +++ b/douyin_scraper/utils.py @@ -0,0 +1,505 @@ +""" +douyin_scraper.utils — 工具函数 +================================ +v5 重构：从 v4 common_utils 中拆出工具函数。 + +保留 v4 验证过的所有生产级特性： + - append_record: os.open + fallback + - retry: 指数退避 + 随机抖动 + - setup_ffmpeg: imageio-ffmpeg → 系统 ffmpeg → 手动提示 + - ensure_dir_writable: 写前检测 + - check_disk_space: 磁盘空间预检 + - 日志轮转 +""" + +import functools +import logging +import os +import random +import re +import shutil +import socket +import subprocess +import sys +import time +from logging.handlers import RotatingFileHandler +from pathlib import Path +from typing import Any, Callable, List, Optional, Sequence, Set, Tuple, Type, Union + +from douyin_scraper.exceptions import ( + FatalError, + NonRetryableError, + RetryableError, + ScraperError, +) + +logger = logging.getLogger("douyin_scraper") + +# 退出码 +EXIT_RETRYABLE = 1 +EXIT_NON_RETRYABLE = 2 +EXIT_FATAL = 3 + + +# ═══════════════════════════════════════════════════════════════════ +# 1. 错误分类 +# ═══════════════════════════════════════════════════════════════════ + +_MESSAGE_PATTERNS: dict = { + EXIT_RETRYABLE: [ + "timeout", "timed out", "connection refused", "connection reset", + "connection error", "429", "rate limit", + "temporarily unavailable", "retry", "503", "502", "500", + "broken pipe", "eof", "network", "socket", + ], + EXIT_NON_RETRYABLE: [ + "not found", "404", "403", "forbidden", "unauthorized", + "permission denied", "config error", "invalid argument", + "version mismatch", "syntax error", + "module not found", "no module named", + "file not found", "no such file", + ], + EXIT_FATAL: [ + "no space left", "disk full", "out of memory", "cannot allocate memory", + "memoryerror", + ], +} + + +def classify_error(e: Exception) -> int: + """ + 根据异常类型 + 消息内容判断退出码。 + + 我实际执行时：OSError 既可能是磁盘满（致命）也可能是临时网络断（可重试）。 + 检查顺序：自定义异常 → 标准异常子类 → 消息内容 → 默认不可重试。 + """ + # 1. 自定义异常（精确匹配） + if isinstance(e, FatalError): + return EXIT_FATAL + if isinstance(e, (NonRetryableError,)): + return EXIT_NON_RETRYABLE + if isinstance(e, RetryableError): + return EXIT_RETRYABLE + + # 2. 标准异常子类（PermissionError 是 OSError 子类，须先检查） + if isinstance(e, MemoryError): + return EXIT_FATAL + if isinstance(e, PermissionError): + return EXIT_NON_RETRYABLE + if isinstance(e, (ConnectionError, TimeoutError)): + return EXIT_RETRYABLE + if isinstance(e, OSError): + pass # fall through to message analysis + + # 3. 消息内容关键词匹配 + msg_lower = str(e).lower() + for exit_code, patterns in _MESSAGE_PATTERNS.items(): + for pattern in patterns: + if pattern in msg_lower: + return exit_code + + # 4. OSError 没匹配到消息 → 默认可重试 + if isinstance(e, OSError): + return EXIT_RETRYABLE + + # 5. 默认不可重试 + return EXIT_NON_RETRYABLE + + +# ═══════════════════════════════════════════════════════════════════ +# 2. 安全文件写入 +# ═══════════════════════════════════════════════════════════════════ + +def safe_write_line(filepath: Path, line: str, fallback_dir: Path) -> Path: + """ + 使用 os.open 低级 I/O 追加写入一行。 + 返回实际写入的文件路径（可能是 fallback）。 + + ★ 禁止在 async 函数内使用 open() ★ + 我实际执行时：open() 在 Windows asyncio 中导致 Bad file descriptor。 + """ + if not line.endswith("\n"): + line += "\n" + flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT + try: + fd = os.open(str(filepath), flags, 0o644) + try: + os.write(fd, line.encode("utf-8")) + finally: + os.close(fd) + return filepath + except OSError: + # fallback + fallback_dir.mkdir(parents=True, exist_ok=True) + fallback_path = fallback_dir / filepath.name + fd = os.open(str(fallback_path), flags, 0o644) + try: + os.write(fd, line.encode("utf-8")) + finally: + os.close(fd) + logger.warning("原路径写入失败，fallback 到: %s", fallback_path) + return fallback_path + + +def append_record(filepath: Path, record: dict, fallback_dir: Path) -> Path: + """ + 追加写入一条 JSON 记录到 JSONL 文件。 + ★ 每完成一个最小原子操作，必须调用此函数立即写盘 ★ + """ + import json + line = json.dumps(record, ensure_ascii=False) + return safe_write_line(filepath, line, fallback_dir) + + +# ═══════════════════════════════════════════════════════════════════ +# 3. 数值字段解析 +# ═══════════════════════════════════════════════════════════════════ + + +def parse_count(value) -> int: + """ + 将抖音可能返回的 '1.2万', '10.3w' 等格式转为整数。 + + 抖音 API 返回的数值字段可能是： + - 整数 / 浮点数：直接转换 + - 字符串带中文/英文单位：'1.2万', '10.3w', '500k' + - None / 空字符串：返回 0 + - 无法解析的字符串：返回 0 + + Args: + value: 原始数值，可能是 int/float/str/None + + Returns: + 解析后的整数 + """ + if value is None: + return 0 + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) + if isinstance(value, str): + s = value.strip().lower() + if not s: + return 0 + multiplier = 1 + if s.endswith(("万", "w")): + multiplier = 10000 + s = s[:-1] + elif s.endswith("k"): + multiplier = 1000 + s = s[:-1] + try: + return int(float(s) * multiplier) + except ValueError: + return 0 + return 0 + + +# ═══════════════════════════════════════════════════════════════════ +# 4. 自动重试装饰器 +# ═══════════════════════════════════════════════════════════════════ + +def retry( + max_retries: int = 3, + base_delay: float = 5.0, + backoff_factor: float = 3.0, + retryable_exceptions: Tuple[Type[Exception], ...] = ( + OSError, ConnectionError, TimeoutError, RetryableError + ), +) -> Callable: + """ + 指数退避 + 随机抖动重试装饰器。 + delay = base_delay * backoff_factor^(attempt-1) + random.uniform(0, 1) + + ★ 禁止在重试循环中不加 random 抖动 ★ + 我实际执行时：多任务同时重试导致请求集中，触发更严厉的风控。 + """ + def decorator(func: Callable) -> Callable: + @functools.wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + last_error: Optional[Exception] = None + for attempt in range(1, max_retries + 1): + try: + return func(*args, **kwargs) + except retryable_exceptions as e: + last_error = e + delay = base_delay * (backoff_factor ** (attempt - 1)) + jitter = random.uniform(0, 1) + total_delay = delay + jitter + logger.warning( + "[%s] 失败 (attempt %d/%d): %s, %.1fs 后重试", + func.__name__, attempt, max_retries, e, total_delay, + ) + if attempt < max_retries: + time.sleep(total_delay) + raise last_error + return wrapper + return decorator + + +def retry_with_degradation( + step_name: str, + state_manager: Any, + max_retries: int = 3, + base_delay: float = 5.0, + backoff_factor: float = 3.0, + retryable_exceptions: Tuple[Type[Exception], ...] = ( + OSError, ConnectionError, TimeoutError, RetryableError + ), +) -> Callable: + """ + 带安全降级的重试装饰器。重试耗尽后标记步骤为 failed 而非崩溃。 + + 我实际执行时：重试耗尽后直接 raise，导致整个脚本崩溃且无状态记录。 + v5：重试耗尽 → mark_step_failed → 抛出原始异常。 + """ + def decorator(func: Callable) -> Callable: + @functools.wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + last_error: Optional[Exception] = None + for attempt in range(1, max_retries + 1): + try: + return func(*args, **kwargs) + except retryable_exceptions as e: + last_error = e + delay = base_delay * (backoff_factor ** (attempt - 1)) + jitter = random.uniform(0, 1) + total_delay = delay + jitter + logger.warning( + "[%s] 失败 (attempt %d/%d): %s, %.1fs 后重试", + func.__name__, attempt, max_retries, e, total_delay, + ) + if attempt < max_retries: + time.sleep(total_delay) + + # ★ 重试耗尽：安全降级 ★ + exit_code = classify_error(last_error) + state_manager.mark_step_failed( + step_name, + error_summary=str(last_error)[:200], + exit_code=exit_code, + ) + raise last_error + return wrapper + return decorator + + +# ═══════════════════════════════════════════════════════════════════ +# 4. 文件名安全处理 +# ═══════════════════════════════════════════════════════════════════ + +def get_safe_filename(name: str) -> str: + """ + 将任意字符串转换为安全的文件名。 + + 替换 Windows/Linux 上非法的文件名字符（\\/*?:<>|）为下划线， + 去除首尾空格和点号（Windows 不允许以点或空格结尾）。 + + Args: + name: 原始字符串 + + Returns: + 安全的文件名字符串 + """ + # 替换所有非法字符为下划线 + safe = re.sub(r'[\\/*?:<>|]', '_', name) + # 去除首尾空格 + safe = safe.strip() + # 去除首尾点号（Windows 不允许以点结尾） + safe = safe.strip('.') + # 如果结果为空，使用默认名 + if not safe: + safe = "unnamed" + # 截断至合理长度（大多数文件系统限制 255 字节） + max_len = 200 + if len(safe) > max_len: + safe = safe[:max_len] + return safe + + +# ═══════════════════════════════════════════════════════════════════ +# 5. 环境检测 +# ═══════════════════════════════════════════════════════════════════ + +def check_command_exists(cmd: str) -> bool: + """检查系统命令是否存在""" + return shutil.which(cmd) is not None + + +def check_port_in_use(port: int) -> bool: + """检查端口是否被占用""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(("127.0.0.1", port)) == 0 + + +def kill_process_on_port(port: int) -> bool: + """安全地终止占用指定端口的进程（使用 psutil，无 shell 注入风险）。""" + try: + import psutil + except ImportError: + logger.warning("psutil 未安装，无法安全终止端口 %d 上的进程", port) + return False + + killed = False + try: + # 使用 net_connections 替代已弃用的 proc.connections() + for conn in psutil.net_connections(kind='inet'): + if conn.status == 'LISTEN' and conn.laddr.port == port: + try: + proc = psutil.Process(conn.pid) + proc.terminate() + # 等待进程退出，避免僵尸进程 + try: + proc.wait(timeout=5) + except psutil.TimeoutExpired: + logger.warning( + "进程 PID=%d terminate 后 5s 未退出，强制 kill", + proc.pid, + ) + proc.kill() + try: + proc.wait(timeout=3) + except psutil.TimeoutExpired: + logger.warning("进程 PID=%d kill 后仍未退出", proc.pid) + killed = True + logger.info("已杀掉端口 %d 上的进程 PID=%s 名称=%s", + port, proc.pid, proc.name()) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + continue + except psutil.AccessDenied: + logger.warning("无权限获取网络连接信息，无法终止端口 %d 上的进程", port) + return False + return killed + + +def setup_ffmpeg() -> bool: + """ + 将 imageio-ffmpeg 内置的 ffmpeg 加入 PATH。 + 返回 True 表示 ffmpeg 可用。 + + 我实际执行时：忘了装 ffmpeg，运行 whisper 报 FileNotFoundError。 + 降级策略：imageio-ffmpeg → 系统 ffmpeg → 手动提示。 + """ + # 方案 A: imageio-ffmpeg + try: + import imageio_ffmpeg + try: + ffmpeg_exe = imageio_ffmpeg.get_ffmpeg_exe() + except RuntimeError: + logger.warning("imageio_ffmpeg.get_ffmpeg_exe() 抛出 RuntimeError，等待 2 秒后重试") + time.sleep(2) + try: + ffmpeg_exe = imageio_ffmpeg.get_ffmpeg_exe() + except RuntimeError: + logger.error("imageio_ffmpeg.get_ffmpeg_exe() 重试仍失败") + ffmpeg_exe = None + + if ffmpeg_exe and os.path.isfile(ffmpeg_exe): + ffmpeg_dir = os.path.dirname(ffmpeg_exe) + os.environ["PATH"] = ffmpeg_dir + os.pathsep + os.environ.get("PATH", "") + os.environ["FFMPEG_BINARY"] = ffmpeg_exe + logger.info("ffmpeg (imageio-ffmpeg): %s", ffmpeg_exe) + return True + except ImportError: + pass + + # 方案 B: 系统 ffmpeg + if check_command_exists("ffmpeg"): + logger.info("ffmpeg (系统 PATH)") + return True + + # 方案 C: 手动提示 + logger.error("ffmpeg 不可用！请执行: pip install imageio-ffmpeg") + return False + + +def setup_env_vars() -> None: + """ + 设置必要的环境变量。 + 我实际执行时：PATHEXT 为空导致 execjs 找不到 node.exe。 + """ + if sys.platform == "win32" and not os.environ.get("PATHEXT"): + os.environ["PATHEXT"] = ".COM;.EXE;.BAT;.CMD;.VBS;.JS" + os.environ["MPLBACKEND"] = "Agg" + + +def check_disk_space(path: Path, min_gb: float = 1.0) -> bool: + """检查磁盘剩余空间""" + try: + usage = shutil.disk_usage(str(path)) + free_gb = usage.free / (1024**3) + return free_gb >= min_gb + except OSError as e: + logger.warning("磁盘空间检查失败: %s", e) + return False + + +def check_disk_space_enforced(path: Path, min_gb: float = 1.0) -> None: + """ + 检查磁盘剩余空间，不足则抛 FatalError。 + ★ 在写入大文件前必须调用此函数 ★ + """ + try: + usage = shutil.disk_usage(str(path)) + free_gb = usage.free / (1024**3) + if free_gb < min_gb: + raise FatalError( + f"磁盘空间不足: {free_gb:.2f}GB < {min_gb}GB (path={path})" + ) + except OSError as e: + logger.error("磁盘空间检查失败: %s", e) + raise FatalError( + f"磁盘空间检查失败: {e} (path={path})" + ) from e + + +def ensure_dir_writable(dir_path: Path, fallback_dir: Path) -> Path: + """ + 检测目录可写性并自动选择可用路径。 + ★ 在写入数据前必须调用此函数 ★ + 我实际执行时：沙箱环境中写入 G 盘被拦截，但没有预检测。 + """ + dir_path = Path(dir_path) + fallback_dir = Path(fallback_dir) + + try: + dir_path.mkdir(parents=True, exist_ok=True) + test_file = dir_path / ".write_test" + test_file.write_text("ok", encoding="utf-8") + test_file.unlink() + return dir_path + except (PermissionError, OSError): + logger.warning("目录不可写: %s，fallback 到 %s", dir_path, fallback_dir) + fallback_dir.mkdir(parents=True, exist_ok=True) + return fallback_dir + + +# ═══════════════════════════════════════════════════════════════════ +# 5. 日志轮转 +# ═══════════════════════════════════════════════════════════════════ + +def setup_log_rotation( + log_path: Path, + max_bytes: int = 10 * 1024 * 1024, + backup_count: int = 5, +) -> RotatingFileHandler: + """ + 配置日志文件自动轮转。 + 我实际执行时：execution_log.jsonl 增长到几百 MB，占满磁盘。 + """ + log_path.parent.mkdir(parents=True, exist_ok=True) + handler = RotatingFileHandler( + str(log_path), + maxBytes=max_bytes, + backupCount=backup_count, + encoding="utf-8", + ) + handler.setFormatter(logging.Formatter("%(message)s")) + + lib_logger = logging.getLogger("douyin_scraper") + lib_logger.setLevel(logging.DEBUG) + if not any(isinstance(h, RotatingFileHandler) for h in lib_logger.handlers): + lib_logger.addHandler(handler) + + return handler diff --git a/tools/async_file_writer.py b/tools/async_file_writer.py index d5397adb6..392fc0ef2 100644 --- a/tools/async_file_writer.py +++ b/tools/async_file_writer.py @@ -35,6 +35,13 @@ def __init__(self, platform: str, crawler_type: str): self.wordcloud_generator = AsyncWordCloudGenerator() if config.ENABLE_GET_WORDCLOUD else None def _get_file_path(self, file_type: str, item_type: str) -> str: + # T015: 当 TASK_OUTPUT_DIR 设置时，直接写稳定文件名到该目录 + if config.TASK_OUTPUT_DIR: + base_path = config.TASK_OUTPUT_DIR + pathlib.Path(base_path).mkdir(parents=True, exist_ok=True) + file_name = f"search_result.{file_type}" + return f"{base_path}/{file_name}" + if config.SAVE_DATA_PATH: base_path = f"{config.SAVE_DATA_PATH}/{self.platform}/{file_type}" else: From be8fbcec387328ef7e754c90290ad646c853e245 Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 09:26:04 +0800 Subject: [PATCH 02/32] feat(scraper): standardize search outputs in task workspaces --- api/routes.py | 14 +- api/tasks.py | 92 ++++---- douyin_scraper/core.py | 210 +++++++++++++++++- douyin_scraper/tests/test_search_outputs.py | 133 +++++++++++ .../tests/test_task_result_selection.py | 29 +++ 5 files changed, 424 insertions(+), 54 deletions(-) create mode 100644 douyin_scraper/tests/test_search_outputs.py create mode 100644 douyin_scraper/tests/test_task_result_selection.py diff --git a/api/routes.py b/api/routes.py index 4b70479ef..40c259abc 100644 --- a/api/routes.py +++ b/api/routes.py @@ -204,6 +204,15 @@ def _error_response(e: Exception) -> HTTPException: # API 端点 # ═══════════════════════════════════════════════════════════════ +def _search_output_result(paths: Dict[str, Any], output: Path) -> Dict[str, Any]: + return { + "video_jsonl": paths.get("video_jsonl", str(output)), + "video_csv": paths.get("video_csv", ""), + "csv_stats": paths.get("csv_stats", {}), + } + + + @router.post("/search", summary="触发搜索采集") async def search(req: SearchRequest) -> Dict[str, Any]: """ @@ -218,7 +227,10 @@ def _do_search() -> Dict[str, Any]: logger.info("API search request task_id=%s keywords=%r", task.task_id, req.keywords) scraper = _make_scraper(req.project_dir, task.workspace) output = scraper.search(keywords=req.keywords, max_count=req.max_count) - return {"video_jsonl": str(output), "status": scraper.get_status()} + paths = scraper.get_paths() + result = _search_output_result(paths, output) + result["status"] = scraper.get_status() + return result tm.submit(task, _do_search) return {"task_id": task.task_id, "status": "submitted", "type": "search"} diff --git a/api/tasks.py b/api/tasks.py index 58565cc2e..29e02a7a0 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -481,78 +481,66 @@ def cleanup_old_tasks(self, max_age_hours: int = 72) -> int: logger.info("清理 %d 个过期任务", removed) return removed + @staticmethod + def _search_result_names() -> tuple[str, ...]: + return ("search_result.csv", "search_result.jsonl") + + + @staticmethod + def _run_all_result_names() -> tuple[str, ...]: + return ("search_result.csv", "search_result.jsonl") + + + @classmethod + def _preferred_result_names(cls, task_type: str) -> tuple[str, ...]: + selector = getattr(cls, f"_{task_type}_result_names", None) + if selector is None: + return () + return selector() + + def get_result_path(self, task_id: str) -> Optional[Path]: - """获取任务的结果文件路径（已验证符号链接安全）""" + """Return the primary result file without changing task-specific semantics.""" task = self.get_task(task_id) if not task or task.status != "completed": return None workspace = Path(task.workspace) - - # ★ Fix 5: 优先路径 — workspace/outputs/search_result.jsonl ★ - preferred = workspace / "outputs" / "search_result.jsonl" - if preferred.exists(): + outputs_dir = workspace / "outputs" + for name in self._preferred_result_names(task.task_type): + preferred = outputs_dir / name + if not preferred.exists(): + continue try: validate_no_symlink(preferred) return preferred except HTTPException: logger.warning("结果路径含符号链接，拒绝访问: %s", preferred) - # 次选：workspace/outputs/ 下任意 jsonl（按修改时间倒序） - outputs_dir = workspace / "outputs" + # Compatibility fallback for legacy/custom outputs. if outputs_dir.exists(): - jsonl_files = sorted( - outputs_dir.rglob("*.jsonl"), - key=lambda p: p.stat().st_mtime, - reverse=True, - ) - for p in jsonl_files: - try: - validate_no_symlink(p) - return p - except HTTPException: - continue - - # 根据任务类型查找结果文件 - result_path: Optional[Path] = None - if task.task_type == "merge": - # CSV 文件 — 只搜索前 2 层深度 - csv_files = [ - p for p in workspace.rglob("*.csv") - if len(p.relative_to(workspace).parts) <= 2 - ] - if csv_files: - result_path = csv_files[0] - elif task.task_type in ("search", "comments", "scripts", "run_all"): - # JSONL 文件 — 只搜索前 2 层深度（旧版兼容兜底） - jsonl_files = [ - p for p in workspace.rglob("*.jsonl") - if len(p.relative_to(workspace).parts) <= 2 - ] - if jsonl_files: - # 优先返回最新文件（忽略已被删除的） - valid_files = [] - for p in jsonl_files: + for pattern in ("*.csv", "*.jsonl"): + candidates = sorted( + outputs_dir.rglob(pattern), + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + for candidate in candidates: try: - valid_files.append((p, p.stat().st_mtime)) - except FileNotFoundError: + validate_no_symlink(candidate) + return candidate + except HTTPException: continue - if valid_files: - result_path = sorted(valid_files, key=lambda x: x[1])[-1][0] - - # 兜底：返回 workspace 目录 - if result_path is None and workspace.exists(): - result_path = workspace - if result_path is not None: - # 符号链接安全检查 + if workspace.exists(): try: - validate_no_symlink(result_path) + validate_no_symlink(workspace) + return workspace except HTTPException: - logger.warning("结果路径含符号链接，拒绝访问: %s", result_path) + logger.warning("结果路径含符号链接，拒绝访问: %s", workspace) return None - return result_path + return None def get_stats(self) -> Dict[str, Any]: """获取任务统计（修复：枚举成员转字符串）""" diff --git a/douyin_scraper/core.py b/douyin_scraper/core.py index d741af226..7714afa25 100644 --- a/douyin_scraper/core.py +++ b/douyin_scraper/core.py @@ -241,16 +241,46 @@ def search( self._state.mark_step_started(step) try: output_path = self._do_search() + result_jsonl_path, result_csv_path = self._prepare_standard_search_outputs( + output_path + ) + + self._state.mark_step_completed( step, detail=f"output={output_path}" ) - self._paths["video_jsonl"] = output_path return output_path except Exception as e: exit_code = classify_error(e) self._state.mark_step_failed(step, str(e)[:200], exit_code) raise + def _prepare_standard_search_outputs(self, output_path: Path) -> tuple[Path, Path]: + """Create and register the standard search_result JSONL/CSV outputs.""" + outputs_dir = self._current_task_workspace_dir() / "outputs" + outputs_dir.mkdir(parents=True, exist_ok=True) + result_jsonl_path = outputs_dir / "search_result.jsonl" + result_csv_path = outputs_dir / "search_result.csv" + + if Path(output_path).resolve() != result_jsonl_path.resolve(): + shutil.copy2(output_path, result_jsonl_path) + + source_keyword = ( + self._config.keywords[0] if self._config.keywords else "unknown" + ) + csv_stats = self._convert_jsonl_to_standard_csv( + jsonl_path=str(result_jsonl_path), + csv_path=str(result_csv_path), + source_keyword=source_keyword, + ) + logger.info("CSV conversion stats: %s", csv_stats) + + self._paths["video_jsonl"] = result_jsonl_path + self._paths["video_csv"] = result_csv_path + self._paths["csv_stats"] = csv_stats + return result_jsonl_path, result_csv_path + + def fetch_comments( self, video_jsonl: Optional[Path] = None, @@ -406,6 +436,36 @@ def reset_step(self, step: str, clear_dedupe: bool = False) -> None: # 内部实现 # ═══════════════════════════════════════════════════════════════ + def get_paths(self) -> Dict[str, Any]: + """Return output paths grouped by release feature.""" + result: Dict[str, Any] = {} + result.update(self._search_output_paths()) + return result + + + def _select_output_values( + self, + path_keys: Sequence[str] = (), + stats_keys: Sequence[str] = (), + ) -> Dict[str, Any]: + selected: Dict[str, Any] = {} + for key in path_keys: + value = self._paths.get(key) + if value is not None: + selected[key] = str(value) + for key in stats_keys: + if key in self._paths: + selected[key] = self._paths[key] + return selected + + + def _search_output_paths(self) -> Dict[str, Any]: + return self._select_output_values( + path_keys=("video_jsonl", "video_csv"), + stats_keys=("csv_stats",), + ) + + def _require_step_ready(self, step: str) -> None: """检查步骤是否可执行，否则抛异常""" if not self._state.check_step_ready(step): @@ -1015,6 +1075,16 @@ def _do_merge( # 辅助方法 # ═══════════════════════════════════════════════════════════════ + def _current_task_workspace_dir(self) -> Path: + """Return the current API task workspace from project_dir/state_dir_name.""" + state_parent = Path(self._config.state_dir_name).parent + if str(state_parent) in ("", "."): + return self._config.project_dir + if self._config.project_dir.name == state_parent.name: + return self._config.project_dir + return self._config.project_dir / state_parent + + def _get_venv_python(self) -> Path: """获取 venv 中的 Python 路径，不存在时回退到当前 Python""" project_dir = self._config.project_dir @@ -1048,6 +1118,144 @@ def _load_jsonl(filepath: Path) -> list: ) return records + def _convert_jsonl_to_standard_csv( + self, + jsonl_path: str, + csv_path: str, + source_keyword: str, + ) -> dict: + """ + 将 search_result.jsonl 转换为标准 CSV（T016）。 + + 返回: { + "rows_in": int, + "rows_out": int, + "duplicates_removed": int, + "csv_generated": bool, + "csv_error": str or None + } + """ + import csv as csv_module + + jsonl_f = Path(jsonl_path) + csv_f = Path(csv_path) + stats = { + "rows_in": 0, + "rows_out": 0, + "duplicates_removed": 0, + "csv_generated": False, + "csv_error": None, + } + + try: + # 读取 JSONL + records = self._load_jsonl(jsonl_f) + stats["rows_in"] = len(records) + if not records: + stats["csv_error"] = "JSONL 文件为空" + return stats + + # 去重（按 aweme_id 或 aweme_url，保留第一条） + seen_ids: set = set() + seen_urls: set = set() + deduped = [] + for rec in records: + aid = str(rec.get("aweme_id", "")).strip() + aurl = str(rec.get("aweme_url", "")).strip() + if aid and aid != "None": + if aid in seen_ids: + continue + seen_ids.add(aid) + elif aurl: + if aurl in seen_urls: + continue + seen_urls.add(aurl) + # 如果既没有 aid 也没有 url，保留（无法去重） + deduped.append(rec) + + stats["duplicates_removed"] = stats["rows_in"] - len(deduped) + + # 转换为标准行 + rows = [] + for rec in deduped: + aid = str(rec.get("aweme_id", "")).strip() + aurl = str(rec.get("aweme_url", "")).strip() + video_id = aid if aid and aid != "None" else "" + + # 整数转换（失败则 0） + def _safe_int(val): + try: + return int(val) + except (ValueError, TypeError): + return 0 + + likes = _safe_int(rec.get("liked_count", 0)) + collected = _safe_int(rec.get("collected_count", 0)) + comments = _safe_int(rec.get("comment_count", 0)) + shares = _safe_int(rec.get("share_count", 0)) + total_engagement = likes + collected + comments + shares + + rows.append({ + "source_keyword": source_keyword, + "platform": "douyin", + "video_id": video_id, + "aweme_id": aid if aid and aid != "None" else "", + "title": str(rec.get("title", "")), + "desc": str(rec.get("desc", "")), + "nickname": str(rec.get("nickname", "")), + "liked_count": likes, + "collected_count": collected, + "comment_count": comments, + "share_count": shares, + "total_engagement": total_engagement, + "aweme_url": aurl, + "cover_url": str(rec.get("cover_url", "")), + "video_download_url": str(rec.get("video_download_url", "")), + "music_download_url": str(rec.get("music_download_url", "")), + "create_time": str(rec.get("create_time", "")), + "last_modify_ts": str(rec.get("last_modify_ts", "")), + }) + + stats["rows_out"] = len(rows) + + # 写入 CSV（UTF-8-SIG 编码，Excel 兼容） + csv_f.parent.mkdir(parents=True, exist_ok=True) + fieldnames = [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "title", + "desc", + "nickname", + "liked_count", + "collected_count", + "comment_count", + "share_count", + "total_engagement", + "aweme_url", + "cover_url", + "video_download_url", + "music_download_url", + "create_time", + "last_modify_ts", + ] + with open(str(csv_f), "w", encoding="utf-8-sig", newline="") as f: + writer = csv_module.DictWriter(f, fieldnames=fieldnames, quoting=csv_module.QUOTE_ALL) + writer.writeheader() + writer.writerows(rows) + + stats["csv_generated"] = True + logger.info( + "CSV 转换完成: %s (%d 行, 去重 %d)", csv_f, len(rows), stats["duplicates_removed"] + ) + except Exception as e: + stats["csv_error"] = str(e)[:500] + logger.error("CSV 转换失败: %s", e) + + return stats + + def _deduplicate(self, records: list, key: str = "aweme_id") -> list: """ 按 key 去重，优先保留 script_status=success。 diff --git a/douyin_scraper/tests/test_search_outputs.py b/douyin_scraper/tests/test_search_outputs.py new file mode 100644 index 000000000..db044a430 --- /dev/null +++ b/douyin_scraper/tests/test_search_outputs.py @@ -0,0 +1,133 @@ +import csv +import json +from pathlib import Path + +from douyin_scraper import DouyinScraper + + +STANDARD_SEARCH_FIELDS = [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "title", + "desc", + "nickname", + "liked_count", + "collected_count", + "comment_count", + "share_count", + "total_engagement", + "aweme_url", + "cover_url", + "video_download_url", + "music_download_url", + "create_time", + "last_modify_ts", +] + + +def _scraper(tmp_path: Path) -> DouyinScraper: + return DouyinScraper( + { + "project_dir": str(tmp_path), + "state_dir_name": "workspaces/search-task/state", + } + ) + + +def test_search_writes_standard_workspace_outputs(tmp_path: Path) -> None: + scraper = _scraper(tmp_path) + source = tmp_path / "raw-search.jsonl" + source.write_text( + json.dumps( + { + "aweme_id": "search-1", + "title": "停车技巧", + "liked_count": 2, + "collected_count": 3, + "comment_count": 4, + "share_count": 5, + "aweme_url": "https://www.douyin.com/video/search-1", + }, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + + scraper.config.keywords = ["停车"] + result_jsonl, result_csv = scraper._prepare_standard_search_outputs(source) + + outputs = tmp_path / "workspaces" / "search-task" / "outputs" + assert result_jsonl == outputs / "search_result.jsonl" + assert result_csv == outputs / "search_result.csv" + assert result_jsonl.read_text(encoding="utf-8") + assert result_csv.exists() + paths = scraper.get_paths() + assert paths["video_jsonl"] == str(outputs / "search_result.jsonl") + assert paths["video_csv"] == str(outputs / "search_result.csv") + assert paths["csv_stats"]["rows_out"] == 1 + + +def test_standard_search_csv_has_18_fields_bom_dedupe_and_engagement( + tmp_path: Path, +) -> None: + scraper = _scraper(tmp_path) + source = tmp_path / "search_result.jsonl" + target = tmp_path / "search_result.csv" + records = [ + { + "aweme_id": "id-1", + "title": "first", + "liked_count": "10", + "collected_count": "3", + "comment_count": "2", + "share_count": "1", + "aweme_url": "https://www.douyin.com/video/id-1", + }, + { + "aweme_id": "id-1", + "title": "duplicate id", + "aweme_url": "https://www.douyin.com/video/other", + }, + { + "title": "url only", + "liked_count": "bad", + "collected_count": 1, + "comment_count": 2, + "share_count": 3, + "aweme_url": "https://www.douyin.com/video/url-only", + }, + { + "title": "duplicate url", + "aweme_url": "https://www.douyin.com/video/url-only", + }, + ] + source.write_text( + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in records), + encoding="utf-8", + ) + + stats = scraper._convert_jsonl_to_standard_csv( + str(source), + str(target), + source_keyword="停车", + ) + + assert stats == { + "rows_in": 4, + "rows_out": 2, + "duplicates_removed": 2, + "csv_generated": True, + "csv_error": None, + } + assert target.read_bytes()[:3] == b"\xef\xbb\xbf" + with open(target, "r", encoding="utf-8-sig", newline="") as handle: + reader = csv.DictReader(handle) + rows = list(reader) + assert reader.fieldnames == STANDARD_SEARCH_FIELDS + assert len(rows) == 2 + assert rows[0]["source_keyword"] == "停车" + assert rows[0]["total_engagement"] == "16" + assert rows[1]["total_engagement"] == "6" diff --git a/douyin_scraper/tests/test_task_result_selection.py b/douyin_scraper/tests/test_task_result_selection.py new file mode 100644 index 000000000..1e12e80a6 --- /dev/null +++ b/douyin_scraper/tests/test_task_result_selection.py @@ -0,0 +1,29 @@ +import pytest + +from pathlib import Path + +from api.tasks import TaskManager + + +def _selected_result(tmp_path: Path, task_type: str, files: tuple[str, ...]): + manager = TaskManager(base_dir=str(tmp_path)) + task = manager.create_task(task_type) + task.status = "completed" + outputs = Path(task.workspace) / "outputs" + outputs.mkdir(parents=True, exist_ok=True) + for name in files: + (outputs / name).write_text("value\n", encoding="utf-8") + return manager.get_result_path(task.task_id) + + +@pytest.mark.parametrize("task_type", ["search", "run_all"]) +def test_search_selector_prefers_standard_csv(tmp_path: Path, task_type: str) -> None: + result = _selected_result(tmp_path, task_type, ("search_result.jsonl", "search_result.csv")) + assert result is not None + assert result.name == "search_result.csv" + + +def test_fallback_selector_uses_supported_file(tmp_path: Path) -> None: + result = _selected_result(tmp_path, "custom", ("fallback.csv",)) + assert result is not None + assert result.name == "fallback.csv" From 00a4946cc5dbe31a163b78d4bfa3f85ba5885ee0 Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 09:36:54 +0800 Subject: [PATCH 03/32] feat(scraper): add raw and cleaned comment outputs --- api/routes.py | 58 +- api/tasks.py | 10 + douyin_scraper/core.py | 505 ++++++++++++++++-- douyin_scraper/tests/test_comments_outputs.py | 167 ++++++ 4 files changed, 697 insertions(+), 43 deletions(-) create mode 100644 douyin_scraper/tests/test_comments_outputs.py diff --git a/api/routes.py b/api/routes.py index 40c259abc..6398896e7 100644 --- a/api/routes.py +++ b/api/routes.py @@ -91,6 +91,11 @@ def validate_keywords(cls, v: List[str]) -> List[str]: class CommentsRequest(BaseModel): """评论采集请求""" + task_id: Optional[str] = Field(None, description="搜索任务 ID") + video_ids: Optional[List[str]] = Field(None, description="直接指定视频 ID 列表") + max_comments_per_video: int = Field( + 50, description="每个视频最多采集评论数", ge=1, le=5000 + ) video_jsonl: Optional[str] = Field(None, description="视频 JSONL 路径") project_dir: Optional[str] = Field(None, description="工作目录") @@ -213,6 +218,23 @@ def _search_output_result(paths: Dict[str, Any], output: Path) -> Dict[str, Any] +def _comments_output_result( + paths: Dict[str, Any], + output: Path, +) -> Dict[str, Any]: + comments_raw_jsonl = paths.get("comments_raw_jsonl", str(output)) + return { + "comments_jsonl": comments_raw_jsonl, + "comments_raw_jsonl": comments_raw_jsonl, + "comments_raw_csv": paths.get("comments_raw_csv", ""), + "comments_clean_jsonl": paths.get("comments_clean_jsonl", ""), + "comments_clean_csv": paths.get("comments_clean_csv", ""), + "comments_stats": paths.get("comments_stats", {}), + "clean_stats": paths.get("clean_stats", {}), + } + + + @router.post("/search", summary="触发搜索采集") async def search(req: SearchRequest) -> Dict[str, Any]: """ @@ -247,13 +269,41 @@ async def fetch_comments(req: CommentsRequest) -> Dict[str, Any]: def _do_comments() -> Dict[str, Any]: scraper = _make_scraper(req.project_dir, task.workspace) - video_path = Path(req.video_jsonl) if req.video_jsonl else None - if video_path: + source_task_id = req.task_id + video_path: Optional[Path] = None + if req.task_id: + source_task = tm.get_task(req.task_id) + if not source_task: + raise NonRetryableError( + f"搜索任务不存在: {req.task_id}", + step="fetch_comments", + ) + source_outputs = (Path(source_task.workspace) / "outputs").resolve() + csv_path = source_outputs / "search_result.csv" + jsonl_path = source_outputs / "search_result.jsonl" + if csv_path.exists(): + video_path = validate_path_in_workspace(str(csv_path.resolve()), source_outputs) + elif jsonl_path.exists(): + video_path = validate_path_in_workspace(str(jsonl_path.resolve()), source_outputs) + else: + raise NonRetryableError( + f"搜索任务无可用输出: {req.task_id}", + step="fetch_comments", + ) + elif req.video_jsonl: video_path = validate_path_in_workspace( req.video_jsonl, Path(task.workspace) ) - output = scraper.fetch_comments(video_jsonl=video_path) - return {"comments_jsonl": str(output), "status": scraper.get_status()} + output = scraper.fetch_comments( + video_jsonl=video_path, + video_ids=req.video_ids, + source_task_id=source_task_id, + max_comments_per_video=req.max_comments_per_video, + ) + paths = scraper.get_paths() + result = _comments_output_result(paths, output) + result["status"] = scraper.get_status() + return result tm.submit(task, _do_comments) return {"task_id": task.task_id, "status": "submitted", "type": "comments"} diff --git a/api/tasks.py b/api/tasks.py index 29e02a7a0..e629f1550 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -491,6 +491,16 @@ def _run_all_result_names() -> tuple[str, ...]: return ("search_result.csv", "search_result.jsonl") + @staticmethod + def _comments_result_names() -> tuple[str, ...]: + return ( + "comments_clean.csv", + "comments_clean.jsonl", + "comments_raw.csv", + "comments_raw.jsonl", + ) + + @classmethod def _preferred_result_names(cls, task_type: str) -> tuple[str, ...]: selector = getattr(cls, f"_{task_type}_result_names", None) diff --git a/douyin_scraper/core.py b/douyin_scraper/core.py index 7714afa25..6b1798a80 100644 --- a/douyin_scraper/core.py +++ b/douyin_scraper/core.py @@ -27,12 +27,14 @@ import logging import os import random +import re import shutil import subprocess import sys import tempfile import threading import time +import unicodedata from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path @@ -284,6 +286,9 @@ def _prepare_standard_search_outputs(self, output_path: Path) -> tuple[Path, Pat def fetch_comments( self, video_jsonl: Optional[Path] = None, + video_ids: Optional[List[str]] = None, + source_task_id: Optional[str] = None, + max_comments_per_video: int = 200, ) -> Path: """ 对已有视频采集评论，返回评论 JSONL 路径。 @@ -292,25 +297,56 @@ def fetch_comments( self._require_step_ready(step) input_path = video_jsonl or self._paths.get("video_jsonl") - if not input_path or not input_path.exists(): + if input_path and not input_path.exists(): raise NonRetryableError( f"视频 JSONL 不存在: {input_path}", step=step, ) + if not input_path and not video_ids: + raise NonRetryableError( + "评论采集缺少输入：请提供 task_id/search_result 文件或 video_ids", + step=step, + ) self._state.mark_step_started(step) try: - output_path = self._do_fetch_comments(input_path) + output_path = self._do_fetch_comments( + input_path=input_path, + video_ids=video_ids, + source_task_id=source_task_id, + max_comments_per_video=max_comments_per_video, + ) + clean_jsonl_path, clean_csv_path, clean_stats = self._do_clean_comments(output_path) self._state.mark_step_completed( step, detail=f"output={output_path}" ) - self._paths["comments_jsonl"] = output_path + self._register_comments_outputs( + output_path, + clean_jsonl_path, + clean_csv_path, + clean_stats, + ) return output_path except Exception as e: exit_code = classify_error(e) self._state.mark_step_failed(step, str(e)[:200], exit_code) raise + def _register_comments_outputs( + self, + raw_jsonl: Path, + clean_jsonl: Path, + clean_csv: Path, + clean_stats: Dict[str, Any], + ) -> None: + """Register comments_raw/comments_clean outputs for get_paths().""" + self._paths["comments_jsonl"] = raw_jsonl + self._paths["comments_raw_jsonl"] = raw_jsonl + self._paths["comments_clean_jsonl"] = clean_jsonl + self._paths["comments_clean_csv"] = clean_csv + self._paths["clean_stats"] = clean_stats + + def extract_scripts( self, video_jsonl: Optional[Path] = None, @@ -440,6 +476,7 @@ def get_paths(self) -> Dict[str, Any]: """Return output paths grouped by release feature.""" result: Dict[str, Any] = {} result.update(self._search_output_paths()) + result.update(self._comments_output_paths()) return result @@ -466,6 +503,19 @@ def _search_output_paths(self) -> Dict[str, Any]: ) + def _comments_output_paths(self) -> Dict[str, Any]: + return self._select_output_values( + path_keys=( + "comments_jsonl", + "comments_raw_jsonl", + "comments_raw_csv", + "comments_clean_jsonl", + "comments_clean_csv", + ), + stats_keys=("comments_stats", "clean_stats"), + ) + + def _require_step_ready(self, step: str) -> None: """检查步骤是否可执行，否则抛异常""" if not self._state.check_step_ready(step): @@ -690,48 +740,81 @@ def _extract_new_lines(self, file_path: Path, skip_count: int) -> List[str]: lines.append(line) return lines - def _do_fetch_comments(self, input_path: Path) -> Path: - """ - 对已有视频采集评论。 - v5: 内置评论采集逻辑，不再依赖外部脚本。 - """ - output_dir = self._config.jsonl_dir - actual_dir = ensure_dir_writable(output_dir, self._config.fallback_dir) - output_path = actual_dir / "search_comments.jsonl" - - # 读取视频列表 - videos = self._load_jsonl(input_path) - video_ids = [str(v.get("aweme_id", "")) for v in videos if v.get("aweme_id")] - - # 断点续传 - done_ids = self._state.load_completed_ids_from_jsonl(output_path, "aweme_id") + def _do_fetch_comments( + self, + input_path: Optional[Path], + video_ids: Optional[List[str]] = None, + source_task_id: Optional[str] = None, + max_comments_per_video: int = 200, + ) -> Path: + """Collect comments and write standard comments_raw outputs.""" + workspace_dir = self._current_task_workspace_dir() + output_dir = workspace_dir / "outputs" + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / "comments_raw.jsonl" + csv_path = output_dir / "comments_raw.csv" + output_path.touch(exist_ok=True) - logger.info("视频 %d 个, 已完成评论 %d 个", len(video_ids), len(done_ids)) + videos_in = len(self._load_comment_video_records(input_path, video_ids)) + if videos_in <= 0: + raise NonRetryableError("评论采集输入中没有可用视频 ID", step=self.STEP_COMMENTS) - # 这里只是占位：实际评论采集需要 Chrome CDP 交互 - # MediaCrawler 的 crawl_comments_v2.py 负责实际采集 - # v5 通过子进程调用 venv_python = self._get_venv_python() comments_script = self._config.project_dir / "crawl_comments_v2.py" + if not comments_script.exists(): + raise NonRetryableError( + f"评论采集脚本不存在: {comments_script}", + step=self.STEP_COMMENTS, + ) - if comments_script.exists(): - logger.info("调用评论采集子进程: %s", comments_script) - try: - subprocess.run( - [str(venv_python), str(comments_script)], - cwd=str(self._config.project_dir), - check=True, timeout=3600, - ) - except subprocess.CalledProcessError as exc: - logger.error( - "评论采集子进程失败 (exit %d): %s", - exc.returncode, exc, - ) - raise NonRetryableError( - f"评论采集子进程失败 (exit {exc.returncode}): {exc}", - step="fetch_comments", - ) from exc + cmd = [ + str(venv_python), + str(comments_script), + "--output", + str(output_path), + "--max-comments", + str(max_comments_per_video), + "--skip-existing", + ] + if input_path: + cmd.extend(["--input", str(input_path)]) + if video_ids: + cmd.extend(["--video-ids", ",".join(str(v) for v in video_ids)]) + if source_task_id: + cmd.extend(["--source-task-id", source_task_id]) + env = os.environ.copy() + env["PYTHONUTF8"] = "1" + env["PYTHONIOENCODING"] = "utf-8" + env["LANG"] = "C.UTF-8" + env["LC_ALL"] = "C.UTF-8" + + logger.info("调用评论采集子进程: %s", " ".join(cmd)) + subprocess_error: Optional[str] = None + try: + subprocess.run( + cmd, + cwd=str(self._config.project_dir), + check=True, + timeout=3600, + env=env, + ) + except subprocess.CalledProcessError as exc: + logger.error("评论采集子进程失败 (exit %d): %s", exc.returncode, exc) + subprocess_error = f"评论采集子进程失败 (exit {exc.returncode}): {exc}" + except subprocess.TimeoutExpired as exc: + logger.error("评论采集子进程超时: %s", exc) + subprocess_error = f"评论采集子进程超时: {exc}" + + comments_stats = self._convert_comments_jsonl_to_csv( + jsonl_path=output_path, + csv_path=csv_path, + videos_in=videos_in, + ) + if subprocess_error: + comments_stats["errors"].append(subprocess_error) + self._paths["comments_raw_csv"] = csv_path + self._paths["comments_stats"] = comments_stats return output_path def _do_extract_scripts(self, input_path: Path, model: str) -> Path: @@ -1085,6 +1168,46 @@ def _current_task_workspace_dir(self) -> Path: return self._config.project_dir / state_parent + @staticmethod + def _comment_video_id(record: Dict[str, Any]) -> str: + for key in ("aweme_id", "video_id", "note_id", "item_id"): + value = record.get(key) + if value is not None and str(value).strip() and str(value).strip() != "None": + return str(value).strip() + return "" + + + def _load_comment_video_records( + self, + input_path: Optional[Path], + video_ids: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + records: List[Dict[str, Any]] = [] + + if input_path: + if input_path.suffix.lower() == ".csv": + with open(str(input_path), "r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + records.extend(dict(row) for row in reader) + else: + records.extend(self._load_jsonl(input_path)) + + for video_id in video_ids or []: + value = str(video_id).strip() + if value: + records.append({"aweme_id": value, "video_id": value}) + + seen: Set[str] = set() + unique_records: List[Dict[str, Any]] = [] + for record in records: + aweme_id = self._comment_video_id(record) + if not aweme_id or aweme_id in seen: + continue + seen.add(aweme_id) + unique_records.append(record) + return unique_records + + def _get_venv_python(self) -> Path: """获取 venv 中的 Python 路径，不存在时回退到当前 Python""" project_dir = self._config.project_dir @@ -1118,6 +1241,310 @@ def _load_jsonl(filepath: Path) -> list: ) return records + @staticmethod + def _safe_int(value: Any) -> int: + try: + return int(value) + except (TypeError, ValueError): + return 0 + + + def _convert_comments_jsonl_to_csv( + self, + jsonl_path: Path, + csv_path: Path, + videos_in: int, + ) -> Dict[str, Any]: + fieldnames = [ + "source_keyword", + "platform", + "source_task_id", + "video_id", + "aweme_id", + "aweme_url", + "comment_id", + "parent_comment_id", + "user_id", + "nickname", + "content", + "liked_count", + "reply_count", + "create_time", + "ip_location", + "crawl_time", + ] + stats: Dict[str, Any] = { + "videos_in": videos_in, + "videos_success": 0, + "comments_out": 0, + "comments_csv_generated": False, + "errors": [], + } + rows: List[Dict[str, Any]] = [] + success_video_ids: Set[str] = set() + + if jsonl_path.exists(): + with open(str(jsonl_path), "r", encoding="utf-8") as f: + for line_num, line in enumerate(f, 1): + if not line.strip(): + continue + try: + rec = json.loads(line) + except json.JSONDecodeError as exc: + stats["errors"].append(f"line {line_num}: {str(exc)[:120]}") + continue + + liked_count = self._safe_int(rec.get("liked_count", 0)) + reply_count = self._safe_int(rec.get("reply_count", 0)) + aweme_id = str(rec.get("aweme_id") or rec.get("video_id") or "").strip() + video_id = str(rec.get("video_id") or aweme_id).strip() + if aweme_id or video_id: + success_video_ids.add(aweme_id or video_id) + + rows.append({ + "source_keyword": str(rec.get("source_keyword", "")), + "platform": str(rec.get("platform") or "douyin"), + "source_task_id": str(rec.get("source_task_id", "")), + "video_id": video_id, + "aweme_id": aweme_id, + "aweme_url": str(rec.get("aweme_url", "")), + "comment_id": str(rec.get("comment_id", "")), + "parent_comment_id": str(rec.get("parent_comment_id", "")), + "user_id": str(rec.get("user_id", "")), + "nickname": str(rec.get("nickname", "")), + "content": str(rec.get("content", "")), + "liked_count": liked_count, + "reply_count": reply_count, + "create_time": str(rec.get("create_time", "")), + "ip_location": str(rec.get("ip_location", "")), + "crawl_time": str(rec.get("crawl_time", "")), + }) + + csv_path.parent.mkdir(parents=True, exist_ok=True) + with open(str(csv_path), "w", encoding="utf-8-sig", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + stats["videos_success"] = len(success_video_ids) + stats["comments_out"] = len(rows) + stats["comments_csv_generated"] = True + return stats + + + @staticmethod + def _normalize_comment_text(value: Any) -> str: + text = "" if value is None else str(value) + text = unicodedata.normalize("NFKC", text) + text = re.sub(r"\s+", " ", text).strip() + text = re.sub(r"([!?！？。,.，、~～])\1{2,}", r"\1\1", text) + return text + + + @staticmethod + def _comment_compact_key(text: str) -> str: + return re.sub(r"[\W_]+", "", text, flags=re.UNICODE).lower() + + + @staticmethod + def _is_emoji_only(text: str) -> bool: + compact = text.strip() + if not compact: + return False + meaningful = re.search(r"[\u4e00-\u9fffA-Za-z0-9]", compact) + if meaningful: + return False + return any(unicodedata.category(ch).startswith("S") for ch in compact) + + + @staticmethod + def _extract_pain_tags(text: str) -> List[str]: + rules = [ + ("压线", r"压线|压到线|轧线|踩线"), + ("30公分", r"30\s*公分|三十\s*公分|30\s*厘米|三十\s*厘米"), + ("看点不准", r"看不准|看点|点位|点不准|找不准"), + ("后视镜", r"后视镜|镜子|倒车镜"), + ("方向盘", r"方向盘|打方向|回方向"), + ("靠边太宽", r"太宽|靠边.*宽|离边.*远|距离.*宽"), + ("挂科", r"挂科|挂了|挂的|没过|不过|不合格"), + ("紧张", r"紧张|一紧张|慌|心态|害怕|怕"), + ("考试忘步骤", r"忘步骤|忘了|忘记|想不起来|流程|顺序"), + ("教练讲不清", r"教练.*没讲|教练.*讲不清|没讲清|讲不清|不清楚"), + ("想要方法", r"方法|技巧|教程|怎么练|求教|教一下|有没有.*办法"), + ] + tags: List[str] = [] + for tag, pattern in rules: + if re.search(pattern, text, flags=re.IGNORECASE): + tags.append(tag) + return tags + + + @staticmethod + def _infer_comment_intent(text: str, pain_tags: List[str], invalid_reason: str) -> str: + if invalid_reason: + if invalid_reason in {"ad_spam", "contact_spam"}: + return "spam" + if invalid_reason == "irrelevant": + return "irrelevant" + return "meaningless" + if re.search(r"怎么|咋|如何|怎么看|怎么.*看|吗|么|哪里|到底|[?？]", text): + return "question" + if re.search(r"求|想要|有没有.*方法|教一下|详细|展开", text): + return "request_more_detail" + if re.search(r"不对|不是|没用|不同意|反而|但是", text): + return "objection" + if pain_tags: + return "pain" + if re.search(r"我也|同感|确实|对对|一样|赞同", text): + return "agreement" + if re.search(r"我|自己|考试|练车|教练|科目|挂|总是|老是", text): + return "experience" + return "experience" + + + def _classify_clean_comment( + self, + rec: Dict[str, Any], + seen_content: Set[str], + ) -> Dict[str, Any]: + raw_content = str(rec.get("content", "")) + clean_content = self._normalize_comment_text(raw_content) + compact = self._comment_compact_key(clean_content) + pain_tags = self._extract_pain_tags(clean_content) + invalid_reason = "" + + if not clean_content: + invalid_reason = "empty_comment" + elif self._is_emoji_only(clean_content): + invalid_reason = "emoji_only" + elif compact in seen_content: + invalid_reason = "duplicate" + elif re.search(r"(微信|加v|加V|加微|v信|vx|VX|qq|QQ|电话|手机号|联系我)", clean_content): + invalid_reason = "contact_spam" + elif re.search(r"(包过|代考|办证|引流|优惠|推广|广告|招生|私教|直播间)", clean_content): + invalid_reason = "ad_spam" + elif re.search(r"(傻逼|sb|SB|滚|去死|脑残)", clean_content): + invalid_reason = "attack_or_abuse" + elif not pain_tags and re.fullmatch(r"(6+|哈+|哈哈+|笑死+|路过|打卡|赞+|牛+|666+)", compact): + invalid_reason = "meaningless" + elif not pain_tags and len(compact) <= 2: + invalid_reason = "too_short" + elif not pain_tags and re.search(r"(音乐|衣服|美女|帅哥|主播|bgm|BGM|哪里买)", clean_content): + invalid_reason = "irrelevant" + + if compact and invalid_reason != "duplicate": + seen_content.add(compact) + + is_valid = not invalid_reason + intent_type = self._infer_comment_intent(clean_content, pain_tags, invalid_reason) + confidence = 0.9 if is_valid and pain_tags else 0.8 if is_valid else 0.75 + + return { + "source_keyword": str(rec.get("source_keyword", "")), + "platform": str(rec.get("platform") or "douyin"), + "source_task_id": str(rec.get("source_task_id", "")), + "video_id": str(rec.get("video_id") or rec.get("aweme_id") or ""), + "aweme_id": str(rec.get("aweme_id") or rec.get("video_id") or ""), + "aweme_url": str(rec.get("aweme_url", "")), + "comment_id": str(rec.get("comment_id", "")), + "raw_content": raw_content, + "clean_content": clean_content, + "is_valid": is_valid, + "invalid_reason": invalid_reason, + "pain_tags": pain_tags, + "intent_type": intent_type, + "confidence": confidence, + "liked_count": self._safe_int(rec.get("liked_count", 0)), + "reply_count": self._safe_int(rec.get("reply_count", 0)), + "create_time": str(rec.get("create_time", "")), + "crawl_time": str(rec.get("crawl_time", "")), + } + + + def _do_clean_comments(self, raw_jsonl_path: Path) -> tuple[Path, Path, Dict[str, Any]]: + clean_jsonl_path = raw_jsonl_path.with_name("comments_clean.jsonl") + clean_csv_path = raw_jsonl_path.with_name("comments_clean.csv") + fieldnames = [ + "source_keyword", + "platform", + "source_task_id", + "video_id", + "aweme_id", + "aweme_url", + "comment_id", + "raw_content", + "clean_content", + "is_valid", + "invalid_reason", + "pain_tags", + "intent_type", + "confidence", + "liked_count", + "reply_count", + "create_time", + "crawl_time", + ] + stats: Dict[str, Any] = { + "comments_in": 0, + "comments_valid": 0, + "comments_invalid": 0, + "duplicates_removed": 0, + "clean_csv_generated": False, + "errors": [], + } + rows: List[Dict[str, Any]] = [] + seen_content: Set[str] = set() + + if raw_jsonl_path.exists(): + with open(str(raw_jsonl_path), "r", encoding="utf-8") as f: + for line_num, line in enumerate(f, 1): + if not line.strip(): + continue + try: + rec = json.loads(line) + except json.JSONDecodeError as exc: + stats["errors"].append(f"line {line_num}: {str(exc)[:120]}") + continue + stats["comments_in"] += 1 + row = self._classify_clean_comment(rec, seen_content) + if row["is_valid"]: + stats["comments_valid"] += 1 + else: + stats["comments_invalid"] += 1 + if row["invalid_reason"] == "duplicate": + stats["duplicates_removed"] += 1 + rows.append(row) + + clean_jsonl_path.parent.mkdir(parents=True, exist_ok=True) + with open(str(clean_jsonl_path), "w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + with open(str(clean_csv_path), "w", encoding="utf-8-sig", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for row in rows: + csv_row = dict(row) + csv_row["pain_tags"] = "|".join(row["pain_tags"]) + writer.writerow(csv_row) + + stats["clean_csv_generated"] = True + return clean_jsonl_path, clean_csv_path, stats + + + @staticmethod + def _dedupe_text_list(items: Sequence[str]) -> List[str]: + result: List[str] = [] + seen: Set[str] = set() + for item in items: + text = str(item or "").strip() + if not text or text in seen: + continue + seen.add(text) + result.append(text) + return result + + def _convert_jsonl_to_standard_csv( self, jsonl_path: str, diff --git a/douyin_scraper/tests/test_comments_outputs.py b/douyin_scraper/tests/test_comments_outputs.py new file mode 100644 index 000000000..60e2c9328 --- /dev/null +++ b/douyin_scraper/tests/test_comments_outputs.py @@ -0,0 +1,167 @@ +import csv +import json +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +from douyin_scraper import DouyinScraper + + +def _scraper(tmp_path: Path) -> DouyinScraper: + return DouyinScraper( + { + "project_dir": str(tmp_path), + "state_dir_name": "workspaces/comments-task/state", + } + ) + + +def _comment(comment_id: str, content: str) -> dict: + return { + "source_keyword": "靠边停车", + "platform": "douyin", + "source_task_id": "search-task", + "video_id": "video-1", + "aweme_id": "video-1", + "aweme_url": "https://www.douyin.com/video/video-1", + "comment_id": comment_id, + "content": content, + "liked_count": "7", + "reply_count": "2", + "create_time": "1710000000", + "crawl_time": "2026-06-16T00:00:00Z", + } + + +def _source_csv(tmp_path: Path) -> Path: + source = tmp_path / "search_result.csv" + source.write_text( + "\ufeffsource_keyword,platform,video_id,aweme_id,aweme_url\n" + "停车,douyin,video-1,video-1,https://www.douyin.com/video/video-1\n", + encoding="utf-8", + ) + return source + + +def test_comments_raw_csv_export(tmp_path: Path) -> None: + scraper = _scraper(tmp_path) + jsonl_path = tmp_path / "comments_raw.jsonl" + csv_path = tmp_path / "comments_raw.csv" + jsonl_path.write_text( + json.dumps(_comment("comment-1", "这个方法很实用"), ensure_ascii=False) + + "\n", + encoding="utf-8", + ) + + stats = scraper._convert_comments_jsonl_to_csv( + jsonl_path, + csv_path, + videos_in=1, + ) + + assert stats["videos_in"] == 1 + assert stats["videos_success"] == 1 + assert stats["comments_out"] == 1 + assert stats["comments_csv_generated"] is True + assert csv_path.read_bytes()[:3] == b"\xef\xbb\xbf" + with open(csv_path, "r", encoding="utf-8-sig", newline="") as handle: + row = next(csv.DictReader(handle)) + assert row["content"] == "这个方法很实用" + assert row["liked_count"] == "7" + + +def test_comments_clean_rule_export(tmp_path: Path) -> None: + scraper = _scraper(tmp_path) + raw_jsonl = tmp_path / "comments_raw.jsonl" + records = [ + _comment("empty", ""), + _comment("emoji", "😂😂"), + _comment("pain", "30公分看不准，后视镜怎么看？"), + _comment("duplicate", "30公分看不准，后视镜怎么看？"), + ] + raw_jsonl.write_text( + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in records), + encoding="utf-8", + ) + + clean_jsonl, clean_csv, stats = scraper._do_clean_comments(raw_jsonl) + + assert stats["comments_in"] == 4 + assert stats["duplicates_removed"] == 1 + assert stats["clean_csv_generated"] is True + assert clean_csv.read_bytes()[:3] == b"\xef\xbb\xbf" + rows = [ + json.loads(line) + for line in clean_jsonl.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + by_id = {row["comment_id"]: row for row in rows} + assert by_id["empty"]["invalid_reason"] == "empty_comment" + assert by_id["emoji"]["invalid_reason"] == "emoji_only" + assert by_id["duplicate"]["invalid_reason"] == "duplicate" + assert by_id["pain"]["is_valid"] is True + assert by_id["pain"]["intent_type"] == "question" + + +def test_fetch_comments_writes_workspace_outputs(tmp_path: Path) -> None: + scraper = _scraper(tmp_path) + (tmp_path / "crawl_comments_v2.py").write_text("# stub\n", encoding="utf-8") + source_csv = _source_csv(tmp_path) + + def fake_run(cmd, **kwargs): + output_path = Path(cmd[cmd.index("--output") + 1]) + output_path.write_text( + json.dumps( + _comment("comment-1", "30公分看不准"), + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + return MagicMock(returncode=0) + + with patch("douyin_scraper.core.subprocess.run", side_effect=fake_run): + output = scraper.fetch_comments( + video_jsonl=source_csv, + source_task_id="search-task", + max_comments_per_video=50, + ) + + outputs = tmp_path / "workspaces" / "comments-task" / "outputs" + assert output == outputs / "comments_raw.jsonl" + for name in ( + "comments_raw.jsonl", + "comments_raw.csv", + "comments_clean.jsonl", + "comments_clean.csv", + ): + assert (outputs / name).exists() + paths = scraper.get_paths() + assert paths["comments_raw_jsonl"] == str(outputs / "comments_raw.jsonl") + assert paths["comments_clean_csv"] == str(outputs / "comments_clean.csv") + assert paths["comments_stats"]["comments_out"] == 1 + + +def test_fetch_comments_subprocess_failure_keeps_outputs(tmp_path: Path) -> None: + scraper = _scraper(tmp_path) + (tmp_path / "crawl_comments_v2.py").write_text("# stub\n", encoding="utf-8") + source_csv = _source_csv(tmp_path) + + with patch( + "douyin_scraper.core.subprocess.run", + side_effect=subprocess.CalledProcessError(1, ["python", "crawl_comments_v2.py"]), + ): + output = scraper.fetch_comments( + video_jsonl=source_csv, + source_task_id="search-task", + max_comments_per_video=5, + ) + + outputs = tmp_path / "workspaces" / "comments-task" / "outputs" + assert output.exists() + assert (outputs / "comments_raw.csv").read_bytes()[:3] == b"\xef\xbb\xbf" + assert (outputs / "comments_clean.jsonl").exists() + assert (outputs / "comments_clean.csv").exists() + paths = scraper.get_paths() + assert paths["comments_stats"]["comments_out"] == 0 + assert paths["comments_stats"]["errors"] From 8bee152f028c48a301be8d4f2cbd4d2bddc17a98 Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 09:37:42 +0800 Subject: [PATCH 04/32] feat(scraper): add cleaned search title outputs --- api/routes.py | 10 + douyin_scraper/core.py | 336 +++++++++++++++++++++++ douyin_scraper/tests/test_title_clean.py | 101 +++++++ 3 files changed, 447 insertions(+) create mode 100644 douyin_scraper/tests/test_title_clean.py diff --git a/api/routes.py b/api/routes.py index 6398896e7..9e9fc81ec 100644 --- a/api/routes.py +++ b/api/routes.py @@ -218,6 +218,15 @@ def _search_output_result(paths: Dict[str, Any], output: Path) -> Dict[str, Any] +def _title_clean_result(paths: Dict[str, Any]) -> Dict[str, Any]: + return { + "title_clean_jsonl": paths.get("title_clean_jsonl", ""), + "title_clean_csv": paths.get("title_clean_csv", ""), + "title_clean_stats": paths.get("title_clean_stats", {}), + } + + + def _comments_output_result( paths: Dict[str, Any], output: Path, @@ -251,6 +260,7 @@ def _do_search() -> Dict[str, Any]: output = scraper.search(keywords=req.keywords, max_count=req.max_count) paths = scraper.get_paths() result = _search_output_result(paths, output) + result.update(_title_clean_result(paths)) result["status"] = scraper.get_status() return result diff --git a/douyin_scraper/core.py b/douyin_scraper/core.py index 6b1798a80..e34eea7f7 100644 --- a/douyin_scraper/core.py +++ b/douyin_scraper/core.py @@ -247,6 +247,10 @@ def search( output_path ) + title_clean_csv = self._prepare_title_clean_outputs( + result_csv_path, result_jsonl_path + ) + self._state.mark_step_completed( step, detail=f"output={output_path}" @@ -283,6 +287,21 @@ def _prepare_standard_search_outputs(self, output_path: Path) -> tuple[Path, Pat return result_jsonl_path, result_csv_path + def _prepare_title_clean_outputs( + self, + result_csv_path: Path, + result_jsonl_path: Path, + ) -> Path: + """Create and register search_title_clean outputs.""" + clean_jsonl, clean_csv, stats = self._do_clean_search_titles( + result_csv_path, result_jsonl_path + ) + self._paths["title_clean_jsonl"] = clean_jsonl + self._paths["title_clean_csv"] = clean_csv + self._paths["title_clean_stats"] = stats + return clean_csv + + def fetch_comments( self, video_jsonl: Optional[Path] = None, @@ -477,6 +496,7 @@ def get_paths(self) -> Dict[str, Any]: result: Dict[str, Any] = {} result.update(self._search_output_paths()) result.update(self._comments_output_paths()) + result.update(self._title_output_paths()) return result @@ -516,6 +536,13 @@ def _comments_output_paths(self) -> Dict[str, Any]: ) + def _title_output_paths(self) -> Dict[str, Any]: + return self._select_output_values( + path_keys=("title_clean_jsonl", "title_clean_csv"), + stats_keys=("title_clean_stats",), + ) + + def _require_step_ready(self, step: str) -> None: """检查步骤是否可执行，否则抛异常""" if not self._state.check_step_ready(step): @@ -1545,6 +1572,315 @@ def _dedupe_text_list(items: Sequence[str]) -> List[str]: return result + @staticmethod + def _is_noise_title_hashtag(tag: str) -> bool: + text = str(tag or "").strip().lower() + if not text: + return True + noise_exact = { + "学车", + "驾考", + "驾校", + "热门", + "上热门", + "抖音热门", + "dou小助手", + "dou+", + "热点", + "挑战", + "日常", + "同城", + "流量", + "汽车", + "干货分享", + "知识分享", + } + if text in noise_exact: + return True + return any(token in text for token in ("上热门", "抖音热门", "dou", "热点挑战")) + + + def _extract_title_hashtags(self, *texts: str) -> tuple[List[str], List[str]]: + hashtags: List[str] = [] + noise_removed: List[str] = [] + seen: Set[str] = set() + pattern = re.compile(r"#([^#\s，,。；;！!？?\r\n]+)") + for text in texts: + for match in pattern.findall(str(text or "")): + tag = match.strip(" #\t\r\n,，.。!！?？;；:：、") + if not tag: + continue + label = f"#{tag}" + if tag in seen: + noise_removed.append(label) + continue + seen.add(tag) + if self._is_noise_title_hashtag(tag): + noise_removed.append(label) + continue + hashtags.append(tag) + return hashtags, self._dedupe_text_list(noise_removed) + + + def _normalize_title_text(self, value: str, noise_removed: List[str]) -> str: + text = str(value or "") + text = re.sub(r"#[^#\s，,。；;！!？?\r\n]+", " ", text) + noise_terms = [ + "点击头像", + "主页还有", + "主页看合集", + "关注我", + "记得关注", + "点赞关注", + "点赞", + "收藏起来", + "建议收藏", + "收藏转发", + "转发收藏", + "评论区", + "蹲后续", + "看完再走", + "快来看看", + "上热门", + "热门推荐", + "DOU小助手", + "抖音热点", + "抖音热门", + ] + for term in noise_terms: + if term in text: + noise_removed.append(term) + text = text.replace(term, " ") + + core_terms = [ + "科目三", + "科三", + "靠边停车", + "直线行驶", + "30公分", + "压线", + "挂科", + "找点", + "看点", + "后视镜", + "方向盘", + "一把过", + "考试紧张", + ] + for term in core_terms: + separated = re.compile( + rf"({re.escape(term)})(?:[\s,，、/|]+{re.escape(term)})+" + ) + compact = re.compile(rf"({re.escape(term)})(?:{re.escape(term)})+") + if separated.search(text) or compact.search(text): + noise_removed.append(term) + text = separated.sub(term, text) + text = compact.sub(term, text) + + text = re.sub(r"[ \t\r\n　]+", " ", text) + text = re.sub(r"[，,。；;！!？?、|]{2,}", "，", text) + return text.strip(" \t\r\n　,，。；;！!？?、|") + + + @staticmethod + def _infer_title_topic(haystack: str) -> str: + text = str(haystack or "") + if "靠边停车" in text or ("靠边" in text and "停车" in text): + return "靠边停车" + if "直线行驶" in text or "直线跑偏" in text: + return "直线行驶" + if "科目三" in text or "科三" in text: + if any(token in text for token in ("流程", "全流程", "步骤")): + return "科目三全流程" + return "科目三" + return "" + + + @staticmethod + def _infer_title_pain_point(haystack: str) -> str: + text = str(haystack or "") + has_30cm = bool(re.search(r"(30\s*公分|三十公分|30cm|30厘米)", text, re.I)) + if has_30cm and any(token in text for token in ("看不准", "找不准", "不准", "不会看", "看不清", "怎么看")): + return "30公分看不准" + if "压线" in text: + return "压线" + if "跑偏" in text: + return "直线跑偏" + if "紧张" in text: + return "考试紧张" + if "挂科" in text or "挂的" in text: + return "挂科" + if has_30cm: + return "30公分" + return "" + + + @staticmethod + def _infer_title_teaching_angle(haystack: str, pain_point: str) -> str: + text = str(haystack or "") + if any(token in text for token in ("找点", "点位", "看点")) or pain_point == "30公分看不准": + return "找点方法" + if "压线" in text or pain_point == "压线": + return "防压线技巧" + if any(token in text for token in ("流程", "步骤")): + return "考试流程讲解" + if "后视镜" in text: + return "后视镜观察方法" + if "方向盘" in text: + return "方向盘控制" + if any(token in text for token in ("保姆级", "教学", "技巧", "一把过")): + return "技巧讲解" + return "" + + + def _clean_search_title_record(self, rec: Dict[str, Any]) -> Dict[str, Any]: + raw_title = str(rec.get("title", rec.get("raw_title", "")) or "") + raw_desc = str(rec.get("desc", rec.get("raw_desc", "")) or "") + hashtags, hashtag_noise = self._extract_title_hashtags(raw_title, raw_desc) + noise_removed = list(hashtag_noise) + clean_title = self._normalize_title_text(raw_title, noise_removed) + clean_desc = self._normalize_title_text(raw_desc, noise_removed) + if not clean_title: + clean_title = clean_desc or " ".join(hashtags[:2]) or str(rec.get("source_keyword", "")).strip() + + haystack = " ".join([raw_title, raw_desc, clean_title, clean_desc, " ".join(hashtags)]) + topic = self._infer_title_topic(haystack) + pain_point = self._infer_title_pain_point(haystack) + teaching_angle = self._infer_title_teaching_angle(haystack, pain_point) + + def _safe_int(value: Any) -> int: + try: + return int(value) + except (TypeError, ValueError): + return 0 + + liked_count = _safe_int(rec.get("liked_count", 0)) + collected_count = _safe_int(rec.get("collected_count", 0)) + comment_count = _safe_int(rec.get("comment_count", 0)) + share_count = _safe_int(rec.get("share_count", 0)) + total_engagement = _safe_int( + rec.get("total_engagement", liked_count + collected_count + comment_count + share_count) + ) + + return { + "source_keyword": str(rec.get("source_keyword", "")), + "platform": str(rec.get("platform", "douyin") or "douyin"), + "video_id": str(rec.get("video_id", "")), + "aweme_id": str(rec.get("aweme_id", "")), + "raw_title": raw_title, + "clean_title": clean_title, + "raw_desc": raw_desc, + "clean_desc": clean_desc, + "hashtags": hashtags, + "topic": topic, + "pain_point": pain_point, + "teaching_angle": teaching_angle, + "title_noise_removed": self._dedupe_text_list(noise_removed), + "aweme_url": str(rec.get("aweme_url", "")), + "liked_count": liked_count, + "collected_count": collected_count, + "comment_count": comment_count, + "share_count": share_count, + "total_engagement": total_engagement, + } + + + def _load_search_rows_for_title_clean( + self, search_csv_path: Path, search_jsonl_path: Optional[Path] + ) -> tuple[List[Dict[str, Any]], List[str]]: + errors: List[str] = [] + if search_csv_path.exists(): + try: + with open(str(search_csv_path), "r", encoding="utf-8-sig", newline="") as f: + return list(csv.DictReader(f)), errors + except Exception as e: + errors.append(f"read_csv_failed: {str(e)[:200]}") + + if search_jsonl_path and search_jsonl_path.exists(): + try: + return self._load_jsonl(search_jsonl_path), errors + except Exception as e: + errors.append(f"read_jsonl_failed: {str(e)[:200]}") + + if not search_csv_path.exists(): + errors.append(f"search_csv_missing: {search_csv_path}") + if search_jsonl_path and not search_jsonl_path.exists(): + errors.append(f"search_jsonl_missing: {search_jsonl_path}") + return [], errors + + + def _do_clean_search_titles( + self, search_csv_path: Path, search_jsonl_path: Optional[Path] = None + ) -> tuple[Path, Path, Dict[str, Any]]: + clean_jsonl_path = search_csv_path.with_name("search_title_clean.jsonl") + clean_csv_path = search_csv_path.with_name("search_title_clean.csv") + fieldnames = [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "raw_title", + "clean_title", + "raw_desc", + "clean_desc", + "hashtags", + "topic", + "pain_point", + "teaching_angle", + "title_noise_removed", + "aweme_url", + "liked_count", + "collected_count", + "comment_count", + "share_count", + "total_engagement", + ] + stats: Dict[str, Any] = { + "rows_in": 0, + "rows_out": 0, + "clean_csv_generated": False, + "errors": [], + } + rows: List[Dict[str, Any]] = [] + try: + records, load_errors = self._load_search_rows_for_title_clean(search_csv_path, search_jsonl_path) + stats["errors"].extend(load_errors) + stats["rows_in"] = len(records) + for idx, rec in enumerate(records, start=1): + try: + rows.append(self._clean_search_title_record(rec)) + except Exception as e: + stats["errors"].append(f"row_{idx}_failed: {str(e)[:200]}") + + stats["rows_out"] = len(rows) + clean_jsonl_path.parent.mkdir(parents=True, exist_ok=True) + with open(str(clean_jsonl_path), "w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + with open(str(clean_csv_path), "w", encoding="utf-8-sig", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for row in rows: + csv_row = dict(row) + csv_row["hashtags"] = "|".join(row["hashtags"]) + csv_row["title_noise_removed"] = "|".join(row["title_noise_removed"]) + writer.writerow(csv_row) + stats["clean_csv_generated"] = True + except Exception as e: + stats["errors"].append(f"title_clean_failed: {str(e)[:500]}") + try: + clean_jsonl_path.parent.mkdir(parents=True, exist_ok=True) + clean_jsonl_path.write_text("", encoding="utf-8") + with open(str(clean_csv_path), "w", encoding="utf-8-sig", newline="") as f: + csv.DictWriter(f, fieldnames=fieldnames).writeheader() + stats["clean_csv_generated"] = True + except Exception as write_error: + stats["errors"].append(f"title_clean_write_failed: {str(write_error)[:500]}") + + return clean_jsonl_path, clean_csv_path, stats + + def _convert_jsonl_to_standard_csv( self, jsonl_path: str, diff --git a/douyin_scraper/tests/test_title_clean.py b/douyin_scraper/tests/test_title_clean.py new file mode 100644 index 000000000..a23b2c722 --- /dev/null +++ b/douyin_scraper/tests/test_title_clean.py @@ -0,0 +1,101 @@ +import csv +import json +from pathlib import Path +from unittest.mock import patch + +from douyin_scraper import DouyinScraper + + +def _scraper(tmp_path: Path) -> DouyinScraper: + return DouyinScraper( + { + "project_dir": str(tmp_path), + "state_dir_name": "workspaces/search-task/state", + } + ) + + +def test_title_clean_extracts_hashtags_topic_pain_and_angle( + tmp_path: Path, +) -> None: + scraper = _scraper(tmp_path) + search_csv = tmp_path / "search_result.csv" + fieldnames = [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "title", + "desc", + "liked_count", + "collected_count", + "comment_count", + "share_count", + "total_engagement", + "aweme_url", + ] + with open(search_csv, "w", encoding="utf-8-sig", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerow( + { + "source_keyword": "靠边停车", + "platform": "douyin", + "video_id": "title-1", + "aweme_id": "title-1", + "title": "科三靠边停车30公分看不准 #科目三技巧 #靠边停车技巧 #学车", + "desc": "后视镜找点方法，建议收藏", + "liked_count": "10", + "collected_count": "3", + "comment_count": "2", + "share_count": "1", + "total_engagement": "16", + "aweme_url": "https://www.douyin.com/video/title-1", + } + ) + + clean_jsonl, clean_csv, stats = scraper._do_clean_search_titles(search_csv) + + assert stats["rows_in"] == 1 + assert stats["rows_out"] == 1 + assert stats["clean_csv_generated"] is True + assert clean_csv.read_bytes()[:3] == b"\xef\xbb\xbf" + row = json.loads(clean_jsonl.read_text(encoding="utf-8").strip()) + assert row["raw_title"] + assert "#" not in row["clean_title"] + assert "科目三技巧" in row["hashtags"] + assert "靠边停车技巧" in row["hashtags"] + assert row["topic"] == "靠边停车" + assert row["pain_point"] == "30公分看不准" + assert row["teaching_angle"] == "找点方法" + + +def test_search_registers_title_clean_outputs_separately(tmp_path: Path) -> None: + scraper = _scraper(tmp_path) + outputs = tmp_path / "workspaces" / "search-task" / "outputs" + outputs.mkdir(parents=True) + result_jsonl = outputs / "search_result.jsonl" + result_jsonl.write_text( + json.dumps( + { + "aweme_id": "title-search-1", + "title": "科三靠边停车 #靠边停车技巧", + "desc": "30公分看不准，找点方法", + "aweme_url": "https://www.douyin.com/video/title-search-1", + }, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + + with ( + patch.object(scraper, "_do_search", return_value=result_jsonl), + patch.object(scraper, "_prepare_script_source_outputs"), + ): + scraper.search(keywords=["靠边停车"], max_count=1) + + paths = scraper.get_paths() + assert paths["title_clean_jsonl"] == str(outputs / "search_title_clean.jsonl") + assert paths["title_clean_csv"] == str(outputs / "search_title_clean.csv") + assert paths["title_clean_stats"]["rows_out"] == 1 From 2d7afaf8b61ea43977858560ef1745ca5aad5f09 Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 09:38:33 +0800 Subject: [PATCH 05/32] feat(scraper): add script source raw and clean outputs --- api/routes.py | 84 +- api/tasks.py | 12 + douyin_scraper/core.py | 823 ++++++++++++++++++++ douyin_scraper/tests/test_script_outputs.py | 333 ++++++++ 4 files changed, 1251 insertions(+), 1 deletion(-) create mode 100644 douyin_scraper/tests/test_script_outputs.py diff --git a/api/routes.py b/api/routes.py index 9e9fc81ec..ea23b5bb7 100644 --- a/api/routes.py +++ b/api/routes.py @@ -102,6 +102,7 @@ class CommentsRequest(BaseModel): class ScriptsRequest(BaseModel): """文案提取请求""" + task_id: Optional[str] = Field(None, description="搜索任务 ID（读取其 script_sources 输出）") video_jsonl: Optional[str] = Field(None, description="视频 JSONL 路径") model: Literal["tiny", "base", "small", "medium", "large"] = Field( "small", description="Whisper 模型大小: tiny/base/small/medium/large" @@ -227,6 +228,15 @@ def _title_clean_result(paths: Dict[str, Any]) -> Dict[str, Any]: +def _script_source_result(paths: Dict[str, Any]) -> Dict[str, Any]: + return { + "script_sources_jsonl": paths.get("script_sources_jsonl", ""), + "script_sources_csv": paths.get("script_sources_csv", ""), + "script_sources_stats": paths.get("script_sources_stats", {}), + } + + + def _comments_output_result( paths: Dict[str, Any], output: Path, @@ -244,6 +254,24 @@ def _comments_output_result( +def _script_output_result( + paths: Dict[str, Any], + output: Path, +) -> Dict[str, Any]: + script_raw_jsonl = paths.get("script_raw_jsonl", str(output)) + return { + "scripts_jsonl": script_raw_jsonl, + "script_raw_jsonl": script_raw_jsonl, + "script_raw_csv": paths.get("script_raw_csv", ""), + "script_raw_stats": paths.get("script_raw_stats", {}), + "script_clean_jsonl": paths.get("script_clean_jsonl", ""), + "script_clean_csv": paths.get("script_clean_csv", ""), + "script_clean_stats": paths.get("script_clean_stats", {}), + } + + +# Search endpoint + @router.post("/search", summary="触发搜索采集") async def search(req: SearchRequest) -> Dict[str, Any]: """ @@ -261,6 +289,7 @@ def _do_search() -> Dict[str, Any]: paths = scraper.get_paths() result = _search_output_result(paths, output) result.update(_title_clean_result(paths)) + result.update(_script_source_result(paths)) result["status"] = scraper.get_status() return result @@ -330,6 +359,55 @@ async def extract_scripts(req: ScriptsRequest) -> Dict[str, Any]: def _do_scripts() -> Dict[str, Any]: scraper = _make_scraper(req.project_dir, task.workspace) + script_sources_jsonl: Optional[Path] = None + script_sources_csv: Optional[Path] = None + title_clean_csv: Optional[Path] = None + if req.task_id: + source_task = tm.get_task(req.task_id) + if not source_task: + raise NonRetryableError( + f"搜索任务不存在: {req.task_id}", + step="extract_scripts", + ) + source_outputs = Path(source_task.workspace) / "outputs" + jsonl_path = source_outputs / "script_sources.jsonl" + csv_path = source_outputs / "script_sources.csv" + title_clean_path = source_outputs / "search_title_clean.csv" + if jsonl_path.exists(): + script_sources_jsonl = jsonl_path + if csv_path.exists(): + script_sources_csv = csv_path + if title_clean_path.exists(): + title_clean_csv = title_clean_path + if not script_sources_jsonl and not script_sources_csv: + raise NonRetryableError( + f"搜索任务无可用 script_sources 输出: {req.task_id}", + step="extract_scripts", + ) + else: + current_outputs = Path(task.workspace) / "outputs" + jsonl_path = current_outputs / "script_sources.jsonl" + csv_path = current_outputs / "script_sources.csv" + title_clean_path = current_outputs / "search_title_clean.csv" + if jsonl_path.exists(): + script_sources_jsonl = jsonl_path + if csv_path.exists(): + script_sources_csv = csv_path + if title_clean_path.exists(): + title_clean_csv = title_clean_path + + if script_sources_jsonl or script_sources_csv: + output = scraper.extract_script_raw( + script_sources_jsonl=script_sources_jsonl, + script_sources_csv=script_sources_csv, + model=req.model, + title_clean_csv=title_clean_csv, + ) + paths = scraper.get_paths() + result = _script_output_result(paths, output) + result["status"] = scraper.get_status() + return result + video_path = Path(req.video_jsonl) if req.video_jsonl else None if video_path: video_path = validate_path_in_workspace( @@ -338,7 +416,11 @@ def _do_scripts() -> Dict[str, Any]: output = scraper.extract_scripts( video_jsonl=video_path, model=req.model ) - return {"scripts_jsonl": str(output), "status": scraper.get_status()} + paths = scraper.get_paths() + result = _script_output_result(paths, output) + result["scripts_jsonl"] = str(output) + result["status"] = scraper.get_status() + return result tm.submit(task, _do_scripts) return {"task_id": task.task_id, "status": "submitted", "type": "scripts"} diff --git a/api/tasks.py b/api/tasks.py index e629f1550..e61e80842 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -501,6 +501,18 @@ def _comments_result_names() -> tuple[str, ...]: ) + @staticmethod + def _scripts_result_names() -> tuple[str, ...]: + return ( + "script_clean.csv", + "script_clean.jsonl", + "script_raw.csv", + "script_raw.jsonl", + "script_sources.csv", + "script_sources.jsonl", + ) + + @classmethod def _preferred_result_names(cls, task_type: str) -> tuple[str, ...]: selector = getattr(cls, f"_{task_type}_result_names", None) diff --git a/douyin_scraper/core.py b/douyin_scraper/core.py index e34eea7f7..4bbf33471 100644 --- a/douyin_scraper/core.py +++ b/douyin_scraper/core.py @@ -251,6 +251,9 @@ def search( result_csv_path, result_jsonl_path ) + self._prepare_script_source_outputs( + result_csv_path, result_jsonl_path, title_clean_csv + ) self._state.mark_step_completed( step, detail=f"output={output_path}" @@ -302,6 +305,21 @@ def _prepare_title_clean_outputs( return clean_csv + def _prepare_script_source_outputs( + self, + result_csv_path: Path, + result_jsonl_path: Path, + title_clean_csv: Optional[Path], + ) -> None: + """Create and register script_sources outputs.""" + sources_jsonl, sources_csv, stats = self._do_build_script_sources( + result_csv_path, result_jsonl_path, title_clean_csv + ) + self._paths["script_sources_jsonl"] = sources_jsonl + self._paths["script_sources_csv"] = sources_csv + self._paths["script_sources_stats"] = stats + + def fetch_comments( self, video_jsonl: Optional[Path] = None, @@ -402,6 +420,96 @@ def extract_scripts( self._state.mark_step_failed(step, str(e)[:200], exit_code) raise + def extract_script_raw( + self, + script_sources_jsonl: Optional[Path] = None, + script_sources_csv: Optional[Path] = None, + model: str = "small", + max_items: Optional[int] = None, + title_clean_csv: Optional[Path] = None, + ) -> Path: + """Build script_raw outputs from standardized script_sources.""" + step = self.STEP_EXTRACT + self._require_step_ready(step) + + output_dir = self._current_task_workspace_dir() / "outputs" + if script_sources_jsonl is None: + candidate = output_dir / "script_sources.jsonl" + if candidate.exists(): + script_sources_jsonl = candidate + if script_sources_csv is None: + candidate = output_dir / "script_sources.csv" + if candidate.exists(): + script_sources_csv = candidate + if title_clean_csv is None: + for source_path in (script_sources_jsonl, script_sources_csv): + if source_path is None: + continue + candidate = source_path.parent / "search_title_clean.csv" + if candidate.exists(): + title_clean_csv = candidate + break + if title_clean_csv is None: + candidate = output_dir / "search_title_clean.csv" + if candidate.exists(): + title_clean_csv = candidate + + if not script_sources_jsonl and not script_sources_csv: + raise NonRetryableError( + "script_sources.jsonl/csv 不存在，无法生成 script_raw", + step=step, + ) + + setup_env_vars() + self._state.mark_step_started(step) + try: + raw_jsonl, raw_csv, stats = self._do_build_script_raw( + script_sources_jsonl=script_sources_jsonl, + script_sources_csv=script_sources_csv, + model_name=model, + max_items=max_items, + ) + clean_jsonl, clean_csv, clean_stats = self._do_build_script_clean( + script_sources_jsonl=script_sources_jsonl, + script_sources_csv=script_sources_csv, + script_raw_jsonl=raw_jsonl, + script_raw_csv=raw_csv, + title_clean_csv=title_clean_csv, + ) + self._state.mark_step_completed(step, detail=f"output={raw_jsonl}") + self._register_script_outputs( + raw_jsonl, + raw_csv, + stats, + clean_jsonl, + clean_csv, + clean_stats, + ) + return raw_jsonl + except Exception as e: + exit_code = classify_error(e) + self._state.mark_step_failed(step, str(e)[:200], exit_code) + raise + + + def _register_script_outputs( + self, + raw_jsonl: Path, + raw_csv: Path, + raw_stats: Dict[str, Any], + clean_jsonl: Path, + clean_csv: Path, + clean_stats: Dict[str, Any], + ) -> None: + """Register script_raw/script_clean outputs for get_paths().""" + self._paths["script_raw_jsonl"] = raw_jsonl + self._paths["script_raw_csv"] = raw_csv + self._paths["script_raw_stats"] = raw_stats + self._paths["script_clean_jsonl"] = clean_jsonl + self._paths["script_clean_csv"] = clean_csv + self._paths["script_clean_stats"] = clean_stats + + def merge( self, video_jsonl: Optional[Path] = None, @@ -497,6 +605,7 @@ def get_paths(self) -> Dict[str, Any]: result.update(self._search_output_paths()) result.update(self._comments_output_paths()) result.update(self._title_output_paths()) + result.update(self._script_output_paths()) return result @@ -543,6 +652,25 @@ def _title_output_paths(self) -> Dict[str, Any]: ) + def _script_output_paths(self) -> Dict[str, Any]: + return self._select_output_values( + path_keys=( + "script_sources_jsonl", + "script_sources_csv", + "script_raw_jsonl", + "script_raw_csv", + "script_clean_jsonl", + "script_clean_csv", + "scripts_jsonl", + ), + stats_keys=( + "script_sources_stats", + "script_raw_stats", + "script_clean_stats", + ), + ) + + def _require_step_ready(self, step: str) -> None: """检查步骤是否可执行，否则抛异常""" if not self._state.check_step_ready(step): @@ -1881,6 +2009,701 @@ def _do_clean_search_titles( return clean_jsonl_path, clean_csv_path, stats + @staticmethod + def _join_source_text(*parts: Any) -> str: + texts: List[str] = [] + seen: Set[str] = set() + for part in parts: + text = re.sub(r"\s+", " ", str(part or "")).strip() + if not text or text in seen: + continue + seen.add(text) + texts.append(text) + return " ".join(texts) + + + @staticmethod + def _script_source_key(rec: Dict[str, Any]) -> str: + for key in ("aweme_id", "video_id", "aweme_url"): + value = str(rec.get(key, "") or "").strip() + if value and value != "None": + return value + return "" + + + def _load_title_clean_map(self, title_clean_csv_path: Optional[Path]) -> tuple[Dict[str, Dict[str, Any]], List[str]]: + errors: List[str] = [] + clean_map: Dict[str, Dict[str, Any]] = {} + if not title_clean_csv_path or not title_clean_csv_path.exists(): + return clean_map, errors + try: + with open(str(title_clean_csv_path), "r", encoding="utf-8-sig", newline="") as f: + for row in csv.DictReader(f): + keys = self._dedupe_text_list([ + str(row.get("aweme_id", "") or "").strip(), + str(row.get("video_id", "") or "").strip(), + str(row.get("aweme_url", "") or "").strip(), + ]) + for key in keys: + if key and key != "None": + clean_map[key] = row + except Exception as e: + errors.append(f"read_title_clean_failed: {str(e)[:200]}") + return clean_map, errors + + + def _build_script_source_record( + self, + rec: Dict[str, Any], + clean_rec: Optional[Dict[str, Any]], + created_at: str, + ) -> Dict[str, Any]: + raw_title = str(rec.get("title", rec.get("raw_title", "")) or "") + raw_desc = str(rec.get("desc", rec.get("raw_desc", "")) or "") + clean_title = str((clean_rec or {}).get("clean_title", rec.get("clean_title", "")) or "") + clean_desc = str((clean_rec or {}).get("clean_desc", rec.get("clean_desc", "")) or "") + video_download_url = str(rec.get("video_download_url", "") or "").strip() + + title_desc_text = self._join_source_text(raw_title, raw_desc) + clean_title_text = self._join_source_text(clean_title, clean_desc) + title_desc_available = bool(title_desc_text) + clean_title_available = bool(clean_title_text) + video_download_available = bool(video_download_url) + asr_planned = video_download_available + + notes: List[str] = [] + if clean_title_available: + quality = "medium" + status = "available" + notes.append("clean_title_desc_available") + elif title_desc_available: + quality = "weak" + status = "available" + notes.append("raw_title_desc_available") + elif video_download_available: + quality = "low" + status = "planned" + notes.append("video_download_available_asr_planned") + else: + quality = "missing" + status = "missing" + notes.append("no_text_or_video_source") + + if video_download_available and quality != "low": + notes.append("video_download_available") + if not video_download_available: + notes.append("video_download_missing") + + return { + "source_keyword": str(rec.get("source_keyword", "")), + "platform": str(rec.get("platform", "douyin") or "douyin"), + "video_id": str(rec.get("video_id", "")), + "aweme_id": str(rec.get("aweme_id", "")), + "aweme_url": str(rec.get("aweme_url", "")), + "raw_title": raw_title, + "clean_title": clean_title, + "raw_desc": raw_desc, + "clean_desc": clean_desc, + "video_download_url": video_download_url, + "source_title_desc_available": title_desc_available, + "source_title_desc_text": title_desc_text, + "source_clean_title_available": clean_title_available, + "source_clean_title_text": clean_title_text, + "source_video_download_available": video_download_available, + "source_video_download_url": video_download_url, + "source_asr_planned": asr_planned, + "source_ocr_planned": False, + "source_subtitle_planned": False, + "script_source_status": status, + "script_source_quality": quality, + "script_source_notes": "|".join(self._dedupe_text_list(notes)), + "created_at": created_at, + } + + + def _do_build_script_sources( + self, + search_csv_path: Path, + search_jsonl_path: Optional[Path] = None, + title_clean_csv_path: Optional[Path] = None, + ) -> tuple[Path, Path, Dict[str, Any]]: + sources_jsonl_path = search_csv_path.with_name("script_sources.jsonl") + sources_csv_path = search_csv_path.with_name("script_sources.csv") + fieldnames = [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "aweme_url", + "raw_title", + "clean_title", + "raw_desc", + "clean_desc", + "video_download_url", + "source_title_desc_available", + "source_title_desc_text", + "source_clean_title_available", + "source_clean_title_text", + "source_video_download_available", + "source_video_download_url", + "source_asr_planned", + "source_ocr_planned", + "source_subtitle_planned", + "script_source_status", + "script_source_quality", + "script_source_notes", + "created_at", + ] + stats: Dict[str, Any] = { + "rows_in": 0, + "rows_out": 0, + "title_desc_available": 0, + "clean_title_available": 0, + "video_download_available": 0, + "asr_planned": 0, + "script_sources_csv_generated": False, + "errors": [], + } + rows: List[Dict[str, Any]] = [] + try: + records, load_errors = self._load_search_rows_for_title_clean(search_csv_path, search_jsonl_path) + stats["errors"].extend(load_errors) + clean_map, clean_errors = self._load_title_clean_map(title_clean_csv_path) + stats["errors"].extend(clean_errors) + stats["rows_in"] = len(records) + created_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + for idx, rec in enumerate(records, start=1): + try: + clean_rec = clean_map.get(self._script_source_key(rec)) + row = self._build_script_source_record(rec, clean_rec, created_at) + if row["source_title_desc_available"]: + stats["title_desc_available"] += 1 + if row["source_clean_title_available"]: + stats["clean_title_available"] += 1 + if row["source_video_download_available"]: + stats["video_download_available"] += 1 + if row["source_asr_planned"]: + stats["asr_planned"] += 1 + rows.append(row) + except Exception as e: + stats["errors"].append(f"row_{idx}_failed: {str(e)[:200]}") + + stats["rows_out"] = len(rows) + sources_jsonl_path.parent.mkdir(parents=True, exist_ok=True) + with open(str(sources_jsonl_path), "w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + with open(str(sources_csv_path), "w", encoding="utf-8-sig", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + stats["script_sources_csv_generated"] = True + except Exception as e: + stats["errors"].append(f"script_sources_failed: {str(e)[:500]}") + try: + sources_jsonl_path.parent.mkdir(parents=True, exist_ok=True) + sources_jsonl_path.write_text("", encoding="utf-8") + with open(str(sources_csv_path), "w", encoding="utf-8-sig", newline="") as f: + csv.DictWriter(f, fieldnames=fieldnames).writeheader() + stats["script_sources_csv_generated"] = True + except Exception as write_error: + stats["errors"].append(f"script_sources_write_failed: {str(write_error)[:500]}") + + return sources_jsonl_path, sources_csv_path, stats + + + @staticmethod + def _script_raw_fieldnames() -> List[str]: + return [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "aweme_url", + "video_download_url", + "local_video_path", + "download_status", + "download_error", + "asr_status", + "asr_raw_text", + "asr_error", + "script_raw_quality", + "created_at", + ] + + + @staticmethod + def _truthy_source_flag(value: Any) -> bool: + if isinstance(value, bool): + return value + text = str(value or "").strip().lower() + return text in ("1", "true", "yes", "y", "on") + + + @staticmethod + def _script_raw_video_name(rec: Dict[str, Any], idx: int) -> str: + raw = str(rec.get("aweme_id") or rec.get("video_id") or f"row_{idx}") + safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", raw).strip("._") + return (safe[:80] or f"row_{idx}") + ".mp4" + + + def _load_script_source_rows( + self, + script_sources_jsonl: Optional[Path], + script_sources_csv: Optional[Path], + ) -> tuple[List[Dict[str, Any]], List[str]]: + errors: List[str] = [] + if script_sources_jsonl and script_sources_jsonl.exists(): + try: + return self._load_jsonl(script_sources_jsonl), errors + except Exception as e: + errors.append(f"read_script_sources_jsonl_failed: {str(e)[:200]}") + + if script_sources_csv and script_sources_csv.exists(): + try: + with open(str(script_sources_csv), "r", encoding="utf-8-sig", newline="") as f: + return list(csv.DictReader(f)), errors + except Exception as e: + errors.append(f"read_script_sources_csv_failed: {str(e)[:200]}") + + errors.append("script_sources_missing") + return [], errors + + + @staticmethod + def _load_script_raw_whisper_model(model_name: str) -> tuple[Optional[Any], str, str]: + try: + from faster_whisper import WhisperModel + except ImportError: + return None, "dependency_missing", "faster-whisper 未安装" + + try: + return WhisperModel(model_name, device="cpu", compute_type="int8"), "", "" + except MemoryError: + return None, "failed", "内存不足，无法加载 Whisper 模型" + except Exception as e: + return None, "failed", f"Whisper 模型加载失败: {str(e)[:300]}" + + + def _do_build_script_raw( + self, + script_sources_jsonl: Optional[Path] = None, + script_sources_csv: Optional[Path] = None, + model_name: str = "small", + max_items: Optional[int] = None, + ) -> tuple[Path, Path, Dict[str, Any]]: + workspace_dir = self._current_task_workspace_dir() + output_dir = workspace_dir / "outputs" + output_dir.mkdir(parents=True, exist_ok=True) + raw_jsonl_path = output_dir / "script_raw.jsonl" + raw_csv_path = output_dir / "script_raw.csv" + tmp_video_dir = workspace_dir / "tmp" / "videos" + fieldnames = self._script_raw_fieldnames() + stats: Dict[str, Any] = { + "rows_in": 0, + "rows_targeted": 0, + "download_success": 0, + "download_failed": 0, + "asr_success": 0, + "asr_failed": 0, + "asr_dependency_missing": 0, + "asr_empty_text": 0, + "rows_out": 0, + "script_raw_csv_generated": False, + "errors": [], + } + rows: List[Dict[str, Any]] = [] + + try: + records, load_errors = self._load_script_source_rows(script_sources_jsonl, script_sources_csv) + stats["errors"].extend(load_errors) + stats["rows_in"] = len(records) + limit = max_items + if limit is None: + try: + limit = int(getattr(self._config, "max_script_raw_items", 5) or 5) + except (TypeError, ValueError): + limit = 5 + limit = max(0, limit) + + eligible_indexes: List[int] = [] + for idx, rec in enumerate(records, start=1): + planned = self._truthy_source_flag(rec.get("source_asr_planned")) + video_available = self._truthy_source_flag(rec.get("source_video_download_available")) + video_url = str( + rec.get("video_download_url") + or rec.get("source_video_download_url") + or "" + ).strip() + if planned and video_available and video_url: + eligible_indexes.append(idx) + + selected_indexes = set(eligible_indexes[:limit]) + stats["rows_targeted"] = len(selected_indexes) + if len(eligible_indexes) > limit: + stats["errors"].append( + f"max_script_raw_items_limit: eligible={len(eligible_indexes)} processed={limit}" + ) + + disk_error = "" + if selected_indexes: + try: + tmp_video_dir.mkdir(parents=True, exist_ok=True) + check_disk_space_enforced(tmp_video_dir, min_gb=0.5) + except Exception as e: + disk_error = str(e)[:300] + stats["errors"].append(f"script_raw_disk_check_failed: {disk_error}") + + whisper_model: Optional[Any] = None + model_error_status = "" + model_error = "" + if selected_indexes and not disk_error: + whisper_model, model_error_status, model_error = self._load_script_raw_whisper_model(model_name) + + created_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + for idx, rec in enumerate(records, start=1): + video_url = str( + rec.get("video_download_url") + or rec.get("source_video_download_url") + or "" + ).strip() + row: Dict[str, Any] = { + "source_keyword": str(rec.get("source_keyword", "")), + "platform": str(rec.get("platform", "douyin") or "douyin"), + "video_id": str(rec.get("video_id", "")), + "aweme_id": str(rec.get("aweme_id", "")), + "aweme_url": str(rec.get("aweme_url", "")), + "video_download_url": video_url, + "local_video_path": "", + "download_status": "skipped", + "download_error": "", + "asr_status": "skipped", + "asr_raw_text": "", + "asr_error": "", + "script_raw_quality": "missing", + "created_at": created_at, + } + + if idx not in selected_indexes: + if idx in eligible_indexes: + row["download_error"] = "max_script_raw_items_limit" + rows.append(row) + continue + + video_file = tmp_video_dir / self._script_raw_video_name(rec, idx) + row["local_video_path"] = str(video_file) + + if disk_error: + row["download_status"] = "failed" + row["download_error"] = disk_error + stats["download_failed"] += 1 + rows.append(row) + continue + + if model_error_status: + row["asr_status"] = model_error_status + row["asr_error"] = model_error + if model_error_status == "dependency_missing": + stats["asr_dependency_missing"] += 1 + else: + stats["asr_failed"] += 1 + rows.append(row) + continue + + try: + ok = self._download_video(video_url, str(video_file)) + except Exception as e: + ok = False + row["download_error"] = str(e)[:500] + + if not ok: + row["download_status"] = "failed" + if not row["download_error"]: + row["download_error"] = "download_failed" + stats["download_failed"] += 1 + rows.append(row) + continue + + row["download_status"] = "success" + stats["download_success"] += 1 + + try: + script_text = self._transcribe_video(str(video_file), whisper_model) + if script_text: + row["asr_status"] = "success" + row["asr_raw_text"] = script_text + row["script_raw_quality"] = "high" + stats["asr_success"] += 1 + else: + row["asr_status"] = "empty_text" + row["script_raw_quality"] = "low" + stats["asr_empty_text"] += 1 + except Exception as e: + row["asr_status"] = "failed" + row["asr_error"] = str(e)[:500] + row["script_raw_quality"] = "low" + stats["asr_failed"] += 1 + finally: + if not self._config.keep_videos and video_file.exists(): + try: + video_file.unlink() + except OSError as e: + stats["errors"].append(f"cleanup_failed: {video_file}: {str(e)[:120]}") + + rows.append(row) + + stats["rows_out"] = len(rows) + with open(str(raw_jsonl_path), "w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + with open(str(raw_csv_path), "w", encoding="utf-8-sig", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + stats["script_raw_csv_generated"] = True + except Exception as e: + stats["errors"].append(f"script_raw_failed: {str(e)[:500]}") + try: + raw_jsonl_path.write_text("", encoding="utf-8") + with open(str(raw_csv_path), "w", encoding="utf-8-sig", newline="") as f: + csv.DictWriter(f, fieldnames=fieldnames).writeheader() + stats["script_raw_csv_generated"] = True + except Exception as write_error: + stats["errors"].append(f"script_raw_write_failed: {str(write_error)[:500]}") + + return raw_jsonl_path, raw_csv_path, stats + + + @staticmethod + def _script_clean_fieldnames() -> List[str]: + return [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "aweme_url", + "script_clean_text", + "script_clean_source", + "script_clean_status", + "script_clean_quality", + "script_clean_notes", + "asr_status", + "asr_raw_text", + "script_raw_quality", + "source_clean_title_available", + "source_clean_title_text", + "source_title_desc_available", + "source_title_desc_text", + "created_at", + ] + + + def _load_script_raw_rows( + self, + script_raw_jsonl: Optional[Path], + script_raw_csv: Optional[Path], + ) -> tuple[List[Dict[str, Any]], List[str]]: + errors: List[str] = [] + if script_raw_jsonl and script_raw_jsonl.exists(): + try: + return self._load_jsonl(script_raw_jsonl), errors + except Exception as e: + errors.append(f"read_script_raw_jsonl_failed: {str(e)[:200]}") + + if script_raw_csv and script_raw_csv.exists(): + try: + with open(str(script_raw_csv), "r", encoding="utf-8-sig", newline="") as f: + return list(csv.DictReader(f)), errors + except Exception as e: + errors.append(f"read_script_raw_csv_failed: {str(e)[:200]}") + + errors.append("script_raw_missing") + return [], errors + + + def _build_script_clean_record( + self, + source_rec: Dict[str, Any], + raw_rec: Optional[Dict[str, Any]], + title_clean_rec: Optional[Dict[str, Any]], + created_at: str, + ) -> Dict[str, Any]: + raw_rec = raw_rec or {} + title_clean_rec = title_clean_rec or {} + + asr_text = str(raw_rec.get("asr_raw_text", "") or "").strip() + source_clean_text = str(source_rec.get("source_clean_title_text", "") or "").strip() + if not source_clean_text: + source_clean_text = self._join_source_text( + title_clean_rec.get("clean_title", ""), + title_clean_rec.get("clean_desc", ""), + ) + if not source_clean_text: + source_clean_text = self._join_source_text( + source_rec.get("clean_title", ""), + source_rec.get("clean_desc", ""), + ) + + source_title_desc_text = str(source_rec.get("source_title_desc_text", "") or "").strip() + if not source_title_desc_text: + source_title_desc_text = self._join_source_text( + source_rec.get("raw_title", source_rec.get("title", "")), + source_rec.get("raw_desc", source_rec.get("desc", "")), + ) + + notes: List[str] = [] + if asr_text: + clean_text = asr_text + clean_source = "asr_raw" + clean_status = "available" + clean_quality = "high" + notes.append("asr_raw_text_available") + elif source_clean_text: + clean_text = source_clean_text + clean_source = "source_clean_title" + clean_status = "available" + clean_quality = "medium" + notes.append("fallback_source_clean_title") + elif source_title_desc_text: + clean_text = source_title_desc_text + clean_source = "source_title_desc" + clean_status = "available" + clean_quality = "weak" + notes.append("fallback_source_title_desc") + else: + clean_text = "" + clean_source = "missing" + clean_status = "missing" + clean_quality = "missing" + notes.append("script_text_missing") + + asr_status = str(raw_rec.get("asr_status", "") or "").strip() + if asr_status and asr_status != "success": + notes.append(f"asr_status_{asr_status}") + + source_clean_available = bool(source_clean_text) + source_title_desc_available = bool(source_title_desc_text) + return { + "source_keyword": str(source_rec.get("source_keyword", raw_rec.get("source_keyword", "")) or ""), + "platform": str(source_rec.get("platform", raw_rec.get("platform", "douyin")) or "douyin"), + "video_id": str(source_rec.get("video_id", raw_rec.get("video_id", "")) or ""), + "aweme_id": str(source_rec.get("aweme_id", raw_rec.get("aweme_id", "")) or ""), + "aweme_url": str(source_rec.get("aweme_url", raw_rec.get("aweme_url", "")) or ""), + "script_clean_text": clean_text, + "script_clean_source": clean_source, + "script_clean_status": clean_status, + "script_clean_quality": clean_quality, + "script_clean_notes": "|".join(self._dedupe_text_list(notes)), + "asr_status": asr_status, + "asr_raw_text": asr_text, + "script_raw_quality": str(raw_rec.get("script_raw_quality", "") or ""), + "source_clean_title_available": source_clean_available, + "source_clean_title_text": source_clean_text, + "source_title_desc_available": source_title_desc_available, + "source_title_desc_text": source_title_desc_text, + "created_at": created_at, + } + + + def _do_build_script_clean( + self, + script_sources_jsonl: Optional[Path] = None, + script_sources_csv: Optional[Path] = None, + script_raw_jsonl: Optional[Path] = None, + script_raw_csv: Optional[Path] = None, + title_clean_csv: Optional[Path] = None, + ) -> tuple[Path, Path, Dict[str, Any]]: + output_dir = self._current_task_workspace_dir() / "outputs" + output_dir.mkdir(parents=True, exist_ok=True) + clean_jsonl_path = output_dir / "script_clean.jsonl" + clean_csv_path = output_dir / "script_clean.csv" + fieldnames = self._script_clean_fieldnames() + stats: Dict[str, Any] = { + "rows_in": 0, + "script_raw_rows": 0, + "rows_out": 0, + "asr_text_used": 0, + "clean_title_used": 0, + "title_desc_used": 0, + "missing": 0, + "script_clean_csv_generated": False, + "errors": [], + } + rows: List[Dict[str, Any]] = [] + + try: + source_records, source_errors = self._load_script_source_rows( + script_sources_jsonl, + script_sources_csv, + ) + raw_records, raw_errors = self._load_script_raw_rows( + script_raw_jsonl, + script_raw_csv, + ) + title_clean_map, title_clean_errors = self._load_title_clean_map(title_clean_csv) + stats["errors"].extend(source_errors) + stats["errors"].extend(raw_errors) + stats["errors"].extend(title_clean_errors) + stats["script_raw_rows"] = len(raw_records) + + records = source_records or raw_records + stats["rows_in"] = len(records) + raw_map = { + self._script_source_key(rec): rec + for rec in raw_records + if self._script_source_key(rec) + } + created_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + for idx, rec in enumerate(records, start=1): + try: + key = self._script_source_key(rec) + raw_rec = raw_map.get(key, rec if not source_records else {}) + title_clean_rec = title_clean_map.get(key) + row = self._build_script_clean_record( + rec, + raw_rec, + title_clean_rec, + created_at, + ) + if row["script_clean_source"] == "asr_raw": + stats["asr_text_used"] += 1 + elif row["script_clean_source"] == "source_clean_title": + stats["clean_title_used"] += 1 + elif row["script_clean_source"] == "source_title_desc": + stats["title_desc_used"] += 1 + else: + stats["missing"] += 1 + rows.append(row) + except Exception as e: + stats["errors"].append(f"row_{idx}_failed: {str(e)[:200]}") + + stats["rows_out"] = len(rows) + with open(str(clean_jsonl_path), "w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + with open(str(clean_csv_path), "w", encoding="utf-8-sig", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + stats["script_clean_csv_generated"] = True + except Exception as e: + stats["errors"].append(f"script_clean_failed: {str(e)[:500]}") + try: + clean_jsonl_path.write_text("", encoding="utf-8") + with open(str(clean_csv_path), "w", encoding="utf-8-sig", newline="") as f: + csv.DictWriter(f, fieldnames=fieldnames).writeheader() + stats["script_clean_csv_generated"] = True + except Exception as write_error: + stats["errors"].append(f"script_clean_write_failed: {str(write_error)[:500]}") + + return clean_jsonl_path, clean_csv_path, stats + + def _convert_jsonl_to_standard_csv( self, jsonl_path: str, diff --git a/douyin_scraper/tests/test_script_outputs.py b/douyin_scraper/tests/test_script_outputs.py new file mode 100644 index 000000000..0beee1c0a --- /dev/null +++ b/douyin_scraper/tests/test_script_outputs.py @@ -0,0 +1,333 @@ +import csv +import json +from pathlib import Path +from unittest.mock import patch + +from douyin_scraper import DouyinScraper + + +def _scraper(tmp_path: Path) -> DouyinScraper: + return DouyinScraper( + { + "project_dir": str(tmp_path), + "state_dir_name": "workspaces/scripts-task/state", + "keep_videos": True, + } + ) + + +def _write_jsonl(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), + encoding="utf-8", + ) + + +def _source( + aweme_id: str, + *, + video_url: str = "", + clean_text: str = "", + raw_text: str = "", +) -> dict: + return { + "source_keyword": "停车", + "platform": "douyin", + "video_id": aweme_id, + "aweme_id": aweme_id, + "aweme_url": f"https://www.douyin.com/video/{aweme_id}", + "video_download_url": video_url, + "source_video_download_url": video_url, + "source_video_download_available": bool(video_url), + "source_asr_planned": bool(video_url), + "source_clean_title_text": clean_text, + "source_title_desc_text": raw_text, + } + + +def test_script_sources_export_from_search_csv(tmp_path: Path) -> None: + scraper = _scraper(tmp_path) + search_csv = tmp_path / "search_result.csv" + title_csv = tmp_path / "search_title_clean.csv" + with open(search_csv, "w", encoding="utf-8-sig", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=[ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "title", + "desc", + "aweme_url", + "video_download_url", + ], + ) + writer.writeheader() + writer.writerow( + { + "source_keyword": "停车", + "platform": "douyin", + "video_id": "source-1", + "aweme_id": "source-1", + "title": "raw title", + "desc": "raw desc", + "aweme_url": "https://www.douyin.com/video/source-1", + "video_download_url": "https://example.com/source-1.mp4", + } + ) + with open(title_csv, "w", encoding="utf-8-sig", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=[ + "video_id", + "aweme_id", + "aweme_url", + "clean_title", + "clean_desc", + ], + ) + writer.writeheader() + writer.writerow( + { + "video_id": "source-1", + "aweme_id": "source-1", + "aweme_url": "https://www.douyin.com/video/source-1", + "clean_title": "clean title", + "clean_desc": "clean desc", + } + ) + + sources_jsonl, sources_csv, stats = scraper._do_build_script_sources( + search_csv, + title_clean_csv_path=title_csv, + ) + + assert stats["rows_in"] == 1 + assert stats["rows_out"] == 1 + assert stats["clean_title_available"] == 1 + assert stats["asr_planned"] == 1 + assert sources_csv.read_bytes()[:3] == b"\xef\xbb\xbf" + row = json.loads(sources_jsonl.read_text(encoding="utf-8").strip()) + assert row["source_clean_title_text"] == "clean title clean desc" + assert row["script_source_quality"] == "medium" + + +def test_script_sources_fallback_to_search_jsonl(tmp_path: Path) -> None: + scraper = _scraper(tmp_path) + missing_csv = tmp_path / "search_result.csv" + search_jsonl = tmp_path / "search_result.jsonl" + _write_jsonl( + search_jsonl, + [ + { + "aweme_id": "fallback-1", + "video_id": "fallback-1", + "title": "fallback title", + "desc": "fallback desc", + "aweme_url": "https://www.douyin.com/video/fallback-1", + } + ], + ) + + sources_jsonl, sources_csv, stats = scraper._do_build_script_sources( + missing_csv, + search_jsonl, + ) + + assert stats["rows_in"] == 1 + assert stats["rows_out"] == 1 + assert sources_jsonl.exists() + assert sources_csv.exists() + row = json.loads(sources_jsonl.read_text(encoding="utf-8").strip()) + assert row["source_title_desc_text"] == "fallback title fallback desc" + + +def test_script_raw_from_script_sources_jsonl_success_and_skips( + tmp_path: Path, +) -> None: + scraper = _scraper(tmp_path) + sources = tmp_path / "script_sources.jsonl" + _write_jsonl( + sources, + [ + _source("raw-1", video_url="https://example.com/raw-1.mp4"), + _source("skip-1"), + ], + ) + + with ( + patch("douyin_scraper.core.check_disk_space_enforced"), + patch.object( + scraper, + "_load_script_raw_whisper_model", + return_value=(object(), "", ""), + ), + patch.object(scraper, "_download_video", return_value=True), + patch.object(scraper, "_transcribe_video", return_value="ASR text"), + ): + raw_jsonl, raw_csv, stats = scraper._do_build_script_raw( + script_sources_jsonl=sources, + max_items=1, + ) + + assert stats["rows_in"] == 2 + assert stats["rows_targeted"] == 1 + assert stats["asr_success"] == 1 + assert raw_csv.read_bytes()[:3] == b"\xef\xbb\xbf" + rows = [ + json.loads(line) + for line in raw_jsonl.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + by_id = {row["aweme_id"]: row for row in rows} + assert by_id["raw-1"]["download_status"] == "success" + assert by_id["raw-1"]["asr_status"] == "success" + assert by_id["skip-1"]["download_status"] == "skipped" + + +def test_script_raw_fallback_to_csv_download_failed_and_empty_asr( + tmp_path: Path, +) -> None: + scraper = _scraper(tmp_path) + sources_csv = tmp_path / "script_sources.csv" + rows = [ + _source("download-failed", video_url="https://example.com/failed.mp4"), + _source("empty-asr", video_url="https://example.com/empty.mp4"), + ] + with open(sources_csv, "w", encoding="utf-8-sig", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + with ( + patch("douyin_scraper.core.check_disk_space_enforced"), + patch.object( + scraper, + "_load_script_raw_whisper_model", + return_value=(object(), "", ""), + ), + patch.object(scraper, "_download_video", side_effect=[False, True]), + patch.object(scraper, "_transcribe_video", return_value=""), + ): + raw_jsonl, _, stats = scraper._do_build_script_raw( + script_sources_csv=sources_csv, + max_items=2, + ) + + assert stats["download_failed"] == 1 + assert stats["asr_empty_text"] == 1 + rows = [ + json.loads(line) + for line in raw_jsonl.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + by_id = {row["aweme_id"]: row for row in rows} + assert by_id["download-failed"]["download_status"] == "failed" + assert by_id["empty-asr"]["asr_status"] == "empty_text" + + +def test_script_raw_dependency_missing_does_not_download(tmp_path: Path) -> None: + scraper = _scraper(tmp_path) + sources = tmp_path / "script_sources.jsonl" + _write_jsonl( + sources, + [_source("dependency-1", video_url="https://example.com/video.mp4")], + ) + + with ( + patch("douyin_scraper.core.check_disk_space_enforced"), + patch.object( + scraper, + "_load_script_raw_whisper_model", + return_value=(None, "dependency_missing", "missing"), + ), + patch.object(scraper, "_download_video") as download, + ): + raw_jsonl, _, stats = scraper._do_build_script_raw( + script_sources_jsonl=sources, + max_items=1, + ) + + download.assert_not_called() + assert stats["asr_dependency_missing"] == 1 + row = json.loads(raw_jsonl.read_text(encoding="utf-8").strip()) + assert row["asr_status"] == "dependency_missing" + assert row["download_status"] == "skipped" + + +def test_script_clean_priority_from_raw_sources_and_title_clean( + tmp_path: Path, +) -> None: + scraper = _scraper(tmp_path) + sources = tmp_path / "script_sources.jsonl" + raw = tmp_path / "script_raw.jsonl" + title_csv = tmp_path / "search_title_clean.csv" + _write_jsonl( + sources, + [ + _source("asr-1", clean_text="clean fallback", raw_text="raw fallback"), + _source("source-clean-1", clean_text="source clean text"), + _source("title-clean-1"), + _source("title-desc-1", raw_text="source title desc"), + _source("missing-1"), + ], + ) + _write_jsonl( + raw, + [ + { + "aweme_id": "asr-1", + "video_id": "asr-1", + "asr_status": "success", + "asr_raw_text": "ASR wins", + "script_raw_quality": "high", + } + ], + ) + with open(title_csv, "w", encoding="utf-8-sig", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=[ + "video_id", + "aweme_id", + "aweme_url", + "clean_title", + "clean_desc", + ], + ) + writer.writeheader() + writer.writerow( + { + "video_id": "title-clean-1", + "aweme_id": "title-clean-1", + "aweme_url": "https://www.douyin.com/video/title-clean-1", + "clean_title": "title clean", + "clean_desc": "title desc", + } + ) + + clean_jsonl, clean_csv, stats = scraper._do_build_script_clean( + script_sources_jsonl=sources, + script_raw_jsonl=raw, + title_clean_csv=title_csv, + ) + + assert stats["rows_out"] == 5 + assert stats["asr_text_used"] == 1 + assert stats["clean_title_used"] == 2 + assert stats["title_desc_used"] == 1 + assert stats["missing"] == 1 + assert clean_csv.read_bytes()[:3] == b"\xef\xbb\xbf" + rows = [ + json.loads(line) + for line in clean_jsonl.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + by_id = {row["aweme_id"]: row for row in rows} + assert by_id["asr-1"]["script_clean_source"] == "asr_raw" + assert by_id["source-clean-1"]["script_clean_source"] == "source_clean_title" + assert by_id["title-clean-1"]["script_clean_text"] == "title clean title desc" + assert by_id["title-desc-1"]["script_clean_source"] == "source_title_desc" + assert by_id["missing-1"]["script_clean_status"] == "missing" From 55199ea50bb577d142899f285ca89e34db440209 Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 13:51:50 +0800 Subject: [PATCH 06/32] feat(content-asset): add content asset builder --- api/routes.py | 50 +++ douyin_scraper/content_asset.py | 379 ++++++++++++++++++ douyin_scraper/core.py | 55 +++ douyin_scraper/tests/test_content_asset.py | 442 +++++++++++++++++++++ 4 files changed, 926 insertions(+) create mode 100644 douyin_scraper/content_asset.py create mode 100644 douyin_scraper/tests/test_content_asset.py diff --git a/api/routes.py b/api/routes.py index ea23b5bb7..6d6fea1a1 100644 --- a/api/routes.py +++ b/api/routes.py @@ -112,6 +112,9 @@ class ScriptsRequest(BaseModel): class MergeRequest(BaseModel): """合并数据请求""" + search_task_id: Optional[str] = Field(None, description="搜索任务 ID（生成 content_asset）") + comments_task_id: Optional[str] = Field(None, description="评论任务 ID（可选）") + scripts_task_id: Optional[str] = Field(None, description="文案任务 ID（可选）") video_jsonl: Optional[str] = Field(None, description="视频 JSONL 路径") comments_jsonl: Optional[str] = Field(None, description="评论 JSONL 路径") scripts_jsonl: Optional[str] = Field(None, description="文案 JSONL 路径") @@ -437,6 +440,53 @@ async def merge(req: MergeRequest) -> Dict[str, Any]: def _do_merge() -> Dict[str, Any]: scraper = _make_scraper(req.project_dir, task.workspace) + if req.search_task_id: + search_task = tm.get_task(req.search_task_id) + if not search_task or search_task.status != "completed": + raise NonRetryableError( + f"搜索任务不可用: {req.search_task_id}", + step="merge_csv", + ) + search_outputs = Path(search_task.workspace) / "outputs" + search_csv = search_outputs / "search_result.csv" + if not search_csv.exists(): + raise NonRetryableError( + f"搜索任务无 search_result.csv: {req.search_task_id}", + step="merge_csv", + ) + + comments_outputs: Optional[Path] = None + if req.comments_task_id: + comments_task = tm.get_task(req.comments_task_id) + if not comments_task or comments_task.status != "completed": + raise NonRetryableError( + f"评论任务不可用: {req.comments_task_id}", + step="merge_csv", + ) + comments_outputs = Path(comments_task.workspace) / "outputs" + + scripts_outputs: Optional[Path] = None + if req.scripts_task_id: + scripts_task = tm.get_task(req.scripts_task_id) + if not scripts_task or scripts_task.status != "completed": + raise NonRetryableError( + f"文案任务不可用: {req.scripts_task_id}", + step="merge_csv", + ) + scripts_outputs = Path(scripts_task.workspace) / "outputs" + + jsonl_path, csv_path, stats = scraper.build_content_asset( + search_outputs_dir=search_outputs, + comments_outputs_dir=comments_outputs, + scripts_outputs_dir=scripts_outputs, + ) + return { + "content_asset_jsonl": str(jsonl_path), + "content_asset_csv": str(csv_path), + "content_asset_stats": stats, + "status": scraper.get_status(), + } + workspace = Path(task.workspace) v_path = Path(req.video_jsonl) if req.video_jsonl else None c_path = Path(req.comments_jsonl) if req.comments_jsonl else None diff --git a/douyin_scraper/content_asset.py b/douyin_scraper/content_asset.py new file mode 100644 index 000000000..20cc88848 --- /dev/null +++ b/douyin_scraper/content_asset.py @@ -0,0 +1,379 @@ +"""Build first-version content_asset outputs from standardized task CSV files.""" + +import csv +import json +import time +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + + +CONTENT_ASSET_FIELDNAMES = [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "aweme_url", + "raw_title", + "clean_title", + "desc", + "clean_desc", + "topic", + "pain_point", + "teaching_angle", + "nickname", + "liked_count", + "collected_count", + "comment_count", + "share_count", + "total_engagement", + "valid_comment_count", + "top_valid_comments", + "comment_pain_tags", + "script_source_status", + "script_source_quality", + "script_clean_text", + "script_clean_source", + "script_clean_quality", + "comment_data_status", + "asr_data_status", + "asset_quality", + "created_at", +] + + +_MATCH_FIELDS = ("aweme_id", "video_id", "aweme_url") +_COMMENT_MATCH_FIELDS = ("aweme_id", "video_id") + + +def _empty_stats() -> Dict[str, Any]: + return { + "rows_in": 0, + "rows_out": 0, + "comments_available": 0, + "scripts_available": 0, + "valid_comments_total": 0, + "asr_available": 0, + "fallback_script_total": 0, + "missing_script_total": 0, + "content_asset_csv_generated": False, + "errors": [], + } + + +def _text(value: Any) -> str: + if value is None: + return "" + text = str(value).strip() + if text == "None": + return "" + return text + + +def _safe_int(value: Any) -> int: + try: + if value is None or value == "": + return 0 + return int(float(str(value).strip())) + except (TypeError, ValueError): + return 0 + + +def _truthy(value: Any) -> bool: + if isinstance(value, bool): + return value + return _text(value).lower() in {"1", "true", "yes", "y", "on"} + + +def _read_csv(path: Optional[Path], label: str, stats: Dict[str, Any]) -> List[Dict[str, Any]]: + if not path or not path.exists(): + stats["errors"].append(f"{label}_missing: {path or ''}") + return [] + try: + with open(str(path), "r", encoding="utf-8-sig", newline="") as f: + return [dict(row) for row in csv.DictReader(f)] + except Exception as exc: + stats["errors"].append(f"{label}_read_failed: {str(exc)[:200]}") + return [] + + +def _build_index( + rows: Iterable[Dict[str, Any]], + fields: Tuple[str, ...] = _MATCH_FIELDS, +) -> Dict[str, Dict[str, Dict[str, Any]]]: + index: Dict[str, Dict[str, Dict[str, Any]]] = {field: {} for field in fields} + for row in rows: + for field in fields: + value = _text(row.get(field)) + if value and value not in index[field]: + index[field][value] = row + return index + + +def _find_match( + index: Dict[str, Dict[str, Dict[str, Any]]], + row: Dict[str, Any], + fields: Tuple[str, ...] = _MATCH_FIELDS, +) -> Dict[str, Any]: + for field in fields: + value = _text(row.get(field)) + if value and value in index.get(field, {}): + return index[field][value] + return {} + + +def _dedupe(items: Iterable[str]) -> List[str]: + result: List[str] = [] + seen = set() + for item in items: + value = _text(item) + if not value or value in seen: + continue + seen.add(value) + result.append(value) + return result + + +def _split_tags(value: Any) -> List[str]: + tags: List[str] = [] + for part in _text(value).split("|"): + part = part.strip() + if part: + tags.append(part) + return tags + + +def _aggregate_comments(rows: Iterable[Dict[str, Any]]) -> Dict[str, Dict[str, Dict[str, Any]]]: + grouped: Dict[str, Dict[str, Any]] = {} + aliases: Dict[str, str] = {} + + for row in rows: + if not _truthy(row.get("is_valid")): + continue + primary = _text(row.get("aweme_id")) or _text(row.get("video_id")) + if not primary: + continue + bucket = grouped.setdefault(primary, {"count": 0, "comments": [], "tags": []}) + bucket["count"] += 1 + clean_content = _text(row.get("clean_content")) + if clean_content and len(bucket["comments"]) < 3: + bucket["comments"].append(clean_content) + bucket["tags"].extend(_split_tags(row.get("pain_tags"))) + + for field in _COMMENT_MATCH_FIELDS: + value = _text(row.get(field)) + if value: + aliases.setdefault(value, primary) + + index = {field: {} for field in _COMMENT_MATCH_FIELDS} + for alias, primary in aliases.items(): + bucket = grouped[primary] + aggregate = { + "valid_comment_count": bucket["count"], + "top_valid_comments": "|".join(bucket["comments"]), + "comment_pain_tags": "|".join(_dedupe(bucket["tags"])), + } + for field in _COMMENT_MATCH_FIELDS: + index[field][alias] = aggregate + return index + + +def _comment_status( + comments_clean_exists: bool, + comment_row: Dict[str, Any], +) -> str: + if not comments_clean_exists: + return "pending_cdp" + if _safe_int(comment_row.get("valid_comment_count")) > 0: + return "available" + return "empty" + + +def _asr_status( + script_clean_exists: bool, + script_source: Dict[str, Any], + script_raw: Dict[str, Any], + script_clean: Dict[str, Any], +) -> str: + raw_status = _text(script_raw.get("asr_status")) or _text(script_clean.get("asr_status")) + clean_source = _text(script_clean.get("script_clean_source")) + clean_text = _text(script_clean.get("script_clean_text")) + + if raw_status == "dependency_missing": + return "dependency_missing" + if not script_clean_exists: + if _truthy(script_source.get("source_asr_planned")): + return "pending_asr" + return "missing" + if clean_source == "asr_raw" and clean_text: + return "available" + if clean_source == "source_clean_title" and clean_text: + return "fallback_title" + if clean_source == "source_title_desc" and clean_text: + return "fallback_desc" + return "missing" + + +def _asset_quality(row: Dict[str, Any]) -> str: + valid_comments = _safe_int(row.get("valid_comment_count")) + script_source = _text(row.get("script_clean_source")) + script_text = _text(row.get("script_clean_text")) + has_clean_title = bool(_text(row.get("clean_title")) or _text(row.get("clean_desc"))) + has_any_title = bool( + _text(row.get("raw_title")) + or _text(row.get("clean_title")) + or _text(row.get("desc")) + or _text(row.get("clean_desc")) + ) + + if valid_comments > 0 and script_source == "asr_raw" and script_text: + return "high" + if not has_any_title and not script_text: + return "missing" + if has_clean_title and script_source in {"source_clean_title", "source_title_desc"} and script_text: + return "medium" + if script_text and ( + row.get("comment_data_status") != "available" + or row.get("asr_data_status") != "available" + ): + return "partial" + return "low" + + +def _build_asset_row( + search_row: Dict[str, Any], + title_row: Dict[str, Any], + comment_row: Dict[str, Any], + script_source_row: Dict[str, Any], + script_raw_row: Dict[str, Any], + script_clean_row: Dict[str, Any], + comments_clean_exists: bool, + script_clean_exists: bool, + created_at: str, +) -> Dict[str, Any]: + liked = _safe_int(search_row.get("liked_count")) + collected = _safe_int(search_row.get("collected_count")) + comment_count = _safe_int(search_row.get("comment_count")) + share = _safe_int(search_row.get("share_count")) + total = _safe_int(search_row.get("total_engagement")) or liked + collected + comment_count + share + + clean_title = _text(title_row.get("clean_title")) or _text(search_row.get("clean_title")) + clean_desc = _text(title_row.get("clean_desc")) or _text(search_row.get("clean_desc")) + script_clean_text = _text(script_clean_row.get("script_clean_text")) + script_clean_source = _text(script_clean_row.get("script_clean_source")) + + row: Dict[str, Any] = { + "source_keyword": _text(search_row.get("source_keyword")), + "platform": _text(search_row.get("platform")) or "douyin", + "video_id": _text(search_row.get("video_id")) or _text(search_row.get("aweme_id")), + "aweme_id": _text(search_row.get("aweme_id")), + "aweme_url": _text(search_row.get("aweme_url")), + "raw_title": _text(search_row.get("title")) or _text(title_row.get("raw_title")), + "clean_title": clean_title, + "desc": _text(search_row.get("desc")) or _text(title_row.get("raw_desc")), + "clean_desc": clean_desc, + "topic": _text(title_row.get("topic")), + "pain_point": _text(title_row.get("pain_point")), + "teaching_angle": _text(title_row.get("teaching_angle")), + "nickname": _text(search_row.get("nickname")), + "liked_count": liked, + "collected_count": collected, + "comment_count": comment_count, + "share_count": share, + "total_engagement": total, + "valid_comment_count": _safe_int(comment_row.get("valid_comment_count")), + "top_valid_comments": _text(comment_row.get("top_valid_comments")), + "comment_pain_tags": _text(comment_row.get("comment_pain_tags")), + "script_source_status": _text(script_source_row.get("script_source_status")), + "script_source_quality": _text(script_source_row.get("script_source_quality")), + "script_clean_text": script_clean_text, + "script_clean_source": script_clean_source, + "script_clean_quality": _text(script_clean_row.get("script_clean_quality")), + "comment_data_status": _comment_status(comments_clean_exists, comment_row), + "asr_data_status": _asr_status( + script_clean_exists, + script_source_row, + script_raw_row, + script_clean_row, + ), + "asset_quality": "", + "created_at": created_at, + } + row["asset_quality"] = _asset_quality(row) + return row + + +def build_content_asset( + *, + search_result_csv: Path, + output_dir: Path, + search_title_clean_csv: Optional[Path] = None, + comments_clean_csv: Optional[Path] = None, + script_sources_csv: Optional[Path] = None, + script_raw_csv: Optional[Path] = None, + script_clean_csv: Optional[Path] = None, +) -> Tuple[Path, Path, Dict[str, Any]]: + """Generate content_asset.jsonl/csv and return their paths plus stats.""" + stats = _empty_stats() + output_dir.mkdir(parents=True, exist_ok=True) + jsonl_path = output_dir / "content_asset.jsonl" + csv_path = output_dir / "content_asset.csv" + + search_rows = _read_csv(search_result_csv, "search_result", stats) + title_rows = _read_csv(search_title_clean_csv, "search_title_clean", stats) + comments_clean_exists = bool(comments_clean_csv and comments_clean_csv.exists()) + comment_rows = _read_csv(comments_clean_csv, "comments_clean", stats) + script_source_rows = _read_csv(script_sources_csv, "script_sources", stats) + script_raw_rows = _read_csv(script_raw_csv, "script_raw", stats) + script_clean_exists = bool(script_clean_csv and script_clean_csv.exists()) + script_clean_rows = _read_csv(script_clean_csv, "script_clean", stats) + + title_index = _build_index(title_rows) + comment_index = _aggregate_comments(comment_rows) + script_source_index = _build_index(script_source_rows) + script_raw_index = _build_index(script_raw_rows) + script_clean_index = _build_index(script_clean_rows) + + stats["rows_in"] = len(search_rows) + created_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + rows: List[Dict[str, Any]] = [] + for search_row in search_rows: + title_row = _find_match(title_index, search_row) + comment_row = _find_match(comment_index, search_row, _COMMENT_MATCH_FIELDS) + script_source_row = _find_match(script_source_index, search_row) + script_raw_row = _find_match(script_raw_index, search_row) + script_clean_row = _find_match(script_clean_index, search_row) + row = _build_asset_row( + search_row, + title_row, + comment_row, + script_source_row, + script_raw_row, + script_clean_row, + comments_clean_exists, + script_clean_exists, + created_at, + ) + rows.append(row) + + stats["rows_out"] = len(rows) + stats["comments_available"] = sum(1 for row in rows if row["comment_data_status"] == "available") + stats["scripts_available"] = sum(1 for row in rows if _text(row.get("script_clean_text"))) + stats["valid_comments_total"] = sum(_safe_int(row.get("valid_comment_count")) for row in rows) + stats["asr_available"] = sum(1 for row in rows if row["asr_data_status"] == "available") + stats["fallback_script_total"] = sum( + 1 for row in rows if row["asr_data_status"] in {"fallback_title", "fallback_desc"} + ) + stats["missing_script_total"] = sum( + 1 for row in rows if not _text(row.get("script_clean_text")) + ) + + with open(str(jsonl_path), "w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + with open(str(csv_path), "w", encoding="utf-8-sig", newline="") as f: + writer = csv.DictWriter(f, fieldnames=CONTENT_ASSET_FIELDNAMES) + writer.writeheader() + writer.writerows(rows) + stats["content_asset_csv_generated"] = True + return jsonl_path, csv_path, stats diff --git a/douyin_scraper/core.py b/douyin_scraper/core.py index 4bbf33471..18e23f11b 100644 --- a/douyin_scraper/core.py +++ b/douyin_scraper/core.py @@ -539,6 +539,54 @@ def merge( self._state.mark_step_failed(step, str(e)[:200], exit_code) raise + def build_content_asset( + self, + search_outputs_dir: Path, + comments_outputs_dir: Optional[Path] = None, + scripts_outputs_dir: Optional[Path] = None, + ) -> tuple[Path, Path, Dict[str, Any]]: + """Build content_asset outputs in the current task workspace.""" + from douyin_scraper.content_asset import build_content_asset + + step = self.STEP_MERGE + self._state.mark_step_started(step) + try: + output_dir = self._current_task_workspace_dir() / "outputs" + jsonl_path, csv_path, stats = build_content_asset( + search_result_csv=search_outputs_dir / "search_result.csv", + search_title_clean_csv=search_outputs_dir / "search_title_clean.csv", + comments_clean_csv=( + comments_outputs_dir / "comments_clean.csv" + if comments_outputs_dir is not None + else None + ), + script_sources_csv=( + scripts_outputs_dir / "script_sources.csv" + if scripts_outputs_dir is not None + else None + ), + script_raw_csv=( + scripts_outputs_dir / "script_raw.csv" + if scripts_outputs_dir is not None + else None + ), + script_clean_csv=( + scripts_outputs_dir / "script_clean.csv" + if scripts_outputs_dir is not None + else None + ), + output_dir=output_dir, + ) + self._state.mark_step_completed(step, detail=f"output={csv_path}") + self._paths["content_asset_jsonl"] = jsonl_path + self._paths["content_asset_csv"] = csv_path + self._paths["content_asset_stats"] = stats + return jsonl_path, csv_path, stats + except Exception as e: + exit_code = classify_error(e) + self._state.mark_step_failed(step, str(e)[:200], exit_code) + raise + def run_all( self, steps: Optional[List[str]] = None, @@ -606,6 +654,7 @@ def get_paths(self) -> Dict[str, Any]: result.update(self._comments_output_paths()) result.update(self._title_output_paths()) result.update(self._script_output_paths()) + result.update(self._content_asset_output_paths()) return result @@ -670,6 +719,12 @@ def _script_output_paths(self) -> Dict[str, Any]: ), ) + def _content_asset_output_paths(self) -> Dict[str, Any]: + return self._select_output_values( + path_keys=("content_asset_jsonl", "content_asset_csv"), + stats_keys=("content_asset_stats",), + ) + def _require_step_ready(self, step: str) -> None: """检查步骤是否可执行，否则抛异常""" diff --git a/douyin_scraper/tests/test_content_asset.py b/douyin_scraper/tests/test_content_asset.py new file mode 100644 index 000000000..ff97216d4 --- /dev/null +++ b/douyin_scraper/tests/test_content_asset.py @@ -0,0 +1,442 @@ +import csv +import json +from pathlib import Path + +from douyin_scraper import DouyinScraper + + +def _write_csv(path: Path, fieldnames: list[str], rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8-sig", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def _read_jsonl(path: Path) -> list[dict]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def test_content_asset_from_search_and_title_only(tmp_path: Path) -> None: + search_outputs = tmp_path / "workspaces" / "search-task" / "outputs" + _write_csv( + search_outputs / "search_result.csv", + [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "title", + "desc", + "nickname", + "liked_count", + "collected_count", + "comment_count", + "share_count", + "total_engagement", + "aweme_url", + ], + [{ + "source_keyword": "parking", + "platform": "douyin", + "video_id": "a1", + "aweme_id": "a1", + "title": "\u4e2d\u6587\u6807\u9898", + "desc": "raw desc", + "nickname": "coach", + "liked_count": "10", + "collected_count": "2", + "comment_count": "3", + "share_count": "1", + "total_engagement": "16", + "aweme_url": "https://www.douyin.com/video/a1", + }], + ) + _write_csv( + search_outputs / "search_title_clean.csv", + [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "raw_title", + "clean_title", + "raw_desc", + "clean_desc", + "topic", + "pain_point", + "teaching_angle", + "aweme_url", + ], + [{ + "source_keyword": "parking", + "platform": "douyin", + "video_id": "a1", + "aweme_id": "a1", + "raw_title": "\u4e2d\u6587\u6807\u9898", + "clean_title": "\u4e2d\u6587", + "raw_desc": "raw desc", + "clean_desc": "clean desc", + "topic": "parking", + "pain_point": "line", + "teaching_angle": "angle", + "aweme_url": "https://www.douyin.com/video/a1", + }], + ) + + scraper = DouyinScraper({ + "project_dir": str(tmp_path), + "state_dir_name": "workspaces/merge-task/state", + }) + jsonl_path, csv_path, stats = scraper.build_content_asset(search_outputs) + + assert stats["rows_in"] == 1 + assert stats["rows_out"] == 1 + assert stats["content_asset_csv_generated"] is True + assert csv_path.read_bytes()[:3] == b"\xef\xbb\xbf" + json_text = jsonl_path.read_text(encoding="utf-8") + assert "\u4e2d\u6587" in json_text + assert "\\u4e2d\\u6587" not in json_text + + row = _read_jsonl(jsonl_path)[0] + assert row["clean_title"] == "\u4e2d\u6587" + assert row["comment_data_status"] == "pending_cdp" + assert row["asr_data_status"] == "missing" + assert row["asset_quality"] == "low" + paths = scraper.get_paths() + assert paths["content_asset_jsonl"] == str(jsonl_path) + assert paths["content_asset_csv"] == str(csv_path) + assert paths["content_asset_stats"]["rows_out"] == 1 + + +def test_content_asset_aggregates_comments_and_scripts(tmp_path: Path) -> None: + search_outputs = tmp_path / "workspaces" / "search-task" / "outputs" + comments_outputs = tmp_path / "workspaces" / "comments-task" / "outputs" + scripts_outputs = tmp_path / "workspaces" / "scripts-task" / "outputs" + search_fields = [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "title", + "desc", + "nickname", + "liked_count", + "collected_count", + "comment_count", + "share_count", + "total_engagement", + "aweme_url", + ] + _write_csv( + search_outputs / "search_result.csv", + search_fields, + [ + { + "source_keyword": "parking", + "platform": "douyin", + "video_id": "video-1", + "aweme_id": "a1", + "title": "raw one", + "desc": "desc one", + "nickname": "coach1", + "liked_count": "5", + "collected_count": "1", + "comment_count": "9", + "share_count": "2", + "total_engagement": "", + "aweme_url": "https://www.douyin.com/video/a1", + }, + { + "source_keyword": "parking", + "platform": "douyin", + "video_id": "video-2", + "aweme_id": "", + "title": "raw two", + "desc": "desc two", + "nickname": "coach2", + "liked_count": "1", + "collected_count": "1", + "comment_count": "0", + "share_count": "1", + "total_engagement": "3", + "aweme_url": "https://www.douyin.com/video/v2", + }, + { + "source_keyword": "parking", + "platform": "douyin", + "video_id": "video-3", + "aweme_id": "a3", + "title": "raw three", + "desc": "desc three", + "nickname": "coach3", + "liked_count": "1", + "collected_count": "0", + "comment_count": "1", + "share_count": "0", + "total_engagement": "2", + "aweme_url": "https://www.douyin.com/video/a3", + }, + ], + ) + _write_csv( + search_outputs / "search_title_clean.csv", + [ + "source_keyword", + "platform", + "video_id", + "aweme_id", + "raw_title", + "clean_title", + "raw_desc", + "clean_desc", + "topic", + "pain_point", + "teaching_angle", + "aweme_url", + ], + [ + { + "source_keyword": "parking", + "platform": "douyin", + "video_id": "video-1", + "aweme_id": "a1", + "raw_title": "raw one", + "clean_title": "clean one", + "raw_desc": "desc one", + "clean_desc": "clean desc one", + "topic": "topic one", + "pain_point": "pain one", + "teaching_angle": "angle one", + "aweme_url": "https://www.douyin.com/video/a1", + }, + { + "source_keyword": "parking", + "platform": "douyin", + "video_id": "video-2", + "aweme_id": "", + "raw_title": "raw two", + "clean_title": "clean two", + "raw_desc": "desc two", + "clean_desc": "clean desc two", + "topic": "topic two", + "pain_point": "pain two", + "teaching_angle": "angle two", + "aweme_url": "https://www.douyin.com/video/v2", + }, + ], + ) + _write_csv( + comments_outputs / "comments_clean.csv", + [ + "video_id", + "aweme_id", + "comment_id", + "clean_content", + "is_valid", + "invalid_reason", + "pain_tags", + ], + [ + { + "video_id": "video-1", + "aweme_id": "a1", + "comment_id": "c1", + "clean_content": "valid one", + "is_valid": "true", + "invalid_reason": "", + "pain_tags": "line|mirror", + }, + { + "video_id": "video-1", + "aweme_id": "a1", + "comment_id": "c2", + "clean_content": "valid two", + "is_valid": "1", + "invalid_reason": "", + "pain_tags": "line|nervous", + }, + { + "video_id": "video-1", + "aweme_id": "a1", + "comment_id": "c3", + "clean_content": "invalid", + "is_valid": "false", + "invalid_reason": "duplicate", + "pain_tags": "ignore", + }, + { + "video_id": "video-3", + "aweme_id": "a3", + "comment_id": "c4", + "clean_content": "valid three", + "is_valid": "true", + "invalid_reason": "", + "pain_tags": "line", + }, + ], + ) + _write_csv( + scripts_outputs / "script_sources.csv", + [ + "video_id", + "aweme_id", + "aweme_url", + "source_asr_planned", + "script_source_status", + "script_source_quality", + ], + [ + { + "video_id": "video-1", + "aweme_id": "a1", + "aweme_url": "https://www.douyin.com/video/a1", + "source_asr_planned": "true", + "script_source_status": "available", + "script_source_quality": "medium", + }, + { + "video_id": "video-2", + "aweme_id": "", + "aweme_url": "https://www.douyin.com/video/v2", + "source_asr_planned": "false", + "script_source_status": "available", + "script_source_quality": "weak", + }, + { + "video_id": "video-3", + "aweme_id": "a3", + "aweme_url": "https://www.douyin.com/video/a3", + "source_asr_planned": "true", + "script_source_status": "available", + "script_source_quality": "high", + }, + ], + ) + _write_csv( + scripts_outputs / "script_raw.csv", + [ + "video_id", + "aweme_id", + "aweme_url", + "asr_status", + "asr_raw_text", + "script_raw_quality", + ], + [ + { + "video_id": "video-1", + "aweme_id": "a1", + "aweme_url": "https://www.douyin.com/video/a1", + "asr_status": "dependency_missing", + "asr_raw_text": "", + "script_raw_quality": "missing", + }, + { + "video_id": "video-3", + "aweme_id": "a3", + "aweme_url": "https://www.douyin.com/video/a3", + "asr_status": "success", + "asr_raw_text": "asr text", + "script_raw_quality": "high", + }, + ], + ) + _write_csv( + scripts_outputs / "script_clean.csv", + [ + "video_id", + "aweme_id", + "aweme_url", + "script_clean_text", + "script_clean_source", + "script_clean_status", + "script_clean_quality", + "asr_status", + ], + [ + { + "video_id": "video-1", + "aweme_id": "a1", + "aweme_url": "https://www.douyin.com/video/a1", + "script_clean_text": "fallback title text", + "script_clean_source": "source_clean_title", + "script_clean_status": "available", + "script_clean_quality": "medium", + "asr_status": "dependency_missing", + }, + { + "video_id": "video-2", + "aweme_id": "", + "aweme_url": "https://www.douyin.com/video/v2", + "script_clean_text": "fallback desc text", + "script_clean_source": "source_title_desc", + "script_clean_status": "available", + "script_clean_quality": "weak", + "asr_status": "skipped", + }, + { + "video_id": "video-3", + "aweme_id": "a3", + "aweme_url": "https://www.douyin.com/video/a3", + "script_clean_text": "asr text", + "script_clean_source": "asr_raw", + "script_clean_status": "available", + "script_clean_quality": "high", + "asr_status": "success", + }, + ], + ) + + scraper = DouyinScraper({ + "project_dir": str(tmp_path), + "state_dir_name": "workspaces/merge-task/state", + }) + jsonl_path, csv_path, stats = scraper.build_content_asset( + search_outputs, + comments_outputs, + scripts_outputs, + ) + + assert csv_path.read_bytes()[:3] == b"\xef\xbb\xbf" + rows = _read_jsonl(jsonl_path) + by_video = {row["video_id"]: row for row in rows} + row1 = by_video["video-1"] + assert row1["valid_comment_count"] == 2 + assert row1["top_valid_comments"] == "valid one|valid two" + assert row1["comment_pain_tags"] == "line|mirror|nervous" + assert row1["comment_data_status"] == "available" + assert row1["script_source_status"] == "available" + assert row1["script_source_quality"] == "medium" + assert row1["script_clean_text"] == "fallback title text" + assert row1["script_clean_source"] == "source_clean_title" + assert row1["script_clean_quality"] == "medium" + assert row1["asr_data_status"] == "dependency_missing" + assert row1["asset_quality"] != "high" + + row2 = by_video["video-2"] + assert row2["aweme_id"] == "" + assert row2["clean_title"] == "clean two" + assert row2["script_clean_text"] == "fallback desc text" + assert row2["script_clean_source"] == "source_title_desc" + assert row2["asr_data_status"] == "fallback_desc" + assert row2["comment_data_status"] == "empty" + + row3 = by_video["video-3"] + assert row3["valid_comment_count"] == 1 + assert row3["script_clean_source"] == "asr_raw" + assert row3["asr_data_status"] == "available" + assert row3["asset_quality"] == "high" + + assert stats["rows_in"] == 3 + assert stats["rows_out"] == 3 + assert stats["comments_available"] == 2 + assert stats["valid_comments_total"] == 3 + assert stats["asr_available"] == 1 + assert stats["fallback_script_total"] == 1 + assert stats["missing_script_total"] == 0 + assert stats["content_asset_csv_generated"] is True From adcc7f293cdad70be0529edfbec1bb1f71cf84cb Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 14:01:51 +0800 Subject: [PATCH 07/32] feat(api): expose content asset result preview and export --- api/routes.py | 108 +++++++++++----- api/tasks.py | 7 ++ .../tests/test_content_asset_api.py | 119 ++++++++++++++++++ .../tests/test_task_result_selection.py | 23 ++++ 4 files changed, 227 insertions(+), 30 deletions(-) create mode 100644 douyin_scraper/tests/test_content_asset_api.py diff --git a/api/routes.py b/api/routes.py index 6d6fea1a1..2b9e8172b 100644 --- a/api/routes.py +++ b/api/routes.py @@ -719,14 +719,25 @@ def _read_csv_rows(path: Path, limit: int = MAX_EXPORT_ROWS) -> List[Dict[str, A def _normalize_row(row: Dict[str, Any]) -> Dict[str, Any]: """规范化一行数据，确保关键字段存在""" + def first_value(*keys: str, default: str = "") -> str: + for key in keys: + value = row.get(key) + if value not in (None, ""): + return str(value) + return default + return { - "video_id": str(row.get("video_id", row.get("id", ""))), - "platform": str(row.get("platform", "douyin")), - "script_text": str(row.get("script_text", row.get("script", row.get("text", "")))), - "likes": str(row.get("likes", row.get("like_count", ""))), - "favorites": str(row.get("favorites", row.get("collect_count", ""))), - "shares": str(row.get("shares", row.get("share_count", ""))), - "comments": str(row.get("comments", row.get("comment_count", ""))), + "video_id": first_value("video_id", "aweme_id", "id"), + "platform": first_value("platform", default="douyin"), + "script_text": first_value( + "script_text", "script_clean_text", "script", "text" + ), + "likes": first_value("likes", "liked_count", "like_count"), + "favorites": first_value( + "favorites", "collected_count", "collect_count" + ), + "shares": first_value("shares", "share_count"), + "comments": first_value("comments", "comment_count"), } @@ -766,35 +777,43 @@ async def list_data_files() -> Dict[str, Any]: return {"items": items, "total": len(items)} -@router.get("/data/preview/{task_id}", summary="预览任务结果数据（前 20 行）") -async def preview_data(task_id: str) -> Dict[str, Any]: - """返回任务结果文件的前 20 行数据""" +@router.get("/data/preview/{task_id}", summary="预览任务结果数据") +async def preview_data( + task_id: str, + limit: int = Query(100, ge=1, le=1000, description="返回行数上限"), +) -> Dict[str, Any]: + """Preview the same primary result selected by /scrape/result.""" tm = get_task_manager() task = tm.get_task(task_id) if not task: raise HTTPException(status_code=404, detail=f"任务不存在: {task_id}") if task.status != "completed": - raise HTTPException(status_code=400, detail=f"任务未完成，当前状态: {task.status}") - - workspace = Path(task.workspace) - files = _find_result_files(workspace) - if not files: - raise HTTPException(status_code=404, detail="未找到结果文件") + raise HTTPException( + status_code=400, + detail=f"任务未完成，当前状态: {task.status}", + ) - # 优先 CSV，其次 JSONL - target = next((f for f in files if f.suffix == ".csv"), files[0]) + target = tm.get_result_path(task_id) + if not target or not target.is_file(): + raise HTTPException(status_code=404, detail="无可用预览数据") if target.suffix == ".csv": - rows = _read_csv_rows(target, limit=20) + rows = _read_csv_rows(target, limit=limit) + total_rows = max(_count_file_rows(target) - 1, 0) + file_format = "csv" + elif target.suffix == ".jsonl": + rows = _read_jsonl_rows(target, limit=limit) + total_rows = _count_file_rows(target) + file_format = "jsonl" else: - raw_rows = _read_jsonl_rows(target, limit=20) - rows = [_normalize_row(r) for r in raw_rows] + raise HTTPException(status_code=404, detail="无可用预览数据") return { "task_id": task_id, "file_name": target.name, "rows": rows, - "total_rows": _count_file_rows(target), + "format": file_format, + "total_rows": total_rows, } @@ -814,7 +833,39 @@ def validate_task_ids(cls, v: List[str]) -> List[str]: return v -@router.post("/data/export", summary="导出数据（CSV 或 TXT）") +@router.get("/data/export", summary="导出任务结果文件（直接下载）") +async def data_export_download( + task_id: str = Query(..., description="任务 ID"), +): + """Download the same primary result selected by /scrape/result.""" + tm = get_task_manager() + task = tm.get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail=f"任务不存在: {task_id}") + if task.status != "completed": + raise HTTPException( + status_code=400, + detail=f"任务未完成，当前状态: {task.status}", + ) + + target = tm.get_result_path(task_id) + if not target or not target.is_file(): + raise HTTPException(status_code=404, detail="无可用导出数据") + + media_type = "application/octet-stream" + if target.suffix == ".csv": + media_type = "text/csv" + elif target.suffix == ".jsonl": + media_type = "application/jsonl" + + return FileResponse( + path=str(target), + media_type=media_type, + filename=target.name, + ) + + +@router.post("/data/export", summary="批量导出数据（CSV 或 TXT）") async def export_data(req: ExportRequest): """ 批量导出多个任务的结果数据。 @@ -831,16 +882,13 @@ async def export_data(req: ExportRequest): task = tm.get_task(task_id) if not task or task.status != "completed": continue - workspace = Path(task.workspace) - files = _find_result_files(workspace) - if not files: + target = tm.get_result_path(task_id) + if not target or not target.is_file(): continue - target = next((f for f in files if f.suffix == ".csv"), files[0]) if target.suffix == ".csv": rows = _read_csv_rows(target, limit=req.limit) else: - raw_rows = _read_jsonl_rows(target, limit=req.limit) - rows = [_normalize_row(r) for r in raw_rows] + rows = _read_jsonl_rows(target, limit=req.limit) all_rows.extend(rows) if len(all_rows) >= req.limit: all_rows = all_rows[:req.limit] @@ -864,7 +912,7 @@ async def export_data(req: ExportRequest): size = output.tell() if size >= MAX_EXPORT_BYTES: break - content = output.getvalue().encode("utf-8") + content = output.getvalue().encode("utf-8-sig") media_type = "text/csv; charset=utf-8" filename = f"export_{len(req.task_ids)}tasks_{written}rows.csv" diff --git a/api/tasks.py b/api/tasks.py index e61e80842..4f218b407 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -512,6 +512,13 @@ def _scripts_result_names() -> tuple[str, ...]: "script_sources.jsonl", ) + @staticmethod + def _merge_result_names() -> tuple[str, ...]: + return ( + "content_asset.csv", + "content_asset.jsonl", + "douyin_koubo_data.csv", + ) @classmethod def _preferred_result_names(cls, task_type: str) -> tuple[str, ...]: diff --git a/douyin_scraper/tests/test_content_asset_api.py b/douyin_scraper/tests/test_content_asset_api.py new file mode 100644 index 000000000..7ee021a95 --- /dev/null +++ b/douyin_scraper/tests/test_content_asset_api.py @@ -0,0 +1,119 @@ +import asyncio +import csv +import io +from pathlib import Path +from typing import Dict + +from api.routes import ( + ExportRequest, + data_export_download, + export_data, + get_result, + preview_data, + set_task_manager, +) +from api.tasks import TaskManager + + +def _create_completed_task( + tmp_path: Path, + task_type: str, + files: Dict[str, bytes], +): + manager = TaskManager(base_dir=str(tmp_path)) + task = manager.create_task(task_type) + task.status = "completed" + outputs = Path(task.workspace) / "outputs" + outputs.mkdir(parents=True, exist_ok=True) + for name, content in files.items(): + (outputs / name).write_bytes(content) + return manager, task + + +def test_content_asset_result_preview_and_exports(tmp_path: Path) -> None: + source = ( + "video_id,platform,script_clean_text,liked_count,collected_count," + "share_count,comment_count,asset_quality\n" + "video-1,douyin,clean script,11,12,13,14,partial\n" + ).encode("utf-8-sig") + manager, task = _create_completed_task( + tmp_path, + "merge", + { + "content_asset.csv": source, + "content_asset.jsonl": b'{"video_id":"jsonl-fallback"}\n', + }, + ) + set_task_manager(manager) + + result_response = asyncio.run(get_result(task.task_id)) + preview = asyncio.run(preview_data(task.task_id, limit=1)) + export_response = asyncio.run(data_export_download(task.task_id)) + batch_response = asyncio.run( + export_data(ExportRequest(task_ids=[task.task_id], format="csv", limit=10)) + ) + + assert Path(result_response.path).name == "content_asset.csv" + assert preview["file_name"] == "content_asset.csv" + assert preview["format"] == "csv" + assert preview["total_rows"] == 1 + assert preview["rows"][0]["script_clean_text"] == "clean script" + assert Path(export_response.path).read_bytes() == source + assert source.startswith(b"\xef\xbb\xbf") + assert batch_response.body.startswith(b"\xef\xbb\xbf") + + rows = list( + csv.DictReader(io.StringIO(batch_response.body.decode("utf-8-sig"))) + ) + assert rows == [ + { + "video_id": "video-1", + "platform": "douyin", + "script_text": "clean script", + "likes": "11", + "favorites": "12", + "shares": "13", + "comments": "14", + } + ] + + +def test_preview_jsonl_fallback_uses_shared_selector(tmp_path: Path) -> None: + manager, task = _create_completed_task( + tmp_path, + "merge", + { + "content_asset.jsonl": ( + '{"video_id":"video-1","script_clean_text":"text"}\n' + ).encode("utf-8"), + }, + ) + set_task_manager(manager) + + preview = asyncio.run(preview_data(task.task_id, limit=1)) + + assert preview["file_name"] == "content_asset.jsonl" + assert preview["format"] == "jsonl" + assert preview["total_rows"] == 1 + assert preview["rows"][0]["video_id"] == "video-1" + + +def test_search_preview_keeps_search_result_semantics(tmp_path: Path) -> None: + manager, task = _create_completed_task( + tmp_path, + "search", + { + "search_result.csv": ( + "video_id,title\nvideo-1,raw title\n" + ).encode("utf-8-sig"), + "search_title_clean.csv": ( + "video_id,clean_title\nvideo-1,clean title\n" + ).encode("utf-8-sig"), + }, + ) + set_task_manager(manager) + + preview = asyncio.run(preview_data(task.task_id, limit=1)) + + assert preview["file_name"] == "search_result.csv" + assert preview["rows"][0]["title"] == "raw title" diff --git a/douyin_scraper/tests/test_task_result_selection.py b/douyin_scraper/tests/test_task_result_selection.py index 1e12e80a6..fd2c7c365 100644 --- a/douyin_scraper/tests/test_task_result_selection.py +++ b/douyin_scraper/tests/test_task_result_selection.py @@ -27,3 +27,26 @@ def test_fallback_selector_uses_supported_file(tmp_path: Path) -> None: result = _selected_result(tmp_path, "custom", ("fallback.csv",)) assert result is not None assert result.name == "fallback.csv" + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + ( + ("douyin_koubo_data.csv", "content_asset.jsonl", "content_asset.csv"), + "content_asset.csv", + ), + ( + ("legacy.csv", "douyin_koubo_data.csv"), + "douyin_koubo_data.csv", + ), + ], +) +def test_merge_selector_prefers_content_asset( + tmp_path: Path, + files: tuple[str, ...], + expected: str, +) -> None: + result = _selected_result(tmp_path, "merge", files) + assert result is not None + assert result.name == expected From 11722ba1c0632e995b6d2c5e16d2b65aa057504e Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 14:11:17 +0800 Subject: [PATCH 08/32] feat(web): establish Vite frontend baseline --- web/index.html | 16 + web/package-lock.json | 3952 +++++++++++++++++ web/package.json | 37 + web/postcss.config.js | 6 + web/public/vite.svg | 4 + web/src/App.tsx | 27 + web/src/api/client.ts | 109 + web/src/api/types.ts | 133 + web/src/components/dashboard/HealthPanel.tsx | 65 + web/src/components/dashboard/RecentTasks.tsx | 67 + web/src/components/dashboard/StatsCards.tsx | 50 + web/src/components/layout/AppLayout.tsx | 103 + .../components/shared/ConnectionIndicator.tsx | 29 + web/src/components/shared/ErrorAlert.tsx | 27 + web/src/components/shared/LoadingOverlay.tsx | 16 + .../components/task/DeleteConfirmDialog.tsx | 31 + web/src/components/task/StatusBadge.tsx | 24 + web/src/components/task/TaskTypeIcon.tsx | 29 + web/src/hooks/usePolling.ts | 17 + web/src/hooks/useWebSocket.ts | 132 + web/src/index.css | 17 + web/src/main.tsx | 23 + web/src/pages/AnalyticsPage.tsx | 134 + web/src/pages/CreateTaskPage.tsx | 237 + web/src/pages/DashboardPage.tsx | 49 + web/src/pages/DataPage.tsx | 558 +++ web/src/pages/SettingsPage.tsx | 99 + web/src/pages/TaskDetailPage.tsx | 779 ++++ web/src/pages/TaskListPage.tsx | 341 ++ web/src/store/useSettingsStore.ts | 36 + web/src/store/useTaskStore.ts | 116 + web/src/theme/theme.ts | 35 + web/src/utils/constants.ts | 36 + web/src/utils/format.ts | 31 + web/src/vite-env.d.ts | 1 + web/tailwind.config.ts | 14 + web/tsconfig.json | 21 + web/tsconfig.node.json | 15 + web/vite.config.ts | 19 + 39 files changed, 7435 insertions(+) create mode 100644 web/index.html create mode 100644 web/package-lock.json create mode 100644 web/package.json create mode 100644 web/postcss.config.js create mode 100644 web/public/vite.svg create mode 100644 web/src/App.tsx create mode 100644 web/src/api/client.ts create mode 100644 web/src/api/types.ts create mode 100644 web/src/components/dashboard/HealthPanel.tsx create mode 100644 web/src/components/dashboard/RecentTasks.tsx create mode 100644 web/src/components/dashboard/StatsCards.tsx create mode 100644 web/src/components/layout/AppLayout.tsx create mode 100644 web/src/components/shared/ConnectionIndicator.tsx create mode 100644 web/src/components/shared/ErrorAlert.tsx create mode 100644 web/src/components/shared/LoadingOverlay.tsx create mode 100644 web/src/components/task/DeleteConfirmDialog.tsx create mode 100644 web/src/components/task/StatusBadge.tsx create mode 100644 web/src/components/task/TaskTypeIcon.tsx create mode 100644 web/src/hooks/usePolling.ts create mode 100644 web/src/hooks/useWebSocket.ts create mode 100644 web/src/index.css create mode 100644 web/src/main.tsx create mode 100644 web/src/pages/AnalyticsPage.tsx create mode 100644 web/src/pages/CreateTaskPage.tsx create mode 100644 web/src/pages/DashboardPage.tsx create mode 100644 web/src/pages/DataPage.tsx create mode 100644 web/src/pages/SettingsPage.tsx create mode 100644 web/src/pages/TaskDetailPage.tsx create mode 100644 web/src/pages/TaskListPage.tsx create mode 100644 web/src/store/useSettingsStore.ts create mode 100644 web/src/store/useTaskStore.ts create mode 100644 web/src/theme/theme.ts create mode 100644 web/src/utils/constants.ts create mode 100644 web/src/utils/format.ts create mode 100644 web/src/vite-env.d.ts create mode 100644 web/tailwind.config.ts create mode 100644 web/tsconfig.json create mode 100644 web/tsconfig.node.json create mode 100644 web/vite.config.ts diff --git a/web/index.html b/web/index.html new file mode 100644 index 000000000..19b87f66b --- /dev/null +++ b/web/index.html @@ -0,0 +1,16 @@ + + + + + + + MediaCrawler Command Center + + + + + +
+ + + diff --git a/web/package-lock.json b/web/package-lock.json new file mode 100644 index 000000000..f33112e2e --- /dev/null +++ b/web/package-lock.json @@ -0,0 +1,3952 @@ +{ + "name": "mediacrawler-webui", + "version": "2.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "mediacrawler-webui", + "version": "2.0.0", + "dependencies": { + "@emotion/react": "^11.13.0", + "@emotion/styled": "^11.13.0", + "@mui/icons-material": "^6.4.0", + "@mui/material": "^6.4.0", + "dayjs": "^1.11.13", + "ky": "^1.7.0", + "papaparse": "^5.4.1", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^7.6.0", + "recharts": "^2.15.0", + "zustand": "^5.0.0" + }, + "devDependencies": { + "@tailwindcss/typography": "^0.5.13", + "@types/papaparse": "^5.3.15", + "@types/react": "^18.3.0", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^4.3.0", + "autoprefixer": "^10.4.19", + "postcss": "^8.4.40", + "tailwindcss": "^3.4.0", + "typescript": "^5.6.0", + "vite": "^6.0.0" + } + }, + "node_modules/@alloc/quick-lru": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", + "integrity": "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/core/node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@babel/generator": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/generator/-/generator-7.29.7.tgz", + "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", + "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.7" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.29.7.tgz", + "integrity": "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.29.7.tgz", + "integrity": "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/traverse/-/traverse-7.29.7.tgz", + "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.7", + "resolved": "https://registry.npmmirror.com/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@emotion/babel-plugin": { + "version": "11.13.5", + "resolved": "https://registry.npmmirror.com/@emotion/babel-plugin/-/babel-plugin-11.13.5.tgz", + "integrity": "sha512-pxHCpT2ex+0q+HH91/zsdHkw/lXd468DIN2zvfvLtPKLLMo6gQj7oLObq8PhkrxOZb/gGCq03S3Z7PDhS8pduQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.16.7", + "@babel/runtime": "^7.18.3", + "@emotion/hash": "^0.9.2", + "@emotion/memoize": "^0.9.0", + "@emotion/serialize": "^1.3.3", + "babel-plugin-macros": "^3.1.0", + "convert-source-map": "^1.5.0", + "escape-string-regexp": "^4.0.0", + "find-root": "^1.1.0", + "source-map": "^0.5.7", + "stylis": "4.2.0" + } + }, + "node_modules/@emotion/cache": { + "version": "11.14.0", + "resolved": "https://registry.npmmirror.com/@emotion/cache/-/cache-11.14.0.tgz", + "integrity": "sha512-L/B1lc/TViYk4DcpGxtAVbx0ZyiKM5ktoIyafGkH6zg/tj+mA+NE//aPYKG0k8kCHSHVJrpLpcAlOBEXQ3SavA==", + "license": "MIT", + "dependencies": { + "@emotion/memoize": "^0.9.0", + "@emotion/sheet": "^1.4.0", + "@emotion/utils": "^1.4.2", + "@emotion/weak-memoize": "^0.4.0", + "stylis": "4.2.0" + } + }, + "node_modules/@emotion/hash": { + "version": "0.9.2", + "resolved": "https://registry.npmmirror.com/@emotion/hash/-/hash-0.9.2.tgz", + "integrity": "sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g==", + "license": "MIT" + }, + "node_modules/@emotion/is-prop-valid": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/@emotion/is-prop-valid/-/is-prop-valid-1.4.0.tgz", + "integrity": "sha512-QgD4fyscGcbbKwJmqNvUMSE02OsHUa+lAWKdEUIJKgqe5IwRSKd7+KhibEWdaKwgjLj0DRSHA9biAIqGBk05lw==", + "license": "MIT", + "dependencies": { + "@emotion/memoize": "^0.9.0" + } + }, + "node_modules/@emotion/memoize": { + "version": "0.9.0", + "resolved": "https://registry.npmmirror.com/@emotion/memoize/-/memoize-0.9.0.tgz", + "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==", + "license": "MIT" + }, + "node_modules/@emotion/react": { + "version": "11.14.0", + "resolved": "https://registry.npmmirror.com/@emotion/react/-/react-11.14.0.tgz", + "integrity": "sha512-O000MLDBDdk/EohJPFUqvnp4qnHeYkVP5B0xEG0D/L7cOKP9kefu2DXn8dj74cQfsEzUqh+sr1RzFqiL1o+PpA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "@emotion/babel-plugin": "^11.13.5", + "@emotion/cache": "^11.14.0", + "@emotion/serialize": "^1.3.3", + "@emotion/use-insertion-effect-with-fallbacks": "^1.2.0", + "@emotion/utils": "^1.4.2", + "@emotion/weak-memoize": "^0.4.0", + "hoist-non-react-statics": "^3.3.1" + }, + "peerDependencies": { + "react": ">=16.8.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@emotion/serialize": { + "version": "1.3.3", + "resolved": "https://registry.npmmirror.com/@emotion/serialize/-/serialize-1.3.3.tgz", + "integrity": "sha512-EISGqt7sSNWHGI76hC7x1CksiXPahbxEOrC5RjmFRJTqLyEK9/9hZvBbiYn70dw4wuwMKiEMCUlR6ZXTSWQqxA==", + "license": "MIT", + "dependencies": { + "@emotion/hash": "^0.9.2", + "@emotion/memoize": "^0.9.0", + "@emotion/unitless": "^0.10.0", + "@emotion/utils": "^1.4.2", + "csstype": "^3.0.2" + } + }, + "node_modules/@emotion/sheet": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/@emotion/sheet/-/sheet-1.4.0.tgz", + "integrity": "sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg==", + "license": "MIT" + }, + "node_modules/@emotion/styled": { + "version": "11.14.1", + "resolved": "https://registry.npmmirror.com/@emotion/styled/-/styled-11.14.1.tgz", + "integrity": "sha512-qEEJt42DuToa3gurlH4Qqc1kVpNq8wO8cJtDzU46TjlzWjDlsVyevtYCRijVq3SrHsROS+gVQ8Fnea108GnKzw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "@emotion/babel-plugin": "^11.13.5", + "@emotion/is-prop-valid": "^1.3.0", + "@emotion/serialize": "^1.3.3", + "@emotion/use-insertion-effect-with-fallbacks": "^1.2.0", + "@emotion/utils": "^1.4.2" + }, + "peerDependencies": { + "@emotion/react": "^11.0.0-rc.0", + "react": ">=16.8.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@emotion/unitless": { + "version": "0.10.0", + "resolved": "https://registry.npmmirror.com/@emotion/unitless/-/unitless-0.10.0.tgz", + "integrity": "sha512-dFoMUuQA20zvtVTuxZww6OHoJYgrzfKM1t52mVySDJnMSEa08ruEvdYQbhvyu6soU+NeLVd3yKfTfT0NeV6qGg==", + "license": "MIT" + }, + "node_modules/@emotion/use-insertion-effect-with-fallbacks": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.2.0.tgz", + "integrity": "sha512-yJMtVdH59sxi/aVJBpk9FQq+OR8ll5GT8oWd57UpeaKEVGab41JWaCFA7FRLoMLloOZF/c/wsPoe+bfGmRKgDg==", + "license": "MIT", + "peerDependencies": { + "react": ">=16.8.0" + } + }, + "node_modules/@emotion/utils": { + "version": "1.4.2", + "resolved": "https://registry.npmmirror.com/@emotion/utils/-/utils-1.4.2.tgz", + "integrity": "sha512-3vLclRofFziIa3J2wDh9jjbkUz9qk5Vi3IZ/FSTKViB0k+ef0fPV7dYrUIugbgupYDx7v9ud/SjrtEP8Y4xLoA==", + "license": "MIT" + }, + "node_modules/@emotion/weak-memoize": { + "version": "0.4.0", + "resolved": "https://registry.npmmirror.com/@emotion/weak-memoize/-/weak-memoize-0.4.0.tgz", + "integrity": "sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg==", + "license": "MIT" + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmmirror.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmmirror.com/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmmirror.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmmirror.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@mui/core-downloads-tracker": { + "version": "6.5.0", + "resolved": "https://registry.npmmirror.com/@mui/core-downloads-tracker/-/core-downloads-tracker-6.5.0.tgz", + "integrity": "sha512-LGb8t8i6M2ZtS3Drn3GbTI1DVhDY6FJ9crEey2lZ0aN2EMZo8IZBZj9wRf4vqbZHaWjsYgtbOnJw5V8UWbmK2Q==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + } + }, + "node_modules/@mui/icons-material": { + "version": "6.5.0", + "resolved": "https://registry.npmmirror.com/@mui/icons-material/-/icons-material-6.5.0.tgz", + "integrity": "sha512-VPuPqXqbBPlcVSA0BmnoE4knW4/xG6Thazo8vCLWkOKusko6DtwFV6B665MMWJ9j0KFohTIf3yx2zYtYacvG1g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@mui/material": "^6.5.0", + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/material": { + "version": "6.5.0", + "resolved": "https://registry.npmmirror.com/@mui/material/-/material-6.5.0.tgz", + "integrity": "sha512-yjvtXoFcrPLGtgKRxFaH6OQPtcLPhkloC0BML6rBG5UeldR0nPULR/2E2BfXdo5JNV7j7lOzrrLX2Qf/iSidow==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/core-downloads-tracker": "^6.5.0", + "@mui/system": "^6.5.0", + "@mui/types": "~7.2.24", + "@mui/utils": "^6.4.9", + "@popperjs/core": "^2.11.8", + "@types/react-transition-group": "^4.4.12", + "clsx": "^2.1.1", + "csstype": "^3.1.3", + "prop-types": "^15.8.1", + "react-is": "^19.0.0", + "react-transition-group": "^4.4.5" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.5.0", + "@emotion/styled": "^11.3.0", + "@mui/material-pigment-css": "^6.5.0", + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + }, + "@mui/material-pigment-css": { + "optional": true + }, + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/private-theming": { + "version": "6.4.9", + "resolved": "https://registry.npmmirror.com/@mui/private-theming/-/private-theming-6.4.9.tgz", + "integrity": "sha512-LktcVmI5X17/Q5SkwjCcdOLBzt1hXuc14jYa7NPShog0GBDCDvKtcnP0V7a2s6EiVRlv7BzbWEJzH6+l/zaCxw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/utils": "^6.4.9", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/styled-engine": { + "version": "6.5.0", + "resolved": "https://registry.npmmirror.com/@mui/styled-engine/-/styled-engine-6.5.0.tgz", + "integrity": "sha512-8woC2zAqF4qUDSPIBZ8v3sakj+WgweolpyM/FXf8jAx6FMls+IE4Y8VDZc+zS805J7PRz31vz73n2SovKGaYgw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@emotion/cache": "^11.13.5", + "@emotion/serialize": "^1.3.3", + "@emotion/sheet": "^1.4.0", + "csstype": "^3.1.3", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.4.1", + "@emotion/styled": "^11.3.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + } + } + }, + "node_modules/@mui/system": { + "version": "6.5.0", + "resolved": "https://registry.npmmirror.com/@mui/system/-/system-6.5.0.tgz", + "integrity": "sha512-XcbBYxDS+h/lgsoGe78ExXFZXtuIlSBpn/KsZq8PtZcIkUNJInkuDqcLd2rVBQrDC1u+rvVovdaWPf2FHKJf3w==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/private-theming": "^6.4.9", + "@mui/styled-engine": "^6.5.0", + "@mui/types": "~7.2.24", + "@mui/utils": "^6.4.9", + "clsx": "^2.1.1", + "csstype": "^3.1.3", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.5.0", + "@emotion/styled": "^11.3.0", + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + }, + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/types": { + "version": "7.2.24", + "resolved": "https://registry.npmmirror.com/@mui/types/-/types-7.2.24.tgz", + "integrity": "sha512-3c8tRt/CbWZ+pEg7QpSwbdxOk36EfmhbKf6AGZsD1EcLDLTSZoxxJ86FVtcjxvjuhdyBiWKSTGZFaXCnidO2kw==", + "license": "MIT", + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/utils": { + "version": "6.4.9", + "resolved": "https://registry.npmmirror.com/@mui/utils/-/utils-6.4.9.tgz", + "integrity": "sha512-Y12Q9hbK9g+ZY0T3Rxrx9m2m10gaphDuUMgWxyV5kNJevVxXYCLclYUCC9vXaIk1/NdNDTcW2Yfr2OGvNFNmHg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/types": "~7.2.24", + "@types/prop-types": "^15.7.14", + "clsx": "^2.1.1", + "prop-types": "^15.8.1", + "react-is": "^19.0.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmmirror.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmmirror.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmmirror.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@popperjs/core": { + "version": "2.11.8", + "resolved": "https://registry.npmmirror.com/@popperjs/core/-/core-2.11.8.tgz", + "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/popperjs" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.27", + "resolved": "https://registry.npmmirror.com/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", + "integrity": "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.61.1.tgz", + "integrity": "sha512-JnBB8MdXj45cajvTuO5FmPlvFVJRQgvrz1uSEl3NwqFnReAPGwb8EanbGi4z2nRaqLzjJSv5/JmycoTKlRZxHA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.61.1.tgz", + "integrity": "sha512-Jx2g7iSjw4AOT0HDPHM9RV3GNjRXwybWtSFZiZAYUTjUwjVrYIwq3kBf+LnhqJlzXFAqTAh2F7IGI+O568exPw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.61.1.tgz", + "integrity": "sha512-0F1L/Z3Eqv8mT2n3dCpeO8GcTvHvVqkP5/t6DMsn0KzhYVcg+s7Ncl5DS8qjKYEeio6Az0Gt6nyBORay5qIlCA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.61.1.tgz", + "integrity": "sha512-qLttcH871ujY4YcVfUSShhOw+CsoTatYz8gRbHO7Bb92QH059/P0y5do1KMs41fY0BpD2x4AJH/gID0zFiqVKQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.61.1.tgz", + "integrity": "sha512-fUI4RapGE0Oh3mb8mgfvC1O2nU1RpDZUKnDQm3xB1Ipg7C2wTs5Kstz7G2uWK99a8S2yTMq8/P4uycwNa0nJyw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.61.1.tgz", + "integrity": "sha512-H5YrdvJaDtI/U9/emrD4b++xkvp3y/JvOe4rizHbxvkyMfRS/CiRYdji+Pl8D0brEaNFWUh1drQxgAGIl6Xudw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.61.1.tgz", + "integrity": "sha512-Q8CBCCQtDFrYtXoeUXSrnFXKOnyUhx6bz+SkL6A0E7V8kAiCJ5pamq1WtbfpVGhR5TSpXY6ak3avmDc5fHTyJA==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.61.1.tgz", + "integrity": "sha512-nwnhk1581l0FBVellGcVCAT0Oi06onEA3WB53sf01VO3I0UPBkMH9sXONYME2K0ovXcNayJfNtHfm6mpJElatQ==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.61.1.tgz", + "integrity": "sha512-x5Xr49hwt3hdW75UOZm3395YwwzPyauktslv29KpWL/T+vVAzoT3azLcTWv0eMciBNrx+DYjH4paehHoLpPvpg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.61.1.tgz", + "integrity": "sha512-unMS3H73DpaoPyyEVPjGKleM/s0mkmsauTENpw4INQY8y4+IuLNjkueQ5QCtC0D3N38Y38yhAU8OoZ20S2Tm6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.61.1.tgz", + "integrity": "sha512-zNZzGRnAhwjFEYmvphJRV5XaQGjs62cCmeYYHUT//NbvEnHauw+I85nGG+SiVg5ld4GX8D1IbKIX+ozITQnhMQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.61.1.tgz", + "integrity": "sha512-LdpWGL8X209B2SIvWjqlc8VZgM6PKfontSerGepuldQmHYrAOtnMCXeJkxXGbC+PPZVOuu5czJo7fNV6aeW8rQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.61.1.tgz", + "integrity": "sha512-EC5kTtNaNGOmbMGqar8dvJy6y/hg99GAwjfBz++pxZhQATXGcRjd6c5en5wcbru0vkRmiMGsQKdMJOOf6sza4g==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.61.1.tgz", + "integrity": "sha512-8hiwp6D4acEcNK78I4rP0/XtS1sknWIAMJBPdR4l6zUtyTm5KiTDr5bXmWt4foY7nAN7AThDHgkLIEZOWKbzWw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.61.1.tgz", + "integrity": "sha512-10dh/h/BqA7DuMPWSxkR8uks18FRwnwOEqr5zOTEl+NOwP/OMzKX8OFR/Of9xxDA7D5qef1Nzar5WDD2kCCr1g==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.61.1.tgz", + "integrity": "sha512-YKJ5lg35DP17gcAOggnihe+APw9HLyj1Xn7gsmGumBJAUDa6NGXNixJzmkWLhcK9TOuuyQjdamzvJefkO7qHZQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.61.1.tgz", + "integrity": "sha512-Mlil5G2Jj6a7B3LWGctg+XPL9vdXYuzCtNXfxOQ0nPjc2m6ueUktocPGH9bnAM0bNRKb/bAWTujUU7IJQdQA+g==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.61.1.tgz", + "integrity": "sha512-bVWIOIk6pV01p4CdUbPP7CJ/434z+OooYjDuFcR+44N35YvKUC66G8MGnvcWx5mWKW3g61J+t74l3Kj15Kwn2Q==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.61.1.tgz", + "integrity": "sha512-qy5pBvZbqNFheBz61R1rzsezjm0J7O2oNGoWtGoY89SZYLUfxAJTBAqDChqAIdB4rCiIbi9nF7yZ83GnNiLwSw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.61.1.tgz", + "integrity": "sha512-E83TXjI4zm0+5f2qO+UOudaCYIhYwpJ5jq6YCZNIZ+6CbfhKrkAGezeiASBL9ElxAxFsRS9ZhESv8mfnj6TKeg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.61.1.tgz", + "integrity": "sha512-fbWnKqVkjrJN38vNe3ahkbk6iejS/3b0Nt7EEtPpE6RBacZcGXNKbzfHN3GUUlXOPghUg0j6XUGrtjX9z1sIvA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.61.1.tgz", + "integrity": "sha512-ArMl38iVAbk0New1ogihQNY6iphLi4ZaRsa037gUzv5yeKPY8TD3Dmy4x2RNC1VztU/uqm+G+/RwFrSka3Oy2g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.61.1.tgz", + "integrity": "sha512-0mYtjHS9ucAbcATycCNK9IGBk/cCe/ma7EmSLGZdsxnOA8cjRIyU04wDpVAD9NiOfLUR9KTxdiO53uOkherqjQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.61.1.tgz", + "integrity": "sha512-gK1iCEPfpoSG9wfBihXxvBMi8ZfcWffYkEsC/Eih+iFENTaewvNcrEQ69lIOWYO5pePHKLHHO7nq5AILGO/HQQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.61.1.tgz", + "integrity": "sha512-X+zaP2x+j4RXGfbp/seSoRHWnPxzApilDszisZxbYH5C/jTxFhCtDNdPGZb9lJyYPs24wGxruPF7Y+sIXt9Gzw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@tailwindcss/typography": { + "version": "0.5.19", + "resolved": "https://registry.npmmirror.com/@tailwindcss/typography/-/typography-0.5.19.tgz", + "integrity": "sha512-w31dd8HOx3k9vPtcQh5QHP9GwKcgbMp87j58qi6xgiBnFFtKEAgCWnDw4qUT8aHwkCp8bKvb/KGKWWHedP0AAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "postcss-selector-parser": "6.0.10" + }, + "peerDependencies": { + "tailwindcss": ">=3.0.0 || insiders || >=4.0.0-alpha.20 || >=4.0.0-beta.1" + } + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmmirror.com/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmmirror.com/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmmirror.com/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmmirror.com/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/d3-array": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/@types/d3-array/-/d3-array-3.2.2.tgz", + "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", + "license": "MIT" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "license": "MIT" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "license": "MIT" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "license": "MIT", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/@types/d3-path/-/d3-path-3.1.1.tgz", + "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", + "license": "MIT" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.9", + "resolved": "https://registry.npmmirror.com/@types/d3-scale/-/d3-scale-4.0.9.tgz", + "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", + "license": "MIT", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-shape": { + "version": "3.1.8", + "resolved": "https://registry.npmmirror.com/@types/d3-shape/-/d3-shape-3.1.8.tgz", + "integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==", + "license": "MIT", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/@types/d3-time/-/d3-time-3.0.4.tgz", + "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", + "license": "MIT" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmmirror.com/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "25.9.1", + "resolved": "https://registry.npmmirror.com/@types/node/-/node-25.9.1.tgz", + "integrity": "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/@types/papaparse": { + "version": "5.5.2", + "resolved": "https://registry.npmmirror.com/@types/papaparse/-/papaparse-5.5.2.tgz", + "integrity": "sha512-gFnFp/JMzLHCwRf7tQHrNnfhN4eYBVYYI897CGX4MY1tzY9l2aLkVyx2IlKZ/SAqDbB3I1AOZW5gTMGGsqWliA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/parse-json": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/@types/parse-json/-/parse-json-4.0.2.tgz", + "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==", + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmmirror.com/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.30", + "resolved": "https://registry.npmmirror.com/@types/react/-/react-18.3.30.tgz", + "integrity": "sha512-3ek6mwJL5/VBewBcY4S66cqlCtK3qi4WIq37Z0m/NHw1hjhI7274Mx1qz/+ggSzyBCOEf7eHjBN6INjPAWYfYw==", + "license": "MIT", + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "resolved": "https://registry.npmmirror.com/@types/react-dom/-/react-dom-18.3.7.tgz", + "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, + "node_modules/@types/react-transition-group": { + "version": "4.4.12", + "resolved": "https://registry.npmmirror.com/@types/react-transition-group/-/react-transition-group-4.4.12.tgz", + "integrity": "sha512-8TV6R3h2j7a91c+1DXdJi3Syo69zzIZbz7Lg5tORM5LEJG7X/E6a1V3drRyBRZq7/utz7A+c4OgYLiLcYGHG6w==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.7.0", + "resolved": "https://registry.npmmirror.com/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz", + "integrity": "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.28.0", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.27", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.17.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", + "dev": true, + "license": "MIT" + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmmirror.com/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "dev": true, + "license": "MIT" + }, + "node_modules/autoprefixer": { + "version": "10.5.0", + "resolved": "https://registry.npmmirror.com/autoprefixer/-/autoprefixer-10.5.0.tgz", + "integrity": "sha512-FMhOoZV4+qR6aTUALKX2rEqGG+oyATvwBt9IIzVR5rMa2HRWPkxf+P+PAJLD1I/H5/II+HuZcBJYEFBpq39ong==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/autoprefixer" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "browserslist": "^4.28.2", + "caniuse-lite": "^1.0.30001787", + "fraction.js": "^5.3.4", + "picocolors": "^1.1.1", + "postcss-value-parser": "^4.2.0" + }, + "bin": { + "autoprefixer": "bin/autoprefixer" + }, + "engines": { + "node": "^10 || ^12 || >=14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/babel-plugin-macros": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/babel-plugin-macros/-/babel-plugin-macros-3.1.0.tgz", + "integrity": "sha512-Cg7TFGpIr01vOQNODXOOaGz2NpCU5gl8x1qJFbb6hbZxR7XrcE2vtbAsTAbJ7/xwJtUuJEw8K8Zr/AE0LHlesg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.12.5", + "cosmiconfig": "^7.0.0", + "resolve": "^1.19.0" + }, + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, + "node_modules/baseline-browser-mapping": { + "version": "2.10.33", + "resolved": "https://registry.npmmirror.com/baseline-browser-mapping/-/baseline-browser-mapping-2.10.33.tgz", + "integrity": "sha512-bA6+tcSLpz2tIEdDXZPpPTIuxBcC4+w6SieaYyfigIa4h8GlFxbA17v22Vx3JUtuZQj9SgOsnbK+aTBzyDyEuw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.28.2", + "resolved": "https://registry.npmmirror.com/browserslist/-/browserslist-4.28.2.tgz", + "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.10.12", + "caniuse-lite": "^1.0.30001782", + "electron-to-chromium": "^1.5.328", + "node-releases": "^2.0.36", + "update-browserslist-db": "^1.2.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase-css": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/camelcase-css/-/camelcase-css-2.0.1.tgz", + "integrity": "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001793", + "resolved": "https://registry.npmmirror.com/caniuse-lite/-/caniuse-lite-1.0.30001793.tgz", + "integrity": "sha512-iwSsYWaCOoh26cV8NwNRViHlrfUvYsHDfRVcbtmw0Kg6PJIZZXwMkj1442FYLBGkeUf1juAsU3DTfxW579mrPA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/convert-source-map": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/convert-source-map/-/convert-source-map-1.9.0.tgz", + "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==", + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cosmiconfig": { + "version": "7.1.0", + "resolved": "https://registry.npmmirror.com/cosmiconfig/-/cosmiconfig-7.1.0.tgz", + "integrity": "sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==", + "license": "MIT", + "dependencies": { + "@types/parse-json": "^4.0.0", + "import-fresh": "^3.2.1", + "parse-json": "^5.0.0", + "path-type": "^4.0.0", + "yaml": "^1.10.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/cosmiconfig/node_modules/yaml": { + "version": "1.10.3", + "resolved": "https://registry.npmmirror.com/yaml/-/yaml-1.10.3.tgz", + "integrity": "sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==", + "license": "ISC", + "engines": { + "node": ">= 6" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true, + "license": "MIT", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmmirror.com/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "license": "MIT" + }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmmirror.com/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/d3-format/-/d3-format-3.1.2.tgz", + "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/dayjs": { + "version": "1.11.21", + "resolved": "https://registry.npmmirror.com/dayjs/-/dayjs-1.11.21.tgz", + "integrity": "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decimal.js-light": { + "version": "2.5.1", + "resolved": "https://registry.npmmirror.com/decimal.js-light/-/decimal.js-light-2.5.1.tgz", + "integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==", + "license": "MIT" + }, + "node_modules/didyoumean": { + "version": "1.2.2", + "resolved": "https://registry.npmmirror.com/didyoumean/-/didyoumean-1.2.2.tgz", + "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/dlv": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/dlv/-/dlv-1.1.3.tgz", + "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", + "dev": true, + "license": "MIT" + }, + "node_modules/dom-helpers": { + "version": "5.2.1", + "resolved": "https://registry.npmmirror.com/dom-helpers/-/dom-helpers-5.2.1.tgz", + "integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.8.7", + "csstype": "^3.0.2" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.368", + "resolved": "https://registry.npmmirror.com/electron-to-chromium/-/electron-to-chromium-1.5.368.tgz", + "integrity": "sha512-7RckJJK4uESJF9PxvfMWd3TGqIiieUTG4HxnKaKuIpGbcr+r2ZEB3g2gAhCP3Fqm42vJSzLfgab9eva/C4/XVw==", + "dev": true, + "license": "ISC" + }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmmirror.com/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmmirror.com/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmmirror.com/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/fast-equals": { + "version": "5.4.0", + "resolved": "https://registry.npmmirror.com/fast-equals/-/fast-equals-5.4.0.tgz", + "integrity": "sha512-jt2DW/aNFNwke7AUd+Z+e6pz39KO5rzdbbFCg2sGafS4mk13MI7Z8O5z9cADNn5lhGODIgLwug6TZO2ctf7kcw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmmirror.com/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-glob/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmmirror.com/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "dev": true, + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmmirror.com/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-root": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/find-root/-/find-root-1.1.0.tgz", + "integrity": "sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==", + "license": "MIT" + }, + "node_modules/fraction.js": { + "version": "5.3.4", + "resolved": "https://registry.npmmirror.com/fraction.js/-/fraction.js-5.3.4.tgz", + "integrity": "sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/rawify" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmmirror.com/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmmirror.com/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmmirror.com/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmmirror.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "license": "BSD-3-Clause", + "dependencies": { + "react-is": "^16.7.0" + } + }, + "node_modules/hoist-non-react-statics/node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmmirror.com/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmmirror.com/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-core-module": { + "version": "2.16.2", + "resolved": "https://registry.npmmirror.com/is-core-module/-/is-core-module-2.16.2.tgz", + "integrity": "sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==", + "license": "MIT", + "dependencies": { + "hasown": "^2.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/jiti": { + "version": "1.21.7", + "resolved": "https://registry.npmmirror.com/jiti/-/jiti-1.21.7.tgz", + "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "bin/jiti.js" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmmirror.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmmirror.com/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/ky": { + "version": "1.14.3", + "resolved": "https://registry.npmmirror.com/ky/-/ky-1.14.3.tgz", + "integrity": "sha512-9zy9lkjac+TR1c2tG+mkNSVlyOpInnWdSMiue4F+kq8TwJSgv6o8jhLRg8Ho6SnZ9wOYUq/yozts9qQCfk7bIw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/ky?sponsor=1" + } + }, + "node_modules/lilconfig": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/lilconfig/-/lilconfig-3.1.3.tgz", + "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmmirror.com/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmmirror.com/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/mz": { + "version": "2.7.0", + "resolved": "https://registry.npmmirror.com/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.12", + "resolved": "https://registry.npmmirror.com/nanoid/-/nanoid-3.3.12.tgz", + "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.47", + "resolved": "https://registry.npmmirror.com/node-releases/-/node-releases-2.0.47.tgz", + "integrity": "sha512-Uzmd6LXpouKo8EUK68IjH4+E01w/hXyV3R3g/geCJo+rXLNfh1xucB+LOzYEOQPSiUK3h/xZf0cQGcSsmyL2Og==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/papaparse": { + "version": "5.5.3", + "resolved": "https://registry.npmmirror.com/papaparse/-/papaparse-5.5.3.tgz", + "integrity": "sha512-5QvjGxYVjxO59MGU2lHVYpRWBBtKHnlIAcSe1uNFCkkptUh63NFRj0FJQm7nR67puEruUci/ZkjmEFrjCAyP4A==", + "license": "MIT" + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmmirror.com/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "license": "MIT" + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmmirror.com/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pirates": { + "version": "4.0.7", + "resolved": "https://registry.npmmirror.com/pirates/-/pirates-4.0.7.tgz", + "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/postcss": { + "version": "8.5.15", + "resolved": "https://registry.npmmirror.com/postcss/-/postcss-8.5.15.tgz", + "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-import": { + "version": "15.1.0", + "resolved": "https://registry.npmmirror.com/postcss-import/-/postcss-import-15.1.0.tgz", + "integrity": "sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==", + "dev": true, + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.0.0", + "read-cache": "^1.0.0", + "resolve": "^1.1.7" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "postcss": "^8.0.0" + } + }, + "node_modules/postcss-js": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/postcss-js/-/postcss-js-4.1.0.tgz", + "integrity": "sha512-oIAOTqgIo7q2EOwbhb8UalYePMvYoIeRY2YKntdpFQXNosSu3vLrniGgmH9OKs/qAkfoj5oB3le/7mINW1LCfw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "camelcase-css": "^2.0.1" + }, + "engines": { + "node": "^12 || ^14 || >= 16" + }, + "peerDependencies": { + "postcss": "^8.4.21" + } + }, + "node_modules/postcss-nested": { + "version": "6.2.0", + "resolved": "https://registry.npmmirror.com/postcss-nested/-/postcss-nested-6.2.0.tgz", + "integrity": "sha512-HQbt28KulC5AJzG+cZtj9kvKB93CFCdLvog1WFLf1D+xmMvPGlBstkpTEZfK5+AN9hfJocyBFCNiqyS48bpgzQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "postcss-selector-parser": "^6.1.1" + }, + "engines": { + "node": ">=12.0" + }, + "peerDependencies": { + "postcss": "^8.2.14" + } + }, + "node_modules/postcss-nested/node_modules/postcss-selector-parser": { + "version": "6.1.2", + "resolved": "https://registry.npmmirror.com/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", + "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-selector-parser": { + "version": "6.0.10", + "resolved": "https://registry.npmmirror.com/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz", + "integrity": "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmmirror.com/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "node_modules/prop-types/node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmmirror.com/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmmirror.com/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmmirror.com/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmmirror.com/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/react-is": { + "version": "19.2.7", + "resolved": "https://registry.npmmirror.com/react-is/-/react-is-19.2.7.tgz", + "integrity": "sha512-kZFnouyVv7eP/Phmrlo9FK+zcAdriZJvzxXHF1Sl1P377WSGe2G/JxVolhTrB/jeV47lKImhNUsijjHAAbcl/A==", + "license": "MIT" + }, + "node_modules/react-refresh": { + "version": "0.17.0", + "resolved": "https://registry.npmmirror.com/react-refresh/-/react-refresh-0.17.0.tgz", + "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-router": { + "version": "7.17.0", + "resolved": "https://registry.npmmirror.com/react-router/-/react-router-7.17.0.tgz", + "integrity": "sha512-FDELK7rTMlCHO5+reyXsPlmfr7N1F91lPHsWYfMEGQm/KQ+F4JFM8jGoeQDmDvdTs93Fw9aSilH+uKRb4/jXvQ==", + "license": "MIT", + "dependencies": { + "cookie": "^1.0.1", + "set-cookie-parser": "^2.6.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + } + } + }, + "node_modules/react-router-dom": { + "version": "7.17.0", + "resolved": "https://registry.npmmirror.com/react-router-dom/-/react-router-dom-7.17.0.tgz", + "integrity": "sha512-fyU2yjGups/hE6Xz0I5ZYbVL8Gx29eCjgpHaRaTaVU+OOAdfRX05KsvyRm0GO8YQwOkhpU3MurW1jyMUJn+zSw==", + "license": "MIT", + "dependencies": { + "react-router": "7.17.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + } + }, + "node_modules/react-smooth": { + "version": "4.0.4", + "resolved": "https://registry.npmmirror.com/react-smooth/-/react-smooth-4.0.4.tgz", + "integrity": "sha512-gnGKTpYwqL0Iii09gHobNolvX4Kiq4PKx6eWBCYYix+8cdw+cGo3do906l1NBPKkSWx1DghC1dlWG9L2uGd61Q==", + "license": "MIT", + "dependencies": { + "fast-equals": "^5.0.1", + "prop-types": "^15.8.1", + "react-transition-group": "^4.4.5" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/react-transition-group": { + "version": "4.4.5", + "resolved": "https://registry.npmmirror.com/react-transition-group/-/react-transition-group-4.4.5.tgz", + "integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==", + "license": "BSD-3-Clause", + "dependencies": { + "@babel/runtime": "^7.5.5", + "dom-helpers": "^5.0.1", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2" + }, + "peerDependencies": { + "react": ">=16.6.0", + "react-dom": ">=16.6.0" + } + }, + "node_modules/read-cache": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/read-cache/-/read-cache-1.0.0.tgz", + "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pify": "^2.3.0" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/recharts": { + "version": "2.15.4", + "resolved": "https://registry.npmmirror.com/recharts/-/recharts-2.15.4.tgz", + "integrity": "sha512-UT/q6fwS3c1dHbXv2uFgYJ9BMFHu3fwnd7AYZaEQhXuYQ4hgsxLvsUXzGdKeZrW5xopzDCvuA2N41WJ88I7zIw==", + "license": "MIT", + "dependencies": { + "clsx": "^2.0.0", + "eventemitter3": "^4.0.1", + "lodash": "^4.17.21", + "react-is": "^18.3.1", + "react-smooth": "^4.0.4", + "recharts-scale": "^0.4.4", + "tiny-invariant": "^1.3.1", + "victory-vendor": "^36.6.8" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "react": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/recharts-scale": { + "version": "0.4.5", + "resolved": "https://registry.npmmirror.com/recharts-scale/-/recharts-scale-0.4.5.tgz", + "integrity": "sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==", + "license": "MIT", + "dependencies": { + "decimal.js-light": "^2.4.1" + } + }, + "node_modules/recharts/node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmmirror.com/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "license": "MIT" + }, + "node_modules/resolve": { + "version": "1.22.12", + "resolved": "https://registry.npmmirror.com/resolve/-/resolve-1.22.12.tgz", + "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "is-core-module": "^2.16.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.61.1", + "resolved": "https://registry.npmmirror.com/rollup/-/rollup-4.61.1.tgz", + "integrity": "sha512-I4KW6iuRpuu2uHBLraZ1wNZe0DP7lnRha+VJ9tNaYVaVgKhW0aI3h4RYnoRPeql0flHm/Co55b7snEDcOfOJrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.61.1", + "@rollup/rollup-android-arm64": "4.61.1", + "@rollup/rollup-darwin-arm64": "4.61.1", + "@rollup/rollup-darwin-x64": "4.61.1", + "@rollup/rollup-freebsd-arm64": "4.61.1", + "@rollup/rollup-freebsd-x64": "4.61.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.61.1", + "@rollup/rollup-linux-arm-musleabihf": "4.61.1", + "@rollup/rollup-linux-arm64-gnu": "4.61.1", + "@rollup/rollup-linux-arm64-musl": "4.61.1", + "@rollup/rollup-linux-loong64-gnu": "4.61.1", + "@rollup/rollup-linux-loong64-musl": "4.61.1", + "@rollup/rollup-linux-ppc64-gnu": "4.61.1", + "@rollup/rollup-linux-ppc64-musl": "4.61.1", + "@rollup/rollup-linux-riscv64-gnu": "4.61.1", + "@rollup/rollup-linux-riscv64-musl": "4.61.1", + "@rollup/rollup-linux-s390x-gnu": "4.61.1", + "@rollup/rollup-linux-x64-gnu": "4.61.1", + "@rollup/rollup-linux-x64-musl": "4.61.1", + "@rollup/rollup-openbsd-x64": "4.61.1", + "@rollup/rollup-openharmony-arm64": "4.61.1", + "@rollup/rollup-win32-arm64-msvc": "4.61.1", + "@rollup/rollup-win32-ia32-msvc": "4.61.1", + "@rollup/rollup-win32-x64-gnu": "4.61.1", + "@rollup/rollup-win32-x64-msvc": "4.61.1", + "fsevents": "~2.3.2" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmmirror.com/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmmirror.com/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/set-cookie-parser": { + "version": "2.7.2", + "resolved": "https://registry.npmmirror.com/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", + "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", + "license": "MIT" + }, + "node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stylis": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/stylis/-/stylis-4.2.0.tgz", + "integrity": "sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw==", + "license": "MIT" + }, + "node_modules/sucrase": { + "version": "3.35.1", + "resolved": "https://registry.npmmirror.com/sucrase/-/sucrase-3.35.1.tgz", + "integrity": "sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.2", + "commander": "^4.0.0", + "lines-and-columns": "^1.1.6", + "mz": "^2.7.0", + "pirates": "^4.0.1", + "tinyglobby": "^0.2.11", + "ts-interface-checker": "^0.1.9" + }, + "bin": { + "sucrase": "bin/sucrase", + "sucrase-node": "bin/sucrase-node" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/tailwindcss": { + "version": "3.4.19", + "resolved": "https://registry.npmmirror.com/tailwindcss/-/tailwindcss-3.4.19.tgz", + "integrity": "sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@alloc/quick-lru": "^5.2.0", + "arg": "^5.0.2", + "chokidar": "^3.6.0", + "didyoumean": "^1.2.2", + "dlv": "^1.1.3", + "fast-glob": "^3.3.2", + "glob-parent": "^6.0.2", + "is-glob": "^4.0.3", + "jiti": "^1.21.7", + "lilconfig": "^3.1.3", + "micromatch": "^4.0.8", + "normalize-path": "^3.0.0", + "object-hash": "^3.0.0", + "picocolors": "^1.1.1", + "postcss": "^8.4.47", + "postcss-import": "^15.1.0", + "postcss-js": "^4.0.1", + "postcss-load-config": "^4.0.2 || ^5.0 || ^6.0", + "postcss-nested": "^6.2.0", + "postcss-selector-parser": "^6.1.2", + "resolve": "^1.22.8", + "sucrase": "^3.35.0" + }, + "bin": { + "tailwind": "lib/cli.js", + "tailwindcss": "lib/cli.js" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tailwindcss/node_modules/postcss-load-config": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/postcss-load-config/-/postcss-load-config-6.0.1.tgz", + "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "lilconfig": "^3.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "jiti": ">=1.21.0", + "postcss": ">=8.0.9", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + }, + "postcss": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/tailwindcss/node_modules/postcss-selector-parser": { + "version": "6.1.2", + "resolved": "https://registry.npmmirror.com/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", + "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "any-promise": "^1.0.0" + } + }, + "node_modules/thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "thenify": ">= 3.1.0 < 4" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/tiny-invariant": { + "version": "1.3.3", + "resolved": "https://registry.npmmirror.com/tiny-invariant/-/tiny-invariant-1.3.3.tgz", + "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmmirror.com/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmmirror.com/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmmirror.com/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/ts-interface-checker": { + "version": "0.1.13", + "resolved": "https://registry.npmmirror.com/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", + "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmmirror.com/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmmirror.com/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/victory-vendor": { + "version": "36.9.2", + "resolved": "https://registry.npmmirror.com/victory-vendor/-/victory-vendor-36.9.2.tgz", + "integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==", + "license": "MIT AND ISC", + "dependencies": { + "@types/d3-array": "^3.0.3", + "@types/d3-ease": "^3.0.0", + "@types/d3-interpolate": "^3.0.1", + "@types/d3-scale": "^4.0.2", + "@types/d3-shape": "^3.1.0", + "@types/d3-time": "^3.0.0", + "@types/d3-timer": "^3.0.0", + "d3-array": "^3.1.6", + "d3-ease": "^3.0.1", + "d3-interpolate": "^3.0.1", + "d3-scale": "^4.0.2", + "d3-shape": "^3.1.0", + "d3-time": "^3.0.0", + "d3-timer": "^3.0.1" + } + }, + "node_modules/vite": { + "version": "6.4.3", + "resolved": "https://registry.npmmirror.com/vite/-/vite-6.4.3.tgz", + "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmmirror.com/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmmirror.com/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmmirror.com/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "extraneous": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/zustand": { + "version": "5.0.14", + "resolved": "https://registry.npmmirror.com/zustand/-/zustand-5.0.14.tgz", + "integrity": "sha512-/8tAspM5LMPr28b3fwLYrtdj77ECpfZviaP75CMTnwO8ISyaE4GDIG/9rDDYq/cH9D2Xw2A2RXglLInmVBQB/g==", + "license": "MIT", + "engines": { + "node": ">=12.20.0" + }, + "peerDependencies": { + "@types/react": ">=18.0.0", + "immer": ">=9.0.6", + "react": ">=18.0.0", + "use-sync-external-store": ">=1.2.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + }, + "use-sync-external-store": { + "optional": true + } + } + } + } +} diff --git a/web/package.json b/web/package.json new file mode 100644 index 000000000..57283a0a9 --- /dev/null +++ b/web/package.json @@ -0,0 +1,37 @@ +{ + "name": "mediacrawler-webui", + "private": true, + "version": "2.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^7.6.0", + "@mui/material": "^6.4.0", + "@mui/icons-material": "^6.4.0", + "@emotion/react": "^11.13.0", + "@emotion/styled": "^11.13.0", + "ky": "^1.7.0", + "zustand": "^5.0.0", + "recharts": "^2.15.0", + "papaparse": "^5.4.1", + "dayjs": "^1.11.13" + }, + "devDependencies": { + "@types/react": "^18.3.0", + "@types/react-dom": "^18.3.0", + "@types/papaparse": "^5.3.15", + "@vitejs/plugin-react": "^4.3.0", + "vite": "^6.0.0", + "typescript": "^5.6.0", + "tailwindcss": "^3.4.0", + "postcss": "^8.4.40", + "autoprefixer": "^10.4.19", + "@tailwindcss/typography": "^0.5.13" + } +} diff --git a/web/postcss.config.js b/web/postcss.config.js new file mode 100644 index 000000000..2aa7205d4 --- /dev/null +++ b/web/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +}; diff --git a/web/public/vite.svg b/web/public/vite.svg new file mode 100644 index 000000000..23bbd51de --- /dev/null +++ b/web/public/vite.svg @@ -0,0 +1,4 @@ + + + MC + diff --git a/web/src/App.tsx b/web/src/App.tsx new file mode 100644 index 000000000..72ba13b07 --- /dev/null +++ b/web/src/App.tsx @@ -0,0 +1,27 @@ +import React from 'react'; +import { Routes, Route, Navigate } from 'react-router-dom'; +import AppLayout from './components/layout/AppLayout'; +import DashboardPage from './pages/DashboardPage'; +import CreateTaskPage from './pages/CreateTaskPage'; +import TaskListPage from './pages/TaskListPage'; +import TaskDetailPage from './pages/TaskDetailPage'; +import SettingsPage from './pages/SettingsPage'; +import AnalyticsPage from './pages/AnalyticsPage'; +import DataPage from './pages/DataPage'; + +export default function App() { + return ( + + }> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + + ); +} diff --git a/web/src/api/client.ts b/web/src/api/client.ts new file mode 100644 index 000000000..b610949a7 --- /dev/null +++ b/web/src/api/client.ts @@ -0,0 +1,109 @@ +import ky, { type KyInstance } from 'ky'; +import { + DEFAULT_API_BASE, API_KEY_HEADER, API_KEY_STORAGE_KEY, + API_BASE_URL_STORAGE_KEY, +} from '../utils/constants'; +import type { + TaskListResponse, TaskInfo, CreateTaskResponse, HealthResponse, + SearchParams, CommentsParams, ScriptsParams, MergeParams, RunAllParams, + DataListResponse, DataPreviewResponse, ExportRequest, +} from './types'; + +function getApiKey(): string { + return localStorage.getItem(API_KEY_STORAGE_KEY) || ''; +} + +function createApiClient(baseUrl?: string): KyInstance { + const base = baseUrl || DEFAULT_API_BASE; + return ky.create({ + prefixUrl: base, + hooks: { + beforeRequest: [ + (request) => { + const key = getApiKey(); + if (key) { + request.headers.set(API_KEY_HEADER, key); + } + }, + ], + }, + timeout: 30_000, + }); +} + +let _client: KyInstance | null = null; + +export function getClient(): KyInstance { + if (!_client) { + _client = createApiClient(); + } + return _client; +} + +export function resetClient(baseUrl?: string): void { + _client = createApiClient(baseUrl); +} + +export const api = { + getHealth: () => + getClient().get('health').json(), + + listTasks: (params?: { task_type?: string; status?: string; limit?: number; offset?: number }) => { + const searchParams: Record = {}; + if (params?.task_type) searchParams.task_type = params.task_type; + if (params?.status) searchParams.status = params.status; + if (params?.limit) searchParams.limit = params.limit; + if (params?.offset) searchParams.offset = params.offset; + return getClient().get('scrape/tasks', { searchParams }).json(); + }, + + getTaskStatus: (taskId: string) => + getClient().get(`scrape/status/${taskId}`).json(), + + deleteTask: (taskId: string) => + getClient().delete(`scrape/tasks/${taskId}`).json(), + + cleanupTasks: (maxAgeHours = 72) => + getClient().post('scrape/cleanup', { searchParams: { max_age_hours: maxAgeHours } }).json<{ removed: number; remaining: number }>(), + + resetStep: (data: { step: string; clear_dedupe?: boolean; project_dir?: string }) => + getClient().post('scrape/reset', { json: data }).json(), + + createSearch: (data: SearchParams) => + getClient().post('scrape/search', { json: data }).json(), + + createComments: (data: CommentsParams) => + getClient().post('scrape/comments', { json: data }).json(), + + createScripts: (data: ScriptsParams) => + getClient().post('scrape/scripts', { json: data }).json(), + + createMerge: (data: MergeParams) => + getClient().post('scrape/merge', { json: data }).json(), + + createRunAll: (data: RunAllParams) => + getClient().post('scrape/run-all', { json: data }).json(), + + getResultUrl: (taskId: string) => { + const base = localStorage.getItem(API_BASE_URL_STORAGE_KEY) || DEFAULT_API_BASE; + return `${base}/scrape/result/${taskId}`; + }, + + // ─── 数据管理 ─────────────────────────────────────────────── + + listDataFiles: () => + getClient().get('scrape/data/list').json(), + + previewData: (taskId: string) => + getClient().get(`scrape/data/preview/${taskId}`).json(), + + /** 导出数据，返回 Blob 供前端触发下载 */ + exportData: async (req: ExportRequest): Promise<{ blob: Blob; filename: string }> => { + const resp = await getClient().post('scrape/data/export', { json: req }); + const blob = await resp.blob(); + const disposition = resp.headers.get('content-disposition') || ''; + const match = disposition.match(/filename="?([^"]+)"?/); + const filename = match ? match[1] : `export.${req.format}`; + return { blob, filename }; + }, +}; diff --git a/web/src/api/types.ts b/web/src/api/types.ts new file mode 100644 index 000000000..c56396792 --- /dev/null +++ b/web/src/api/types.ts @@ -0,0 +1,133 @@ +export interface TaskInfo { + task_id: string; + task_type: string; + workspace: string; + params: Record; + status: 'pending' | 'running' | 'completed' | 'failed'; + created_at: string; + started_at: string | null; + completed_at: string | null; + error: string | null; + exit_code: number; + result: Record | null; + progress: string; +} + +export interface TaskStats { + total: number; + pending: number; + running: number; + completed: number; + failed: number; +} + +export interface TaskListResponse { + tasks: TaskInfo[]; + total: number; + offset: number; + limit: number; + stats: TaskStats; +} + +export interface CreateTaskResponse { + task_id: string; + status: string; + type: string; +} + +export interface HealthResponse { + status: 'healthy' | 'degraded' | 'unhealthy'; + uptime_seconds: number; + checks: { + chrome_cdp: { status: string; detail?: string }; + disk: { status: string; available_gb?: number; detail?: string }; + ffmpeg: { status: string; detail?: string }; + }; + system: Record; + tasks: TaskStats; +} + +export interface WSTaskEvent { + type: 'task_created' | 'task_started' | 'task_progress' | 'task_completed' | 'task_failed'; + task_id: string; + status: string; + progress?: string; + result?: Record; + error?: string; + timestamp: string; +} + +export interface WSLogMessage { + type: 'log'; + data: { + id?: number; + timestamp?: string; + level?: string; + message?: string; + }; +} + +export type WSMessage = WSTaskEvent | WSLogMessage; + +export interface SearchParams { + keywords: string[]; + max_count?: number; + project_dir?: string; +} + +export interface CommentsParams { + video_jsonl?: string; + project_dir?: string; +} + +export interface ScriptsParams { + video_jsonl?: string; + model?: string; + project_dir?: string; +} + +export interface MergeParams { + video_jsonl?: string; + comments_jsonl?: string; + scripts_jsonl?: string; + output_csv?: string; + project_dir?: string; +} + +export interface RunAllParams { + keywords: string[]; + max_count?: number; + steps?: string[]; + project_dir?: string; +} + +// ─── 数据管理 ─────────────────────────────────────────────── + +export interface DataFileItem { + task_id: string; + task_type: string; + file_name: string; + file_path: string; + file_size: number; + row_count: number; + created_at: string; + keywords: string[]; +} + +export interface DataListResponse { + items: DataFileItem[]; + total: number; +} + +export interface DataPreviewResponse { + task_id: string; + file_name: string; + rows: Record[]; + total_rows: number; +} + +export interface ExportRequest { + task_ids: string[]; + format: 'csv' | 'txt'; + limit?: number; +} diff --git a/web/src/components/dashboard/HealthPanel.tsx b/web/src/components/dashboard/HealthPanel.tsx new file mode 100644 index 000000000..8993e27ce --- /dev/null +++ b/web/src/components/dashboard/HealthPanel.tsx @@ -0,0 +1,65 @@ +import React from 'react'; +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import Typography from '@mui/material/Typography'; +import Box from '@mui/material/Box'; +import LinearProgress from '@mui/material/LinearProgress'; +import CheckCircleIcon from '@mui/icons-material/CheckCircle'; +import ErrorIcon from '@mui/icons-material/Error'; +import type { HealthResponse } from '../../api/types'; + +interface Props { + health: HealthResponse; +} + +export default function HealthPanel({ health }: Props) { + const checks = [ + { + label: 'Chrome CDP', + status: health.checks?.chrome_cdp?.status, + detail: health.checks?.chrome_cdp?.detail, + }, + { + label: '磁盘空间', + status: health.checks?.disk?.status, + detail: health.checks?.disk?.detail, + value: health.checks?.disk?.available_gb, + }, + { + label: 'FFmpeg', + status: health.checks?.ffmpeg?.status, + detail: health.checks?.ffmpeg?.detail, + }, + ]; + + return ( + + + 系统健康 + + {checks.map((check) => ( + + {check.status === 'ok' || check.status === 'healthy' ? ( + + ) : ( + + )} + {check.label} + + {check.value ? `${check.value} GB 可用` : check.status || '未知'} + + {check.detail && ( + ({check.detail}) + )} + + ))} + + {health.uptime_seconds != null && ( + + 运行时间: {Math.floor(health.uptime_seconds / 3600)}h {Math.floor((health.uptime_seconds % 3600) / 60)}m + + )} + + + ); +} diff --git a/web/src/components/dashboard/RecentTasks.tsx b/web/src/components/dashboard/RecentTasks.tsx new file mode 100644 index 000000000..7a81de655 --- /dev/null +++ b/web/src/components/dashboard/RecentTasks.tsx @@ -0,0 +1,67 @@ +import React from 'react'; +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import Typography from '@mui/material/Typography'; +import Table from '@mui/material/Table'; +import TableBody from '@mui/material/TableBody'; +import TableCell from '@mui/material/TableCell'; +import TableHead from '@mui/material/TableHead'; +import TableRow from '@mui/material/TableRow'; +import { useNavigate } from 'react-router-dom'; +import StatusBadge from '../task/StatusBadge'; +import TaskTypeIcon from '../task/TaskTypeIcon'; +import type { TaskInfo } from '../../api/types'; +import { formatDateTime } from '../../utils/format'; + +interface Props { + tasks: TaskInfo[]; +} + +export default function RecentTasks({ tasks }: Props) { + const navigate = useNavigate(); + + return ( + + + 最近任务 + + + + Task ID + 类型 + 状态 + 创建时间 + + + + {tasks.length === 0 ? ( + + + 暂无任务 + + + ) : ( + tasks.slice(0, 5).map((task) => ( + navigate(`/tasks/${task.task_id}`)} + > + + + {task.task_id.slice(0, 8)} + + + + + {formatDateTime(task.created_at)} + + )) + )} + +
+ + + ); +} diff --git a/web/src/components/dashboard/StatsCards.tsx b/web/src/components/dashboard/StatsCards.tsx new file mode 100644 index 000000000..055710072 --- /dev/null +++ b/web/src/components/dashboard/StatsCards.tsx @@ -0,0 +1,50 @@ +import React from 'react'; +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import Typography from '@mui/material/Typography'; +import Box from '@mui/material/Box'; +import { STATUS_CONFIG, type TaskStatus } from '../../utils/constants'; + +interface Props { + stats: { + pending: number; + running: number; + completed: number; + failed: number; + }; + onCardClick?: (status: string) => void; +} + +const CARDS: { key: TaskStatus; icon: string }[] = [ + { key: 'pending', icon: '⏳' }, + { key: 'running', icon: '⚡' }, + { key: 'completed', icon: '✅' }, + { key: 'failed', icon: '❌' }, +]; + +export default function StatsCards({ stats, onCardClick }: Props) { + return ( + + {CARDS.map(({ key, icon }) => { + const cfg = STATUS_CONFIG[key]; + return ( + onCardClick?.(key)} + > + + + {cfg.label} + {icon} + + + {stats[key]} + + + + ); + })} + + ); +} diff --git a/web/src/components/layout/AppLayout.tsx b/web/src/components/layout/AppLayout.tsx new file mode 100644 index 000000000..1d7b191f9 --- /dev/null +++ b/web/src/components/layout/AppLayout.tsx @@ -0,0 +1,103 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import Drawer from '@mui/material/Drawer'; +import List from '@mui/material/List'; +import ListItem from '@mui/material/ListItem'; +import ListItemButton from '@mui/material/ListItemButton'; +import ListItemIcon from '@mui/material/ListItemIcon'; +import ListItemText from '@mui/material/ListItemText'; +import Typography from '@mui/material/Typography'; +import Badge from '@mui/material/Badge'; +import DashboardIcon from '@mui/icons-material/Dashboard'; +import AddCircleIcon from '@mui/icons-material/AddCircle'; +import ListAltIcon from '@mui/icons-material/ListAlt'; +import BarChartIcon from '@mui/icons-material/BarChart'; +import SettingsIcon from '@mui/icons-material/Settings'; +import StorageIcon from '@mui/icons-material/Storage'; +import ConnectionIndicator from '../shared/ConnectionIndicator'; +import { useWebSocket } from '../../hooks/useWebSocket'; +import { useNavigate, useLocation, Outlet } from 'react-router-dom'; +import { useTaskStore } from '../../store/useTaskStore'; + +const DRAWER_WIDTH = 240; + +export default function AppLayout() { + const navigate = useNavigate(); + const location = useLocation(); + const { status: wsStatus } = useWebSocket(); + const { stats } = useTaskStore(); + + const runningCount = stats?.running ?? 0; + + const NAV_ITEMS = [ + { path: '/', label: '仪表盘', icon: }, + { path: '/create', label: '创建任务', icon: }, + { + path: '/tasks', + label: '任务列表', + icon: ( + + + + ), + }, + { path: '/data', label: '数据管理', icon: }, + { path: '/analytics', label: '数据可视化', icon: }, + { path: '/settings', label: '设置', icon: }, + ]; + + return ( + + + + + MediaCrawler + + + + + + + {NAV_ITEMS.map((item) => { + const isActive = location.pathname === item.path || + (item.path !== '/' && location.pathname.startsWith(item.path)); + return ( + + navigate(item.path)} + sx={{ + mx: 1, + borderRadius: 2, + '&.Mui-selected': { + backgroundColor: 'primary.main', + color: 'white', + '&:hover': { backgroundColor: 'primary.dark' }, + '& .MuiListItemIcon-root': { color: 'white' }, + }, + }} + > + {item.icon} + + + + ); + })} + + + + + + + ); +} diff --git a/web/src/components/shared/ConnectionIndicator.tsx b/web/src/components/shared/ConnectionIndicator.tsx new file mode 100644 index 000000000..40dc495b5 --- /dev/null +++ b/web/src/components/shared/ConnectionIndicator.tsx @@ -0,0 +1,29 @@ +import React from 'react'; +import CircleIcon from '@mui/icons-material/Circle'; +import Tooltip from '@mui/material/Tooltip'; +import { type WSStatus } from '../../hooks/useWebSocket'; + +interface Props { + status: WSStatus; +} + +const STATUS_MAP: Record = { + connected: { color: '#4caf50', label: 'WebSocket 已连接' }, + connecting: { color: '#ff9800', label: 'WebSocket 连接中...' }, + disconnected: { color: '#f44336', label: 'WebSocket 已断开' }, +}; + +export default function ConnectionIndicator({ status }: Props) { + const cfg = STATUS_MAP[status]; + return ( + + + + ); +} diff --git a/web/src/components/shared/ErrorAlert.tsx b/web/src/components/shared/ErrorAlert.tsx new file mode 100644 index 000000000..f5b697350 --- /dev/null +++ b/web/src/components/shared/ErrorAlert.tsx @@ -0,0 +1,27 @@ +import React from 'react'; +import Alert from '@mui/material/Alert'; +import IconButton from '@mui/material/IconButton'; +import CloseIcon from '@mui/icons-material/Close'; + +interface Props { + message: string; + onClose?: () => void; +} + +export default function ErrorAlert({ message, onClose }: Props) { + return ( + + + + ) : undefined + } + > + {message} + + ); +} diff --git a/web/src/components/shared/LoadingOverlay.tsx b/web/src/components/shared/LoadingOverlay.tsx new file mode 100644 index 000000000..d1fdf036d --- /dev/null +++ b/web/src/components/shared/LoadingOverlay.tsx @@ -0,0 +1,16 @@ +import React from 'react'; +import CircularProgress from '@mui/material/CircularProgress'; +import Box from '@mui/material/Box'; + +interface Props { + loading?: boolean; +} + +export default function LoadingOverlay({ loading = true }: Props) { + if (!loading) return null; + return ( + + + + ); +} diff --git a/web/src/components/task/DeleteConfirmDialog.tsx b/web/src/components/task/DeleteConfirmDialog.tsx new file mode 100644 index 000000000..fbbebc149 --- /dev/null +++ b/web/src/components/task/DeleteConfirmDialog.tsx @@ -0,0 +1,31 @@ +import React, { useState } from 'react'; +import Dialog from '@mui/material/Dialog'; +import DialogTitle from '@mui/material/DialogTitle'; +import DialogContent from '@mui/material/DialogContent'; +import DialogActions from '@mui/material/DialogActions'; +import Button from '@mui/material/Button'; +import Typography from '@mui/material/Typography'; + +interface Props { + open: boolean; + taskId: string; + onConfirm: () => void; + onCancel: () => void; +} + +export default function DeleteConfirmDialog({ open, taskId, onConfirm, onCancel }: Props) { + return ( + + 确认删除 + + + 确定要删除任务 {taskId} 吗？此操作不可恢复。 + + + + + + + + ); +} diff --git a/web/src/components/task/StatusBadge.tsx b/web/src/components/task/StatusBadge.tsx new file mode 100644 index 000000000..d8d6c8d1c --- /dev/null +++ b/web/src/components/task/StatusBadge.tsx @@ -0,0 +1,24 @@ +import React from 'react'; +import Chip from '@mui/material/Chip'; +import { STATUS_CONFIG, type TaskStatus } from '../../utils/constants'; + +interface Props { + status: string; + size?: 'small' | 'medium'; +} + +export default function StatusBadge({ status, size = 'medium' }: Props) { + const cfg = STATUS_CONFIG[status as TaskStatus] || STATUS_CONFIG.pending; + return ( + + ); +} diff --git a/web/src/components/task/TaskTypeIcon.tsx b/web/src/components/task/TaskTypeIcon.tsx new file mode 100644 index 000000000..f00b688f2 --- /dev/null +++ b/web/src/components/task/TaskTypeIcon.tsx @@ -0,0 +1,29 @@ +import React from 'react'; +import SearchIcon from '@mui/icons-material/Search'; +import CommentIcon from '@mui/icons-material/Comment'; +import TranscribeIcon from '@mui/icons-material/Transcribe'; +import MergeIcon from '@mui/icons-material/Merge'; +import PlayArrowIcon from '@mui/icons-material/PlayArrow'; +import { TASK_TYPE_LABELS } from '../../utils/constants'; + +const ICON_MAP: Record = { + search: , + comments: , + scripts: , + merge: , + run_all: , +}; + +interface Props { + type: string; + showLabel?: boolean; +} + +export default function TaskTypeIcon({ type, showLabel = true }: Props) { + return ( + + {ICON_MAP[type] || } + {showLabel && {TASK_TYPE_LABELS[type] || type}} + + ); +} diff --git a/web/src/hooks/usePolling.ts b/web/src/hooks/usePolling.ts new file mode 100644 index 000000000..423a7fc55 --- /dev/null +++ b/web/src/hooks/usePolling.ts @@ -0,0 +1,17 @@ +import { useEffect, useRef } from 'react'; +import { useTaskStore } from '../store/useTaskStore'; +import { POLLING_INTERVAL_MS } from '../utils/constants'; + +export function usePolling(enabled: boolean, params?: Record) { + const fetchTasks = useTaskStore((s) => s.fetchTasks); + const timerRef = useRef>(); + + useEffect(() => { + if (!enabled) return; + fetchTasks(params as any); + timerRef.current = setInterval(() => fetchTasks(params as any), POLLING_INTERVAL_MS); + return () => { + if (timerRef.current) clearInterval(timerRef.current); + }; + }, [enabled, fetchTasks]); +} diff --git a/web/src/hooks/useWebSocket.ts b/web/src/hooks/useWebSocket.ts new file mode 100644 index 000000000..ab05592f9 --- /dev/null +++ b/web/src/hooks/useWebSocket.ts @@ -0,0 +1,132 @@ +import { useEffect, useRef, useCallback, useState } from 'react'; +import { useTaskStore } from '../store/useTaskStore'; +import { useSettingsStore } from '../store/useSettingsStore'; +import { + WS_RECONNECT_BASE_MS, WS_RECONNECT_MAX_MS, + WS_MAX_RETRIES, POLLING_INTERVAL_MS, API_KEY_STORAGE_KEY, +} from '../utils/constants'; +import type { WSMessage } from '../api/types'; + +export type WSStatus = 'connected' | 'connecting' | 'disconnected'; + +export interface UseWebSocketReturn { + status: WSStatus; + logs: Array<{ id: number; timestamp: string; level: string; message: string }>; + clearLogs: () => void; +} + +export function useWebSocket(): UseWebSocketReturn { + const [status, setStatus] = useState('disconnected'); + const [logs, setLogs] = useState>([]); + const wsRef = useRef(null); + const retriesRef = useRef(0); + const timerRef = useRef>(); + const pollingRef = useRef>(); + + const updateTaskStatus = useTaskStore((s) => s.updateTaskStatus); + const fetchTasks = useTaskStore((s) => s.fetchTasks); + const apiBaseUrl = useSettingsStore((s) => s.apiBaseUrl); + + const clearTimers = useCallback(() => { + if (timerRef.current) clearTimeout(timerRef.current); + if (pollingRef.current) clearInterval(pollingRef.current); + }, []); + + const startPolling = useCallback(() => { + if (pollingRef.current) clearInterval(pollingRef.current); + fetchTasks({ limit: 50 }); + pollingRef.current = setInterval(() => { + fetchTasks({ limit: 50 }); + }, POLLING_INTERVAL_MS); + }, [fetchTasks]); + + const clearLogs = useCallback(() => setLogs([]), []); + + const connect = useCallback(() => { + clearTimers(); + const wsUrl = apiBaseUrl.replace(/^http/, 'ws') + '/ws/tasks'; + const apiKey = localStorage.getItem(API_KEY_STORAGE_KEY); + const fullUrl = apiKey ? `${wsUrl}?api_key=${apiKey}` : wsUrl; + + try { + const ws = new WebSocket(fullUrl); + wsRef.current = ws; + setStatus('connecting'); + + ws.onopen = () => { + setStatus('connected'); + retriesRef.current = 0; + // Stop polling if it was running + if (pollingRef.current) { + clearInterval(pollingRef.current); + pollingRef.current = undefined; + } + }; + + ws.onmessage = (event) => { + try { + const msg: WSMessage = JSON.parse(event.data); + + // Handle task status events + if (msg.type !== 'log' && 'task_id' in msg) { + updateTaskStatus(msg.task_id, msg.status, msg.progress); + } + + // Handle log messages (pushed via same WS connection) + if (msg.type === 'log') { + const d = msg.data; + setLogs((prev) => { + const next = [...prev, { + id: d.id ?? prev.length, + timestamp: d.timestamp ?? '', + level: d.level ?? 'info', + message: d.message ?? '', + }]; + // Keep last 500 logs + return next.length > 500 ? next.slice(-500) : next; + }); + } + } catch { + // ignore parse errors + } + }; + + ws.onclose = () => { + setStatus('disconnected'); + wsRef.current = null; + scheduleReconnect(); + }; + + ws.onerror = () => { + ws.close(); + }; + } catch { + setStatus('disconnected'); + scheduleReconnect(); + } + }, [apiBaseUrl, updateTaskStatus, clearTimers]); + + const scheduleReconnect = useCallback(() => { + if (retriesRef.current >= WS_MAX_RETRIES) { + console.warn('WebSocket 重连失败，降级为轮询'); + startPolling(); + return; + } + const delay = Math.min( + WS_RECONNECT_BASE_MS * Math.pow(2, retriesRef.current), + WS_RECONNECT_MAX_MS + ); + retriesRef.current++; + timerRef.current = setTimeout(connect, delay); + }, [connect, startPolling]); + + useEffect(() => { + connect(); + return () => { + wsRef.current?.close(); + clearTimers(); + }; + }, [connect, clearTimers]); + + return { status, logs, clearLogs }; +} diff --git a/web/src/index.css b/web/src/index.css new file mode 100644 index 000000000..b5aa69bb9 --- /dev/null +++ b/web/src/index.css @@ -0,0 +1,17 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +body { + font-family: 'Inter', system-ui, sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +@keyframes pulse-blue { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.5; } +} +.animate-pulse-blue { + animation: pulse-blue 2s ease-in-out infinite; +} diff --git a/web/src/main.tsx b/web/src/main.tsx new file mode 100644 index 000000000..ff969db25 --- /dev/null +++ b/web/src/main.tsx @@ -0,0 +1,23 @@ +import React from 'react'; +import ReactDOM from 'react-dom/client'; +import { BrowserRouter } from 'react-router-dom'; +import { ThemeProvider } from '@mui/material/styles'; +import CssBaseline from '@mui/material/CssBaseline'; +import App from './App'; +import theme from './theme/theme'; +import { useSettingsStore } from './store/useSettingsStore'; +import './index.css'; + +// Load settings from localStorage on startup +useSettingsStore.getState().loadFromStorage(); + +ReactDOM.createRoot(document.getElementById('root')!).render( + + + + + + + + , +); diff --git a/web/src/pages/AnalyticsPage.tsx b/web/src/pages/AnalyticsPage.tsx new file mode 100644 index 000000000..ce671853c --- /dev/null +++ b/web/src/pages/AnalyticsPage.tsx @@ -0,0 +1,134 @@ +import React, { useEffect, useMemo } from 'react'; +import Typography from '@mui/material/Typography'; +import Box from '@mui/material/Box'; +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import Grid from '@mui/material/Grid'; +import { BarChart, Bar, XAxis, YAxis, CartesianGrid, Tooltip, ResponsiveContainer, PieChart, Pie, Cell, Legend } from 'recharts'; +import { useTaskStore } from '../store/useTaskStore'; +import { useNavigate } from 'react-router-dom'; +import { TASK_TYPE_LABELS } from '../utils/constants'; + +const PIE_COLORS = ['#2e7d32', '#d32f2f', '#1976d2', '#9e9e9e']; + +export default function AnalyticsPage() { + const navigate = useNavigate(); + const { tasks, stats, fetchTasks } = useTaskStore(); + + useEffect(() => { + fetchTasks({ limit: 200 }); + }, [fetchTasks]); + + const typeData = useMemo(() => { + const counts: Record = {}; + tasks.forEach((t) => { + const label = TASK_TYPE_LABELS[t.task_type] || t.task_type; + counts[label] = (counts[label] || 0) + 1; + }); + return Object.entries(counts).map(([name, count]) => ({ name, count })); + }, [tasks]); + + const statusData = useMemo(() => { + if (!stats) return []; + return [ + { name: '已完成', value: stats.completed, status: 'completed' }, + { name: '失败', value: stats.failed, status: 'failed' }, + { name: '执行中', value: stats.running, status: 'running' }, + { name: '待执行', value: stats.pending, status: 'pending' }, + ].filter((d) => d.value > 0); + }, [stats]); + + return ( + + 数据可视化 + + {/* 概览卡片 */} + + + + 总任务数 + {stats?.total || 0} + + + + + 已完成 + {stats?.completed || 0} + + + + + 执行中 + {stats?.running || 0} + + + + + 成功率 + + {stats && stats.total > 0 + ? `${((stats.completed / stats.total) * 100).toFixed(1)}%` + : '-'} + + + + + + + {/* 任务类型柱状图 */} + + + + 任务类型分布 + {typeData.length === 0 ? ( + 暂无数据 + ) : ( + + + + + + + + + + )} + + + + + {/* 状态饼图 */} + + + + 任务状态分布 + {statusData.length === 0 ? ( + 暂无数据 + ) : ( + + + + {statusData.map((_, idx) => ( + + ))} + + + + + + )} + + + + + + ); +} diff --git a/web/src/pages/CreateTaskPage.tsx b/web/src/pages/CreateTaskPage.tsx new file mode 100644 index 000000000..11a799e31 --- /dev/null +++ b/web/src/pages/CreateTaskPage.tsx @@ -0,0 +1,237 @@ +import React, { useState } from 'react'; +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import Typography from '@mui/material/Typography'; +import Box from '@mui/material/Box'; +import TextField from '@mui/material/TextField'; +import Button from '@mui/material/Button'; +import Slider from '@mui/material/Slider'; +import Select from '@mui/material/Select'; +import MenuItem from '@mui/material/MenuItem'; +import InputLabel from '@mui/material/InputLabel'; +import FormControl from '@mui/material/FormControl'; +import ToggleButton from '@mui/material/ToggleButton'; +import ToggleButtonGroup from '@mui/material/ToggleButtonGroup'; +import Alert from '@mui/material/Alert'; +import Snackbar from '@mui/material/Snackbar'; +import Grid from '@mui/material/Grid'; +import { useNavigate } from 'react-router-dom'; +import { useTaskStore } from '../store/useTaskStore'; +import { TASK_TYPE_LABELS, WHISPER_MODELS } from '../utils/constants'; +import ErrorAlert from '../components/shared/ErrorAlert'; + +const TASK_TYPES = ['search', 'comments', 'scripts', 'merge', 'run_all'] as const; + +export default function CreateTaskPage() { + const navigate = useNavigate(); + const { loading, error, createTask, clearError } = useTaskStore(); + const [taskType, setTaskType] = useState('search'); + const [keywords, setKeywords] = useState(''); + const [maxCount, setMaxCount] = useState(20); + const [model, setModel] = useState('small'); + const [videoJsonl, setVideoJsonl] = useState(''); + const [commentsJsonl, setCommentsJsonl] = useState(''); + const [scriptsJsonl, setScriptsJsonl] = useState(''); + const [snackOpen, setSnackOpen] = useState(false); + const [createdId, setCreatedId] = useState(''); + + const parsedKeywords = keywords + .split(/[\n,]/) + .map((s) => s.trim()) + .filter(Boolean); + + const handleSubmit = async () => { + clearError(); + let taskId: string | null = null; + + switch (taskType) { + case 'search': + taskId = await createTask('search', { + keywords: parsedKeywords, + max_count: maxCount, + }); + break; + case 'comments': + taskId = await createTask('comments', { video_jsonl: videoJsonl || undefined }); + break; + case 'scripts': + taskId = await createTask('scripts', { video_jsonl: videoJsonl || undefined, model }); + break; + case 'merge': + taskId = await createTask('merge', { + video_jsonl: videoJsonl || undefined, + comments_jsonl: commentsJsonl || undefined, + scripts_jsonl: scriptsJsonl || undefined, + }); + break; + case 'run_all': + taskId = await createTask('run_all', { + keywords: parsedKeywords, + max_count: maxCount, + }); + break; + } + + if (taskId) { + setCreatedId(taskId); + setSnackOpen(true); + setTimeout(() => navigate(`/tasks/${taskId}`), 1500); + } + }; + + const showKeywords = taskType === 'search' || taskType === 'run_all'; + const showMaxCount = taskType === 'search' || taskType === 'run_all'; + const showVideoInput = taskType === 'comments' || taskType === 'scripts' || taskType === 'merge'; + const showCommentsInput = taskType === 'merge'; + const showScriptsInput = taskType === 'merge'; + const showModel = taskType === 'scripts'; + + const canSubmit = (() => { + if (taskType === 'search' || taskType === 'run_all') return parsedKeywords.length > 0; + return true; + })(); + + return ( + + 创建任务 + {error && } + + + + + + {/* 任务类型选择 */} + + 任务类型 + + + + {/* 关键词输入 */} + {showKeywords && ( + <> + setKeywords(e.target.value)} + helperText={`已输入 ${parsedKeywords.length}/50 个关键词`} + error={parsedKeywords.length > 50} + /> + + )} + + {/* 最大采集数 */} + {showMaxCount && ( + + + 每个关键词最大采集数: {maxCount} + + setMaxCount(v as number)} + min={1} + max={200} + valueLabelDisplay="auto" + /> + + )} + + {/* 视频文件路径 */} + {showVideoInput && ( + setVideoJsonl(e.target.value)} + placeholder="留空使用上次搜索结果" + /> + )} + + {/* 评论文件路径 */} + {showCommentsInput && ( + setCommentsJsonl(e.target.value)} + /> + )} + + {/* 文案文件路径 */} + {showScriptsInput && ( + setScriptsJsonl(e.target.value)} + /> + )} + + {/* Whisper 模型选择 */} + {showModel && ( + + Whisper 模型 + + + )} + + {/* 提交按钮 */} + + + + + + {/* 参数预览 */} + + + + 参数预览 + + + 类型: {TASK_TYPE_LABELS[taskType]} + + {showKeywords && ( + + 关键词数: {parsedKeywords.length} + + )} + {showMaxCount && ( + + 最大采集数: {maxCount} + + )} + {showModel && ( + + Whisper 模型: {model} + + )} + + + + + + + setSnackOpen(false)} + message={`任务已创建: ${createdId.slice(0, 8)}...`} + /> + + ); +} diff --git a/web/src/pages/DashboardPage.tsx b/web/src/pages/DashboardPage.tsx new file mode 100644 index 000000000..d962603b9 --- /dev/null +++ b/web/src/pages/DashboardPage.tsx @@ -0,0 +1,49 @@ +import React, { useEffect, useState } from 'react'; +import Typography from '@mui/material/Typography'; +import Box from '@mui/material/Box'; +import Grid from '@mui/material/Grid'; +import { useTaskStore } from '../store/useTaskStore'; +import { api } from '../api/client'; +import StatsCards from '../components/dashboard/StatsCards'; +import HealthPanel from '../components/dashboard/HealthPanel'; +import RecentTasks from '../components/dashboard/RecentTasks'; +import LoadingOverlay from '../components/shared/LoadingOverlay'; +import ErrorAlert from '../components/shared/ErrorAlert'; +import type { HealthResponse } from '../api/types'; +import { useNavigate } from 'react-router-dom'; + +export default function DashboardPage() { + const navigate = useNavigate(); + const { tasks, stats, loading, error, fetchTasks, clearError } = useTaskStore(); + const [health, setHealth] = useState(null); + + useEffect(() => { + fetchTasks({ limit: 5 }); + api.getHealth().then(setHealth).catch(() => {}); + }, [fetchTasks]); + + const handleCardClick = (status: string) => { + navigate(`/tasks?status=${status}`); + }; + + if (loading && tasks.length === 0) return ; + + return ( + + 仪表盘 + {error && } + + + + {health && } + + + + + + + ); +} diff --git a/web/src/pages/DataPage.tsx b/web/src/pages/DataPage.tsx new file mode 100644 index 000000000..0eb26c3d3 --- /dev/null +++ b/web/src/pages/DataPage.tsx @@ -0,0 +1,558 @@ +import React, { useCallback, useEffect, useState } from 'react'; +import Typography from '@mui/material/Typography'; +import Box from '@mui/material/Box'; +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import Table from '@mui/material/Table'; +import TableBody from '@mui/material/TableBody'; +import TableCell from '@mui/material/TableCell'; +import TableHead from '@mui/material/TableHead'; +import TableRow from '@mui/material/TableRow'; +import Checkbox from '@mui/material/Checkbox'; +import Button from '@mui/material/Button'; +import IconButton from '@mui/material/IconButton'; +import Tooltip from '@mui/material/Tooltip'; +import Chip from '@mui/material/Chip'; +import Divider from '@mui/material/Divider'; +import CircularProgress from '@mui/material/CircularProgress'; +import LinearProgress from '@mui/material/LinearProgress'; +import Dialog from '@mui/material/Dialog'; +import DialogTitle from '@mui/material/DialogTitle'; +import DialogContent from '@mui/material/DialogContent'; +import DialogActions from '@mui/material/DialogActions'; +import ToggleButton from '@mui/material/ToggleButton'; +import ToggleButtonGroup from '@mui/material/ToggleButtonGroup'; +import Alert from '@mui/material/Alert'; +import Stack from '@mui/material/Stack'; +import Paper from '@mui/material/Paper'; +import TableContainer from '@mui/material/TableContainer'; +import Collapse from '@mui/material/Collapse'; +import DownloadIcon from '@mui/icons-material/Download'; +import PreviewIcon from '@mui/icons-material/Preview'; +import RefreshIcon from '@mui/icons-material/Refresh'; +import StorageIcon from '@mui/icons-material/Storage'; +import CheckBoxOutlineBlankIcon from '@mui/icons-material/CheckBoxOutlineBlank'; +import CheckBoxIcon from '@mui/icons-material/CheckBox'; +import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; +import ExpandLessIcon from '@mui/icons-material/ExpandLess'; +import ArticleIcon from '@mui/icons-material/Article'; +import GridOnIcon from '@mui/icons-material/GridOn'; + +import { api } from '../api/client'; +import type { DataFileItem, DataPreviewResponse } from '../api/types'; +import { formatDateTime, formatFileSize } from '../utils/format'; +import TaskTypeIcon from '../components/task/TaskTypeIcon'; +import ErrorAlert from '../components/shared/ErrorAlert'; + +// ─── 预览对话框 ──────────────────────────────────────────────── + +function PreviewDialog({ + open, taskId, onClose, +}: { open: boolean; taskId: string | null; onClose: () => void }) { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + const [err, setErr] = useState(''); + + useEffect(() => { + if (!open || !taskId) return; + setLoading(true); + setErr(''); + api.previewData(taskId) + .then(setData) + .catch((e) => setErr(e.message || '预览失败')) + .finally(() => setLoading(false)); + }, [open, taskId]); + + const columns = data?.rows?.[0] ? Object.keys(data.rows[0]) : []; + + return ( + + + + + 数据预览 + {data && ( + + )} + + + + {loading && } + {err && {err}} + {data && data.rows.length > 0 && ( + + + + + {columns.map((col) => ( + + {col} + + ))} + + + + {data.rows.map((row, i) => ( + + {columns.map((col) => ( + + + {String(row[col] ?? '')} + + + ))} + + ))} + +
+ + )} + {data && data.rows.length === 0 && ( + + 文件为空 + + )} + + + + + + ); +} + +// ─── 导出对话框 ──────────────────────────────────────────────── + +function ExportDialog({ + open, + selectedIds, + selectedItems, + onClose, +}: { + open: boolean; + selectedIds: string[]; + selectedItems: DataFileItem[]; + onClose: () => void; +}) { + const [format, setFormat] = useState<'csv' | 'txt'>('csv'); + const [exporting, setExporting] = useState(false); + const [err, setErr] = useState(''); + const [showFormatGuide, setShowFormatGuide] = useState(false); + + const totalRows = selectedItems.reduce((s, i) => s + i.row_count, 0); + const totalSize = selectedItems.reduce((s, i) => s + i.file_size, 0); + const willTruncate = totalRows > 200; + + const handleExport = async () => { + setExporting(true); + setErr(''); + try { + const { blob, filename } = await api.exportData({ + task_ids: selectedIds, + format, + limit: 200, + }); + // 触发浏览器下载 + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + setTimeout(() => { + document.body.removeChild(a); + URL.revokeObjectURL(url); + }, 100); + onClose(); + } catch (e: any) { + setErr(e.message || '导出失败，请重试'); + } finally { + setExporting(false); + } + }; + + return ( + + + + + 批量导出数据 + + + + + {/* 选中概览 */} + + 已选中 + + } label={`${selectedIds.length} 个任务`} size="small" /> + + + + {willTruncate && ( + + 数据超过 200 行上限，将自动截断至前 200 行 + + )} + + + {/* 格式选择 */} + + 选择导出格式 + v && setFormat(v)} + size="small" + > + + + CSV + + + + TXT + + + + + {/* 格式说明（可折叠） */} + + + + + {format === 'csv' ? ( + <> + + CSV 格式：标准逗号分隔，UTF-8 编码 + + + {`列：video_id, platform, script_text,\n likes, favorites, shares, comments\n评论多值用 | 分隔`} + + + ) : ( + <> + + TXT 格式：每行一条，字段用 || 分隔，UTF-8 编码 + + + {`video_id||script_text||likes||favorites||shares||comments\n\n上限：200 行 / 2MB`} + + + )} + + + + + {err && {err}} + + + + + + + + ); +} + +// ─── 主页面 ───────────────────────────────────────────────────── + +export default function DataPage() { + const [items, setItems] = useState([]); + const [loading, setLoading] = useState(false); + const [err, setErr] = useState(''); + const [selected, setSelected] = useState>(new Set()); + const [previewTaskId, setPreviewTaskId] = useState(null); + const [exportOpen, setExportOpen] = useState(false); + + const loadData = useCallback(async () => { + setLoading(true); + setErr(''); + try { + const resp = await api.listDataFiles(); + setItems(resp.items); + } catch (e: any) { + setErr(e.message || '获取数据列表失败'); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { loadData(); }, [loadData]); + + const allSelected = items.length > 0 && selected.size === items.length; + const someSelected = selected.size > 0 && !allSelected; + + const toggleAll = () => { + if (allSelected) { + setSelected(new Set()); + } else { + setSelected(new Set(items.map((i) => i.task_id))); + } + }; + + const toggleItem = (taskId: string) => { + setSelected((prev) => { + const next = new Set(prev); + if (next.has(taskId)) next.delete(taskId); + else next.add(taskId); + return next; + }); + }; + + const selectedItems = items.filter((i) => selected.has(i.task_id)); + const totalSelectedRows = selectedItems.reduce((s, i) => s + i.row_count, 0); + const totalSelectedSize = selectedItems.reduce((s, i) => s + i.file_size, 0); + + return ( + + {/* 标题栏 */} + + + 数据管理 + + + + + {err && setErr('')} />} + + {/* 选中状态栏 */} + {selected.size > 0 && ( + setSelected(new Set())}> + 取消全选 + + } + > + + + 已选 {selected.size} 个任务 + + + + {totalSelectedRows > 200 && ( + + )} + + + )} + + + {loading && } + + {items.length === 0 && !loading ? ( + + + + 暂无可导出的数据 + + + 完成采集任务后，结果数据将在此显示 + + + ) : ( + + + + + } + checkedIcon={} + /> + + Task ID + 类型 + 关键词 + 行数 + 文件大小 + 完成时间 + 操作 + + + + {items.map((item) => ( + toggleItem(item.task_id)} + > + e.stopPropagation()}> + toggleItem(item.task_id)} + /> + + + + {item.task_id.slice(0, 8)} + + + + + + + {item.keywords && item.keywords.length > 0 ? ( + + {item.keywords.slice(0, 3).map((kw) => ( + + ))} + {item.keywords.length > 3 && ( + + )} + + ) : ( + + {item.file_name} + + )} + + + + {item.row_count.toLocaleString()} + + + + + {formatFileSize(item.file_size)} + + + + {formatDateTime(item.created_at)} + + e.stopPropagation()}> + + setPreviewTaskId(item.task_id)} + > + + + + + { + setSelected(new Set([item.task_id])); + setExportOpen(true); + }} + > + + + + + + ))} + +
+ )} + + + + {/* 格式说明卡片 */} + + + 导出格式说明 + + + + + + CSV 格式 + + + video_id, platform, script_text,
+ likes, favorites, shares, comments
+ （comments 多值用 | 分隔） + + + + + + TXT 格式 + + + video_id||script_text||likes
+ ||favorites||shares||comments
+ + 字段用 || 分隔 · 上限 200 行 / 2MB · UTF-8 + + + + + + + + {/* 预览对话框 */} + setPreviewTaskId(null)} + /> + + {/* 导出对话框 */} + setExportOpen(false)} + /> + + ); +} diff --git a/web/src/pages/SettingsPage.tsx b/web/src/pages/SettingsPage.tsx new file mode 100644 index 000000000..2e8b285bb --- /dev/null +++ b/web/src/pages/SettingsPage.tsx @@ -0,0 +1,99 @@ +import React, { useState } from 'react'; +import Typography from '@mui/material/Typography'; +import Box from '@mui/material/Box'; +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import TextField from '@mui/material/TextField'; +import Button from '@mui/material/Button'; +import Alert from '@mui/material/Alert'; +import Divider from '@mui/material/Divider'; +import DeleteSweepIcon from '@mui/icons-material/DeleteSweep'; +import { useSettingsStore } from '../store/useSettingsStore'; +import { useTaskStore } from '../store/useTaskStore'; +import { DEFAULT_API_BASE } from '../utils/constants'; + +export default function SettingsPage() { + const { apiKey, apiBaseUrl, setApiKey, setApiBaseUrl, loadFromStorage } = useSettingsStore(); + const { cleanupTasks } = useTaskStore(); + const [keyInput, setKeyInput] = useState(apiKey); + const [urlInput, setUrlInput] = useState(apiBaseUrl); + const [cleanupHours, setCleanupHours] = useState(72); + const [saved, setSaved] = useState(false); + const [cleanupResult, setCleanupResult] = useState(null); + + const handleSave = () => { + setApiKey(keyInput); + setApiBaseUrl(urlInput); + setSaved(true); + setTimeout(() => setSaved(false), 3000); + }; + + const handleCleanup = async () => { + const removed = await cleanupTasks(cleanupHours); + setCleanupResult(`已清理 ${removed} 个过期任务`); + setTimeout(() => setCleanupResult(null), 5000); + }; + + React.useEffect(() => { + loadFromStorage(); + }, [loadFromStorage]); + + return ( + + 设置 + + + + API 配置 + + + setUrlInput(e.target.value)} + placeholder={DEFAULT_API_BASE} + helperText="后端 API 地址" + /> + setKeyInput(e.target.value)} + placeholder="可选，配置后端认证时需要" + helperText="存储在浏览器 localStorage 中" + /> + + {saved && 设置已保存} + + + + + + + 任务清理 + + + setCleanupHours(Number(e.target.value))} + inputProps={{ min: 1 }} + /> + + {cleanupResult && {cleanupResult}} + + + + + ); +} diff --git a/web/src/pages/TaskDetailPage.tsx b/web/src/pages/TaskDetailPage.tsx new file mode 100644 index 000000000..bb747bdbc --- /dev/null +++ b/web/src/pages/TaskDetailPage.tsx @@ -0,0 +1,779 @@ +import React, { useEffect, useRef, useState } from 'react'; +import Typography from '@mui/material/Typography'; +import Box from '@mui/material/Box'; +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import Grid from '@mui/material/Grid'; +import Button from '@mui/material/Button'; +import Chip from '@mui/material/Chip'; +import Divider from '@mui/material/Divider'; +import LinearProgress from '@mui/material/LinearProgress'; +import Tooltip from '@mui/material/Tooltip'; +import IconButton from '@mui/material/IconButton'; +import Collapse from '@mui/material/Collapse'; +import DownloadIcon from '@mui/icons-material/Download'; +import ArrowBackIcon from '@mui/icons-material/ArrowBack'; +import RefreshIcon from '@mui/icons-material/Refresh'; +import ContentCopyIcon from '@mui/icons-material/ContentCopy'; +import CheckCircleOutlineIcon from '@mui/icons-material/CheckCircleOutline'; +import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; +import HourglassTopIcon from '@mui/icons-material/HourglassTop'; +import ScheduleIcon from '@mui/icons-material/Schedule'; +import FolderOpenIcon from '@mui/icons-material/FolderOpen'; +import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; +import ExpandLessIcon from '@mui/icons-material/ExpandLess'; +import SearchIcon from '@mui/icons-material/Search'; +import CommentIcon from '@mui/icons-material/Comment'; +import TextSnippetIcon from '@mui/icons-material/TextSnippet'; +import MergeTypeIcon from '@mui/icons-material/MergeType'; +import PlayArrowIcon from '@mui/icons-material/PlayArrow'; +import { useParams, useNavigate } from 'react-router-dom'; +import { useTaskStore } from '../store/useTaskStore'; +import { api } from '../api/client'; +import StatusBadge from '../components/task/StatusBadge'; +import LoadingOverlay from '../components/shared/LoadingOverlay'; +import ErrorAlert from '../components/shared/ErrorAlert'; +import { formatDateTime, formatDuration } from '../utils/format'; +import { TASK_TYPE_LABELS } from '../utils/constants'; +import { useWebSocket } from '../hooks/useWebSocket'; + +// ─── 步骤定义 ──────────────────────────────────────────────── + +const STEP_DEFS: Record = { + search: { label: '搜索采集', steps: ['初始化环境', '关键词搜索', '数据清洗', '保存结果'] }, + comments: { label: '评论采集', steps: ['初始化环境', '读取视频列表', '批量采集评论', '保存结果'] }, + scripts: { label: '文案提取', steps: ['初始化环境', '下载音频', '语音识别', '保存结果'] }, + merge: { label: '数据合并', steps: ['读取数据文件', '数据校验', '合并处理', '输出 CSV'] }, + run_all: { label: '一键全流程', steps: ['搜索采集', '评论采集', '文案提取', '数据合并'] }, +}; + +const TASK_ICONS: Record = { + search: , + comments: , + scripts: , + merge: , + run_all: , +}; + +// ─── 颜色 ───────────────────────────────────────────────────── + +const STATUS_COLORS: Record = { + pending: { bg: '#f5f5f5', color: '#757575', border: '#bdbdbd' }, + running: { bg: '#e3f2fd', color: '#1565c0', border: '#1976d2' }, + completed: { bg: '#e8f5e9', color: '#2e7d32', border: '#43a047' }, + failed: { bg: '#ffebee', color: '#c62828', border: '#e53935' }, +}; + +// ─── 时间线步骤组件 ─────────────────────────────────────────── + +function TaskTimeline({ + taskType, + status, + progress, + startedAt, + completedAt, + logs, +}: { + taskType: string; + status: string; + progress: string; + startedAt: string | null; + completedAt: string | null; + logs?: Array<{ id: number; timestamp: string; level: string; message: string }>; +}) { + const def = STEP_DEFS[taskType]; + if (!def) return null; + const steps = def.steps; + + // 根据 progress 文本粗略判断当前进行到第几步 + const currentStep = + status === 'completed' ? steps.length : + status === 'failed' ? -1 : + status === 'running' ? Math.max(1, Math.min(steps.length - 1, Math.round(steps.length / 2))) : + 0; + + return ( + + {steps.map((step, idx) => { + const done = status === 'completed' || idx < currentStep; + const active = status === 'running' && idx === currentStep; + const failed = status === 'failed' && idx === currentStep; + const pending = !done && !active && !failed; + + return ( + + {/* 竖线 + 圆点 */} + + + {done ? '✓' : failed ? '✕' : idx + 1} + + {idx < steps.length - 1 && ( + + )} + + + {/* 步骤内容 */} + + + {step} + {active && ( + + )} + + {active && progress && ( + + {progress} + + )} + + + ); + })} + + ); +} + +// ─── 参数展示组件 ───────────────────────────────────────────── + +function ParamsDisplay({ params }: { params: Record }) { + const [expanded, setExpanded] = useState(false); + + const renderValue = (key: string, val: any) => { + if (Array.isArray(val)) { + return ( + + {val.map((v, i) => ( + + ))} + + ); + } + if (val === null || val === undefined || val === '') { + return —; + } + return {String(val)}; + }; + + const PARAM_LABELS: Record = { + keywords: '关键词', + max_count: '每词数量', + steps: '执行步骤', + project_dir: '工作目录', + video_jsonl: '视频文件', + model: 'Whisper 模型', + output_csv: '输出 CSV', + }; + + const entries = Object.entries(params).filter(([, v]) => v !== null && v !== undefined); + const visible = expanded ? entries : entries.slice(0, 4); + + return ( + + + {visible.map(([k, v]) => ( + + + {PARAM_LABELS[k] || k} + + {renderValue(k, v)} + + ))} + + {entries.length > 4 && ( + + )} + + ); +} + +// ─── 结果文件组件 ────────────────────────────────────────────── + +function ResultSection({ task }: { task: any }) { + if (task.status !== 'completed') return null; + + const result = task.result; + const resultEntries = result + ? Object.entries(result).filter(([k]) => k !== 'status') + : []; + + return ( + + + + + 任务已完成 + + + {resultEntries.length > 0 && ( + + {resultEntries.map(([k, v]) => ( + + {k} + + {String(v)} + + + ))} + + )} + + + + + ); +} + +// ─── 错误组件 ───────────────────────────────────────────────── + +function ErrorSection({ error, exitCode }: { error: string; exitCode: number }) { + const [showRaw, setShowRaw] = useState(false); + + const exitCodeLabel: Record = { + 1: { label: '可重试 (exit 1)', color: 'warning' }, + 2: { label: '参数错误 (exit 2)', color: 'error' }, + 3: { label: '致命错误 (exit 3)', color: 'error' }, + 4: { label: '服务关闭 (exit 4)', color: 'info' }, + }; + + const cfg = exitCodeLabel[exitCode] || { label: `exit ${exitCode}`, color: 'error' as const }; + const isInterrupted = exitCode === 4; + + return ( + + + + + + + {isInterrupted ? '任务被中断' : '任务失败'} + + + {exitCode > 0 && ( + + )} + + + {isInterrupted && ( + + 容器重启或进程退出导致任务中断。已采集的数据保存在 workspace 中，可以重新执行任务。 + + )} + + + {error.split('\n')[0]} + + + {error.includes('\n') && ( + <> + + + + {error} + + + + )} + + + ); +} + +// ─── 主组件 ─────────────────────────────────────────────────── + +export default function TaskDetailPage() { + const { taskId } = useParams<{ taskId: string }>(); + const navigate = useNavigate(); + const { currentTask, loading, error, fetchTaskDetail, clearError } = useTaskStore(); + const [elapsed, setElapsed] = useState(''); + const [copied, setCopied] = useState(false); + const { logs, clearLogs } = useWebSocket(); + const [logPanelOpen, setLogPanelOpen] = useState(false); + const pollRef = useRef>(); + + // 初始加载 + useEffect(() => { + if (taskId) fetchTaskDetail(taskId); + }, [taskId, fetchTaskDetail]); + + // running 时每 3s 轮询一次 + useEffect(() => { + if (currentTask?.status === 'running') { + pollRef.current = setInterval(() => { + if (taskId) fetchTaskDetail(taskId); + }, 3000); + } else { + clearInterval(pollRef.current); + } + return () => clearInterval(pollRef.current); + }, [currentTask?.status, taskId, fetchTaskDetail]); + + // 实时计时 + useEffect(() => { + if (!currentTask || currentTask.status !== 'running') return; + const timer = setInterval(() => { + setElapsed(formatDuration(currentTask.started_at, null)); + }, 1000); + return () => clearInterval(timer); + }, [currentTask]); + + const handleCopyId = () => { + if (!currentTask) return; + navigator.clipboard.writeText(currentTask.task_id); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + }; + + if (loading && !currentTask) return ; + if (!currentTask) return ( + + 任务不存在或已被删除 + + + ); + + const sc = STATUS_COLORS[currentTask.status] || STATUS_COLORS.pending; + const typeLabel = TASK_TYPE_LABELS[currentTask.task_type] || currentTask.task_type; + const isRunning = currentTask.status === 'running'; + const isFailed = currentTask.status === 'failed'; + const isDone = currentTask.status === 'completed'; + + return ( + + {/* 顶部导航 */} + + + + + taskId && fetchTaskDetail(taskId)}> + + + + + + {error && } + + {/* ── 状态大卡片 ── */} + + {isRunning && ( + + )} + + + {/* 任务类型图标 */} + + {TASK_ICONS[currentTask.task_type] || } + + + + {/* 类型 + 状态 */} + + + {typeLabel} + + + {isRunning && elapsed && ( + } + label={elapsed} + size="small" + color="primary" + variant="outlined" + /> + )} + + + {/* Task ID */} + + + #{currentTask.task_id} + + + + + + + + + {/* 进度文字 */} + {currentTask.progress && ( + + {currentTask.progress} + + )} + + + {/* 完成时的耗时 */} + {(isDone || isFailed) && currentTask.started_at && ( + + 耗时 + + {formatDuration(currentTask.started_at, currentTask.completed_at)} + + + )} + + + + + {/* ── 主体双栏 ── */} + + + {/* 左列：执行进度 + 时间线 */} + + {/* 执行时间线 */} + + + + 执行步骤 + + + + + + + {/* 时间信息 */} + + + + 时间记录 + + + + } label="创建" value={formatDateTime(currentTask.created_at)} /> + } label="开始" value={formatDateTime(currentTask.started_at)} /> + + : } + label="结束" + value={formatDateTime(currentTask.completed_at)} + hidden={isRunning} + /> + {!isRunning && currentTask.started_at && ( + } + label="耗时" + value={formatDuration(currentTask.started_at, currentTask.completed_at)} + /> + )} + + + + + + {/* 右列：参数 + 结果/错误 + workspace */} + + {/* 参数 */} + + + + 执行参数 + + + + + + + {/* 实时日志面板（任务运行中时显示） */} + {isRunning && ( + + + + + {logs.length > 0 && ( + + )} + + {logs.length > 0 && ( + + + 清空 + + + )} + + + + + {logs.length === 0 ? ( + + ⏳ 等待日志输出... + + ) : ( + logs.map((log) => { + const lvl = log.level || 'info'; + const color = + lvl === 'error' ? '#c62828' : + lvl === 'warning' ? '#e65100' : + lvl === 'success' ? '#2e7d32' : + lvl === 'debug' ? '#6a1b9a' : + '#263238'; + return ( + + + {log.timestamp} + + + {log.message} + + + ); + }) + )} + + + + + )} + + {/* 结果 */} + {isDone && ( + + + + )} + + {/* 错误 */} + {isFailed && currentTask.error && ( + + + + )} + + {/* Workspace */} + + + + + 工作目录 + + + {currentTask.workspace} + + + + + + + ); +} + +// ─── 辅助组件 ───────────────────────────────────────────────── + +function TimeRow({ + icon, + label, + value, + hidden, +}: { + icon: React.ReactNode; + label: string; + value: string; + hidden?: boolean; +}) { + if (hidden) return null; + return ( + + {icon} + + {label} + + + {value} + + + ); +} diff --git a/web/src/pages/TaskListPage.tsx b/web/src/pages/TaskListPage.tsx new file mode 100644 index 000000000..406f58d52 --- /dev/null +++ b/web/src/pages/TaskListPage.tsx @@ -0,0 +1,341 @@ +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import Typography from '@mui/material/Typography'; +import Box from '@mui/material/Box'; +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import Table from '@mui/material/Table'; +import TableBody from '@mui/material/TableBody'; +import TableCell from '@mui/material/TableCell'; +import TableHead from '@mui/material/TableHead'; +import TableRow from '@mui/material/TableRow'; +import TablePagination from '@mui/material/TablePagination'; +import FormControl from '@mui/material/FormControl'; +import InputLabel from '@mui/material/InputLabel'; +import Select from '@mui/material/Select'; +import MenuItem from '@mui/material/MenuItem'; +import IconButton from '@mui/material/IconButton'; +import Tooltip from '@mui/material/Tooltip'; +import Chip from '@mui/material/Chip'; +import LinearProgress from '@mui/material/LinearProgress'; +import Alert from '@mui/material/Alert'; +import Button from '@mui/material/Button'; +import DeleteIcon from '@mui/icons-material/Delete'; +import DownloadIcon from '@mui/icons-material/Download'; +import VisibilityIcon from '@mui/icons-material/Visibility'; +import AddIcon from '@mui/icons-material/Add'; +import RefreshIcon from '@mui/icons-material/Refresh'; +import { useNavigate, useSearchParams } from 'react-router-dom'; +import { useTaskStore } from '../store/useTaskStore'; +import { api } from '../api/client'; +import StatusBadge from '../components/task/StatusBadge'; +import TaskTypeIcon from '../components/task/TaskTypeIcon'; +import DeleteConfirmDialog from '../components/task/DeleteConfirmDialog'; +import LoadingOverlay from '../components/shared/LoadingOverlay'; +import ErrorAlert from '../components/shared/ErrorAlert'; +import { formatDateTime, formatDuration } from '../utils/format'; +import { PAGE_SIZE } from '../utils/constants'; + +// TaskListPage 内部用于计算耗时的辅助函数 +function calcDuration(startedAt: string | null, completedAt: string | null): string { + if (!startedAt) return ''; + return formatDuration(startedAt, completedAt); +} + +const POLL_INTERVAL = 5000; // 5s 轮询 + +export default function TaskListPage() { + const navigate = useNavigate(); + const [searchParams] = useSearchParams(); + const { tasks, stats, loading, error, fetchTasks, deleteTask, clearError } = useTaskStore(); + + const [statusFilter, setStatusFilter] = useState(searchParams.get('status') || ''); + const [typeFilter, setTypeFilter] = useState(''); + const [page, setPage] = useState(0); + const [deleteTarget, setDeleteTarget] = useState(null); + const [refreshing, setRefreshing] = useState(false); + + const pollTimerRef = useRef | null>(null); + + const loadTasks = useCallback(async (silent = false) => { + if (!silent) setRefreshing(true); + const params: any = { limit: PAGE_SIZE, offset: page * PAGE_SIZE }; + if (statusFilter) params.status = statusFilter; + if (typeFilter) params.task_type = typeFilter; + await fetchTasks(params); + if (!silent) setRefreshing(false); + }, [statusFilter, typeFilter, page, fetchTasks]); + + // 首次 & 条件变化时加载 + useEffect(() => { + loadTasks(); + }, [loadTasks]); + + // 有运行中任务时自动轮询 + useEffect(() => { + const hasRunning = tasks.some((t) => t.status === 'running' || t.status === 'pending'); + if (hasRunning) { + pollTimerRef.current = setTimeout(() => loadTasks(true), POLL_INTERVAL); + } + return () => { + if (pollTimerRef.current) clearTimeout(pollTimerRef.current); + }; + }, [tasks, loadTasks]); + + const handleDelete = async () => { + if (deleteTarget) { + await deleteTask(deleteTarget); + setDeleteTarget(null); + } + }; + + const runningCount = stats?.running ?? 0; + const pendingCount = stats?.pending ?? 0; + + return ( + + {/* 顶部标题栏 */} + + 任务列表 + + + + + {/* 后台执行提示 */} + {(runningCount > 0 || pendingCount > 0) && ( + + + + + {runningCount > 0 && `${runningCount} 个任务正在后台执行`} + {runningCount > 0 && pendingCount > 0 && '，'} + {pendingCount > 0 && `${pendingCount} 个等待中`} +  — 离开页面不影响后台执行，每 5 秒自动刷新 + + + + )} + + {error && } + + + + {/* 筛选栏 */} + + + 状态 + + + + 类型 + + + + + 共 {stats?.total ?? 0} 个任务 + + + + + {loading && tasks.length === 0 ? : ( + <> + {/* 有运行中任务时显示全局进度条 */} + {(loading || refreshing) && ( + + )} + + + + Task ID + 类型 + 状态 + 关键词 + 创建时间 + 耗时 + 操作 + + + + {tasks.length === 0 ? ( + + + + 暂无任务 + + + + + ) : ( + tasks.map((task) => { + const isRunning = task.status === 'running'; + const isPending = task.status === 'pending'; + const keywords = task.params?.keywords as string[] | undefined; + + // 计算耗时 + const duration = calcDuration(task.started_at, task.completed_at); + + return ( + + + navigate(`/tasks/${task.task_id}`)} + > + {task.task_id.slice(0, 8)} + + + + + + + + + + {keywords && keywords.length > 0 ? ( + + {keywords.slice(0, 3).map((kw) => ( + + ))} + {keywords.length > 3 && ( + + )} + + ) : ( + — + )} + + + {formatDateTime(task.created_at)} + + + + {duration || '—'} + + + + + navigate(`/tasks/${task.task_id}`)}> + + + + {task.status === 'completed' && ( + + window.open(api.getResultUrl(task.task_id))} + > + + + + )} + + + setDeleteTarget(task.task_id)} + disabled={isRunning} + > + + + + + + + ); + }) + )} + +
+ setPage(p)} + rowsPerPage={PAGE_SIZE} + rowsPerPageOptions={[PAGE_SIZE]} + labelDisplayedRows={({ from, to, count }) => `${from}–${to} / ${count}`} + /> + + )} + + + + setDeleteTarget(null)} + /> + + ); +} diff --git a/web/src/store/useSettingsStore.ts b/web/src/store/useSettingsStore.ts new file mode 100644 index 000000000..507a9df23 --- /dev/null +++ b/web/src/store/useSettingsStore.ts @@ -0,0 +1,36 @@ +import { create } from 'zustand'; +import { API_KEY_STORAGE_KEY, API_BASE_URL_STORAGE_KEY, DEFAULT_API_BASE } from '../utils/constants'; +import { resetClient } from '../api/client'; + +interface SettingsStore { + apiKey: string; + apiBaseUrl: string; + setApiKey: (key: string) => void; + setApiBaseUrl: (url: string) => void; + loadFromStorage: () => void; +} + +export const useSettingsStore = create((set) => ({ + apiKey: '', + apiBaseUrl: DEFAULT_API_BASE, + + setApiKey: (key) => { + localStorage.setItem(API_KEY_STORAGE_KEY, key); + set({ apiKey: key }); + }, + + setApiBaseUrl: (url) => { + localStorage.setItem(API_BASE_URL_STORAGE_KEY, url); + resetClient(url); + set({ apiBaseUrl: url }); + }, + + loadFromStorage: () => { + const key = localStorage.getItem(API_KEY_STORAGE_KEY) || ''; + const url = localStorage.getItem(API_BASE_URL_STORAGE_KEY) || DEFAULT_API_BASE; + set({ apiKey: key, apiBaseUrl: url }); + if (url !== DEFAULT_API_BASE) { + resetClient(url); + } + }, +})); diff --git a/web/src/store/useTaskStore.ts b/web/src/store/useTaskStore.ts new file mode 100644 index 000000000..529ec8b82 --- /dev/null +++ b/web/src/store/useTaskStore.ts @@ -0,0 +1,116 @@ +import { create } from 'zustand'; +import { api } from '../api/client'; +import type { TaskInfo, TaskStats } from '../api/types'; + +interface TaskStore { + tasks: TaskInfo[]; + stats: TaskStats | null; + loading: boolean; + error: string | null; + currentTask: TaskInfo | null; + + fetchTasks: (params?: { task_type?: string; status?: string; limit?: number; offset?: number }) => Promise; + fetchTaskDetail: (taskId: string) => Promise; + createTask: (type: string, params: any) => Promise; + deleteTask: (taskId: string) => Promise; + cleanupTasks: (hours: number) => Promise; + upsertTask: (task: TaskInfo) => void; + updateTaskStatus: (taskId: string, status: string, progress?: string) => void; + setCurrentTask: (task: TaskInfo | null) => void; + clearError: () => void; +} + +export const useTaskStore = create((set, get) => ({ + tasks: [], + stats: null, + loading: false, + error: null, + currentTask: null, + + fetchTasks: async (params) => { + set({ loading: true, error: null }); + try { + const resp = await api.listTasks(params); + set({ tasks: resp.tasks, stats: resp.stats, loading: false }); + } catch (e: any) { + set({ error: e.message || '获取任务列表失败', loading: false }); + } + }, + + fetchTaskDetail: async (taskId) => { + set({ loading: true, error: null }); + try { + const task = await api.getTaskStatus(taskId); + set({ currentTask: task, loading: false }); + } catch (e: any) { + set({ error: e.message || '获取任务详情失败', loading: false }); + } + }, + + createTask: async (type, params) => { + set({ loading: true, error: null }); + try { + let resp; + switch (type) { + case 'search': resp = await api.createSearch(params); break; + case 'comments': resp = await api.createComments(params); break; + case 'scripts': resp = await api.createScripts(params); break; + case 'merge': resp = await api.createMerge(params); break; + case 'run_all': resp = await api.createRunAll(params); break; + default: throw new Error(`未知任务类型: ${type}`); + } + set({ loading: false }); + return resp.task_id; + } catch (e: any) { + set({ error: e.message || '创建任务失败', loading: false }); + return null; + } + }, + + deleteTask: async (taskId) => { + try { + await api.deleteTask(taskId); + set((s) => ({ tasks: s.tasks.filter((t) => t.task_id !== taskId) })); + } catch (e: any) { + set({ error: e.message || '删除任务失败' }); + } + }, + + cleanupTasks: async (hours) => { + try { + const resp = await api.cleanupTasks(hours); + return resp.removed || 0; + } catch (e: any) { + set({ error: e.message || '清理任务失败' }); + return 0; + } + }, + + upsertTask: (task) => { + set((s) => { + const idx = s.tasks.findIndex((t) => t.task_id === task.task_id); + if (idx >= 0) { + const newTasks = [...s.tasks]; + newTasks[idx] = task; + return { tasks: newTasks }; + } + return { tasks: [task, ...s.tasks] }; + }); + }, + + updateTaskStatus: (taskId, status, progress) => { + set((s) => ({ + tasks: s.tasks.map((t) => + t.task_id === taskId + ? { ...t, status: status as TaskInfo['status'], progress: progress ?? t.progress } + : t + ), + currentTask: s.currentTask?.task_id === taskId + ? { ...s.currentTask, status: status as TaskInfo['status'], progress: progress ?? s.currentTask.progress } + : s.currentTask, + })); + }, + + setCurrentTask: (task) => set({ currentTask: task }), + clearError: () => set({ error: null }), +})); diff --git a/web/src/theme/theme.ts b/web/src/theme/theme.ts new file mode 100644 index 000000000..61352550f --- /dev/null +++ b/web/src/theme/theme.ts @@ -0,0 +1,35 @@ +import { createTheme } from '@mui/material/styles'; + +const theme = createTheme({ + palette: { + primary: { main: '#1976d2' }, + secondary: { main: '#9c27b0' }, + success: { main: '#2e7d32' }, + error: { main: '#d32f2f' }, + warning: { main: '#ed6c02' }, + background: { default: '#f5f5f5' }, + }, + typography: { + fontFamily: '"Inter", "system-ui", sans-serif', + h4: { fontWeight: 700 }, + h5: { fontWeight: 600 }, + h6: { fontWeight: 600 }, + }, + components: { + MuiCard: { + styleOverrides: { + root: { borderRadius: 12, boxShadow: '0 1px 3px rgba(0,0,0,0.08)' }, + }, + }, + MuiButton: { + styleOverrides: { + root: { textTransform: 'none', fontWeight: 600, borderRadius: 8 }, + }, + }, + MuiChip: { + styleOverrides: { root: { fontWeight: 500 } }, + }, + }, +}); + +export default theme; diff --git a/web/src/utils/constants.ts b/web/src/utils/constants.ts new file mode 100644 index 000000000..598ae8f00 --- /dev/null +++ b/web/src/utils/constants.ts @@ -0,0 +1,36 @@ +export const DEFAULT_API_BASE = 'http://localhost:8000'; +export const API_KEY_HEADER = 'X-API-Key'; +export const API_KEY_STORAGE_KEY = 'mc_api_key'; +export const API_BASE_URL_STORAGE_KEY = 'mc_api_base_url'; +export const WS_BASE = 'ws://localhost:8000'; + +export const STATUS_CONFIG = { + pending: { color: '#9e9e9e', bg: '#f5f5f5', label: '待执行' }, + running: { color: '#1976d2', bg: '#e3f2fd', label: '执行中', pulse: true }, + completed: { color: '#2e7d32', bg: '#e8f5e9', label: '已完成' }, + failed: { color: '#d32f2f', bg: '#ffebee', label: '失败' }, +} as const; + +export type TaskStatus = keyof typeof STATUS_CONFIG; + +export const TASK_TYPE_LABELS: Record = { + search: '搜索采集', + comments: '评论采集', + scripts: '文案提取', + merge: '数据合并', + run_all: '一键全流程', +}; + +export const WHISPER_MODELS = [ + { value: 'tiny', label: 'Tiny (最快)' }, + { value: 'base', label: 'Base' }, + { value: 'small', label: 'Small (推荐)' }, + { value: 'medium', label: 'Medium' }, + { value: 'large', label: 'Large (最准)' }, +] as const; + +export const PAGE_SIZE = 20; +export const WS_RECONNECT_BASE_MS = 1000; +export const WS_RECONNECT_MAX_MS = 30000; +export const WS_MAX_RETRIES = 5; +export const POLLING_INTERVAL_MS = 5000; diff --git a/web/src/utils/format.ts b/web/src/utils/format.ts new file mode 100644 index 000000000..e972fd3f1 --- /dev/null +++ b/web/src/utils/format.ts @@ -0,0 +1,31 @@ +import dayjs from 'dayjs'; +import duration from 'dayjs/plugin/duration'; +import relativeTime from 'dayjs/plugin/relativeTime'; + +dayjs.extend(duration); +dayjs.extend(relativeTime); + +export function formatDateTime(iso: string | null | undefined): string { + if (!iso) return '-'; + return dayjs(iso).format('YYYY-MM-DD HH:mm'); +} + +export function formatDuration(startIso: string | null, endIso: string | null): string { + if (!startIso) return '-'; + const end = endIso ? dayjs(endIso) : dayjs(); + const ms = end.diff(dayjs(startIso)); + if (ms < 0) return '-'; + const d = dayjs.duration(ms); + const h = Math.floor(d.asHours()); + const m = d.minutes(); + const s = d.seconds(); + if (h > 0) return `${h}h ${m}m ${s}s`; + if (m > 0) return `${m}m ${s}s`; + return `${s}s`; +} + +export function formatFileSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} diff --git a/web/src/vite-env.d.ts b/web/src/vite-env.d.ts new file mode 100644 index 000000000..11f02fe2a --- /dev/null +++ b/web/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/web/tailwind.config.ts b/web/tailwind.config.ts new file mode 100644 index 000000000..3db70e165 --- /dev/null +++ b/web/tailwind.config.ts @@ -0,0 +1,14 @@ +import type { Config } from 'tailwindcss'; + +export default { + content: ['./index.html', './src/**/*.{js,ts,jsx,tsx}'], + theme: { + extend: { + fontFamily: { + sans: ['Inter', 'system-ui', 'sans-serif'], + mono: ['JetBrains Mono', 'monospace'], + }, + }, + }, + plugins: [require('@tailwindcss/typography')], +} satisfies Config; diff --git a/web/tsconfig.json b/web/tsconfig.json new file mode 100644 index 000000000..75e2ff7ac --- /dev/null +++ b/web/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noFallthroughCasesInSwitch": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src"] +} diff --git a/web/tsconfig.node.json b/web/tsconfig.node.json new file mode 100644 index 000000000..59e73c851 --- /dev/null +++ b/web/tsconfig.node.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2023"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "strict": true + }, + "include": ["vite.config.ts"] +} diff --git a/web/vite.config.ts b/web/vite.config.ts new file mode 100644 index 000000000..7291dfad1 --- /dev/null +++ b/web/vite.config.ts @@ -0,0 +1,19 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; + +export default defineConfig({ + plugins: [react()], + base: '/ui/', + build: { + outDir: '../api/webui', + emptyOutDir: true, + }, + server: { + port: 5173, + proxy: { + '/scrape': 'http://localhost:8000', + '/health': 'http://localhost:8000', + '/ws': { target: 'ws://localhost:8000', ws: true }, + }, + }, +}); From 31ceb012c1c599f78dd943ddeea55cb48122fe15 Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 14:17:04 +0800 Subject: [PATCH 09/32] chore(web): align frontend API and dev ports --- web/src/utils/constants.ts | 4 ++-- web/vite.config.ts | 37 ++++++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/web/src/utils/constants.ts b/web/src/utils/constants.ts index 598ae8f00..3622cb203 100644 --- a/web/src/utils/constants.ts +++ b/web/src/utils/constants.ts @@ -1,8 +1,8 @@ -export const DEFAULT_API_BASE = 'http://localhost:8000'; +export const DEFAULT_API_BASE = import.meta.env.VITE_API_BASE_URL || 'http://localhost:18080'; export const API_KEY_HEADER = 'X-API-Key'; export const API_KEY_STORAGE_KEY = 'mc_api_key'; export const API_BASE_URL_STORAGE_KEY = 'mc_api_base_url'; -export const WS_BASE = 'ws://localhost:8000'; +export const WS_BASE = import.meta.env.VITE_WS_BASE_URL || 'ws://localhost:18080'; export const STATUS_CONFIG = { pending: { color: '#9e9e9e', bg: '#f5f5f5', label: '待执行' }, diff --git a/web/vite.config.ts b/web/vite.config.ts index 7291dfad1..a9ed7763f 100644 --- a/web/vite.config.ts +++ b/web/vite.config.ts @@ -1,19 +1,26 @@ -import { defineConfig } from 'vite'; +import { defineConfig, loadEnv } from 'vite'; import react from '@vitejs/plugin-react'; -export default defineConfig({ - plugins: [react()], - base: '/ui/', - build: { - outDir: '../api/webui', - emptyOutDir: true, - }, - server: { - port: 5173, - proxy: { - '/scrape': 'http://localhost:8000', - '/health': 'http://localhost:8000', - '/ws': { target: 'ws://localhost:8000', ws: true }, +export default defineConfig(({ mode }) => { + const env = loadEnv(mode, process.cwd(), ''); + const apiTarget = env.VITE_API_BASE_URL || 'http://localhost:18080'; + const wsTarget = env.VITE_WS_BASE_URL || 'ws://localhost:18080'; + const devPort = Number(env.WEB_DEV_PORT || 15173); + + return { + plugins: [react()], + base: '/ui/', + build: { + outDir: '../api/webui', + emptyOutDir: true, + }, + server: { + port: devPort, + proxy: { + '/scrape': apiTarget, + '/health': apiTarget, + '/ws': { target: wsTarget, ws: true }, + }, }, - }, + }; }); From f88a0ff4b90cac3469a0dca8d7760a1813f08a4f Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 14:17:55 +0800 Subject: [PATCH 10/32] feat(web): add content asset preview and download UX --- web/src/api/client.ts | 11 +- web/src/api/types.ts | 1 + web/src/pages/DataPage.tsx | 172 +++++++++++++++++++++++++------ web/src/pages/TaskDetailPage.tsx | 69 ++++++++++++- 4 files changed, 218 insertions(+), 35 deletions(-) diff --git a/web/src/api/client.ts b/web/src/api/client.ts index b610949a7..b10cc2e75 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -89,13 +89,20 @@ export const api = { return `${base}/scrape/result/${taskId}`; }, + getDataExportUrl: (taskId: string) => { + const base = localStorage.getItem(API_BASE_URL_STORAGE_KEY) || DEFAULT_API_BASE; + return `${base}/scrape/data/export?task_id=${encodeURIComponent(taskId)}`; + }, + // ─── 数据管理 ─────────────────────────────────────────────── listDataFiles: () => getClient().get('scrape/data/list').json(), - previewData: (taskId: string) => - getClient().get(`scrape/data/preview/${taskId}`).json(), + previewData: (taskId: string, limit?: number) => + getClient().get(`scrape/data/preview/${taskId}`, { + searchParams: limit ? { limit } : undefined, + }).json(), /** 导出数据，返回 Blob 供前端触发下载 */ exportData: async (req: ExportRequest): Promise<{ blob: Blob; filename: string }> => { diff --git a/web/src/api/types.ts b/web/src/api/types.ts index c56396792..868484410 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -122,6 +122,7 @@ export interface DataListResponse { export interface DataPreviewResponse { task_id: string; file_name: string; + format?: 'csv' | 'jsonl' | string; rows: Record[]; total_rows: number; } diff --git a/web/src/pages/DataPage.tsx b/web/src/pages/DataPage.tsx index 0eb26c3d3..76046a564 100644 --- a/web/src/pages/DataPage.tsx +++ b/web/src/pages/DataPage.tsx @@ -37,6 +37,7 @@ import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import ExpandLessIcon from '@mui/icons-material/ExpandLess'; import ArticleIcon from '@mui/icons-material/Article'; import GridOnIcon from '@mui/icons-material/GridOn'; +import ViewColumnIcon from '@mui/icons-material/ViewColumn'; import { api } from '../api/client'; import type { DataFileItem, DataPreviewResponse } from '../api/types'; @@ -44,6 +45,83 @@ import { formatDateTime, formatFileSize } from '../utils/format'; import TaskTypeIcon from '../components/task/TaskTypeIcon'; import ErrorAlert from '../components/shared/ErrorAlert'; +const PRIMARY_FILE_PRIORITY = [ + 'content_asset.csv', + 'script_clean.csv', + 'comments_clean.csv', + 'search_result.csv', + 'content_asset.jsonl', + 'script_clean.jsonl', + 'comments_clean.jsonl', + 'search_result.jsonl', +]; + +const CONTENT_ASSET_CORE_COLUMNS = [ + 'source_keyword', + 'platform', + 'aweme_id', + 'clean_title', + 'topic', + 'pain_point', + 'teaching_angle', + 'nickname', + 'liked_count', + 'collected_count', + 'comment_count', + 'share_count', + 'total_engagement', + 'valid_comment_count', + 'comment_data_status', + 'asr_data_status', + 'asset_quality', +]; + +const LONG_TEXT_COLUMNS = new Set([ + 'script_text', + 'script_clean_text', + 'top_valid_comments', + 'comment_pain_tags', + 'desc', + 'clean_desc', +]); + +function filePriority(item: DataFileItem): number { + const exact = PRIMARY_FILE_PRIORITY.indexOf(item.file_name); + if (exact >= 0) return exact; + if (item.file_name.endsWith('.csv')) return PRIMARY_FILE_PRIORITY.length; + if (item.file_name.endsWith('.jsonl')) return PRIMARY_FILE_PRIORITY.length + 1; + return PRIMARY_FILE_PRIORITY.length + 2; +} + +function selectPrimaryFiles(items: DataFileItem[]): DataFileItem[] { + const grouped = new Map(); + items + .filter((item) => item.file_name !== 'execution_log.jsonl') + .forEach((item) => { + const group = grouped.get(item.task_id) || []; + group.push(item); + grouped.set(item.task_id, group); + }); + + return Array.from(grouped.values()).map((files) => + [...files].sort((a, b) => filePriority(a) - filePriority(b))[0] + ); +} + +function fileLabel(item: DataFileItem): string { + if (item.file_name.startsWith('content_asset.') || item.task_type === 'merge') { + return '内容资产表'; + } + if (item.file_name.startsWith('search_result.')) return '搜索结果表'; + if (item.file_name.startsWith('comments_clean.')) return '评论清洗表'; + if (item.file_name.startsWith('script_clean.')) return '文案清洗表'; + return '结果文件'; +} + +function fileFormat(fileName: string): string { + return fileName.split('.').pop()?.toUpperCase() || 'FILE'; +} + // ─── 预览对话框 ──────────────────────────────────────────────── function PreviewDialog({ @@ -52,18 +130,25 @@ function PreviewDialog({ const [data, setData] = useState(null); const [loading, setLoading] = useState(false); const [err, setErr] = useState(''); + const [showAllColumns, setShowAllColumns] = useState(false); useEffect(() => { if (!open || !taskId) return; setLoading(true); setErr(''); - api.previewData(taskId) + setData(null); + setShowAllColumns(false); + api.previewData(taskId, 20) .then(setData) .catch((e) => setErr(e.message || '预览失败')) .finally(() => setLoading(false)); }, [open, taskId]); - const columns = data?.rows?.[0] ? Object.keys(data.rows[0]) : []; + const allColumns = data?.rows?.[0] ? Object.keys(data.rows[0]) : []; + const isContentAsset = data?.file_name.startsWith('content_asset.') ?? false; + const columns = isContentAsset && !showAllColumns + ? CONTENT_ASSET_CORE_COLUMNS.filter((column) => allColumns.includes(column)) + : allColumns; return ( @@ -72,12 +157,31 @@ function PreviewDialog({ 数据预览 {data && ( - + + + + )} + + {isContentAsset && data && ( + )} @@ -103,7 +207,8 @@ function PreviewDialog({ setExportOpen(true)} > - 导出选中 {selected.size > 0 && `(${selected.size})`} + 批量导出旧七字段 {selected.size > 0 && `(${selected.size})`} @@ -412,7 +517,8 @@ export default function DataPage() { Task ID 类型 - 关键词 + 主结果文件 + 格式 行数 文件大小 完成时间 @@ -422,7 +528,7 @@ export default function DataPage() { {items.map((item) => ( - {item.keywords && item.keywords.length > 0 ? ( - - {item.keywords.slice(0, 3).map((kw) => ( - - ))} - {item.keywords.length > 3 && ( - - )} - - ) : ( - - {item.file_name} - - )} + + {fileLabel(item)} + + + {item.file_name} + + + + @@ -485,13 +590,13 @@ export default function DataPage() { - + { - setSelected(new Set([item.task_id])); - setExportOpen(true); - }} + component="a" + href={api.getDataExportUrl(item.task_id)} + target="_blank" + rel="noreferrer" > @@ -505,10 +610,15 @@ export default function DataPage() { + + “下载原始文件”会保留 content_asset.csv 的完整字段；“批量导出旧七字段”会统一导出 + video_id、platform、script_text、likes、favorites、shares、comments。 + + {/* 格式说明卡片 */} - 导出格式说明 + 批量旧格式说明 diff --git a/web/src/pages/TaskDetailPage.tsx b/web/src/pages/TaskDetailPage.tsx index bb747bdbc..77b36510d 100644 --- a/web/src/pages/TaskDetailPage.tsx +++ b/web/src/pages/TaskDetailPage.tsx @@ -11,6 +11,8 @@ import LinearProgress from '@mui/material/LinearProgress'; import Tooltip from '@mui/material/Tooltip'; import IconButton from '@mui/material/IconButton'; import Collapse from '@mui/material/Collapse'; +import Paper from '@mui/material/Paper'; +import Alert from '@mui/material/Alert'; import DownloadIcon from '@mui/icons-material/Download'; import ArrowBackIcon from '@mui/icons-material/ArrowBack'; import RefreshIcon from '@mui/icons-material/Refresh'; @@ -249,9 +251,26 @@ function ResultSection({ task }: { task: any }) { if (task.status !== 'completed') return null; const result = task.result; + const isContentAsset = task.task_type === 'merge' || Boolean( + result?.content_asset_csv || result?.content_asset_jsonl || result?.content_asset_stats + ); + const contentAssetStats = result?.content_asset_stats; const resultEntries = result - ? Object.entries(result).filter(([k]) => k !== 'status') + ? Object.entries(result).filter(([k]) => + k !== 'status' && k !== 'content_asset_stats' + ) : []; + const statsFields = [ + ['rows_in', '输入行数'], + ['rows_out', '输出行数'], + ['comments_available', '有评论数据'], + ['scripts_available', '有文案数据'], + ['valid_comments_total', '有效评论数'], + ['asr_available', '真实 ASR 可用'], + ['fallback_script_total', '降级文案数'], + ['missing_script_total', '缺失文案数'], + ['content_asset_csv_generated', 'CSV 已生成'], + ] as const; return ( @@ -261,6 +280,17 @@ function ResultSection({ task }: { task: any }) { 任务已完成 + {isContentAsset && ( + + + 主结果：内容资产表 + + + content_asset.csv + + + )} + {resultEntries.length > 0 && ( {resultEntries.map(([k, v]) => ( @@ -274,6 +304,41 @@ function ResultSection({ task }: { task: any }) { )} + {contentAssetStats && typeof contentAssetStats === 'object' && ( + + + 内容资产统计 + + + {statsFields.map(([key, label]) => ( + contentAssetStats[key] !== undefined && ( + + + {label} + + + {typeof contentAssetStats[key] === 'boolean' + ? (contentAssetStats[key] ? '是' : '否') + : String(contentAssetStats[key])} + + + ) + ))} + + {Array.isArray(contentAssetStats.errors) && contentAssetStats.errors.length > 0 && ( + + {contentAssetStats.errors.join('；')} + + )} + + )} + From 1f74686de6be1594180862e273e4ff6b6d0552d9 Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 15:34:09 +0800 Subject: [PATCH 11/32] docs(content-asset): document schema workflow and acceptance --- README.md | 6 + api/README.md | 73 ++++++++ docs/CONTENT_ASSET.md | 400 ++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 1 + 4 files changed, 480 insertions(+) create mode 100644 api/README.md create mode 100644 docs/CONTENT_ASSET.md diff --git a/README.md b/README.md index 0ab3177bb..e09d2cdd4 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,12 @@ python main.py --help MediaCrawler 支持多种数据存储方式，包括 CSV、JSON、JSONL、Excel、SQLite 和 MySQL 数据库。 +### Content Asset 内容资产表 + +T017-5 提供 `content_asset.jsonl` 和 `content_asset.csv`，用于汇总搜索、标题清洗、评论和文案数据，并通过状态字段标明可用性与 fallback 来源。 + +完整字段、接口和验收说明：[Content Asset 数据字典与验收说明](docs/CONTENT_ASSET.md) + 📖 **详细使用说明请查看：[数据存储指南](docs/data_storage_guide.md)** diff --git a/api/README.md b/api/README.md new file mode 100644 index 000000000..4fbf2c397 --- /dev/null +++ b/api/README.md @@ -0,0 +1,73 @@ +# Content Asset API + +本页说明 Content Asset 的 task-id 构建流程、主结果选择、预览与导出契约。完整字段和状态定义见 [Content Asset 数据字典与验收说明](../docs/CONTENT_ASSET.md)。 + +## task-id merge workflow + +`POST /scrape/merge` 提供 `search_task_id` 时，会读取已完成任务的标准输出，并在新的 merge task workspace 中生成 `content_asset.jsonl` 和 `content_asset.csv`。 + +```json +{ + "search_task_id": "", + "comments_task_id": "", + "scripts_task_id": "" +} +``` + +- `search_task_id` 必需，且对应任务必须已完成并包含 `search_result.csv`。 +- `comments_task_id`、`scripts_task_id` 可选；提供时对应任务必须已完成。 +- 没有评论或 scripts task 时仍可生成资产表，并通过状态字段标记缺失或 pending。 +- merge 响应返回的 `task_id` 用于后续 status、result、preview 和 export。 + +任务完成后，status 响应的 `result` 包含： + +```text +content_asset_jsonl +content_asset_csv +content_asset_stats +``` + +## result、preview 与 export + +| 方法 | 路径 | 说明 | +|---|---|---| +| GET | `/scrape/result/{task_id}` | 下载主结果文件 | +| GET | `/scrape/data/preview/{task_id}?limit=20` | 预览与 result 相同的主结果 | +| GET | `/scrape/data/export?task_id={task_id}` | 原样下载主结果文件 | +| POST | `/scrape/data/export` | 批量归一化导出旧七字段 CSV/TXT | + +merge task 的主结果优先级为： + +```text +content_asset.csv +content_asset.jsonl +douyin_koubo_data.csv +``` + +`result`、preview 和 GET export 共用该选择语义。GET export 保留 `content_asset.csv` 的完整 schema 和原文件 BOM。 + +preview 返回： + +```text +task_id +file_name +format +total_rows +rows +``` + +## POST export 兼容字段 + +POST export 保持旧七字段兼容契约： + +```text +video_id +platform +script_text +likes +favorites +shares +comments +``` + +Content Asset 映射关系： diff --git a/docs/CONTENT_ASSET.md b/docs/CONTENT_ASSET.md new file mode 100644 index 000000000..bfd6fcfb4 --- /dev/null +++ b/docs/CONTENT_ASSET.md @@ -0,0 +1,400 @@ +# Content Asset 数据字典与验收说明 + +## 1. 定位 + +`content_asset` 是 T017-5 输出的内容资产表，用于把搜索结果、标题清洗、评论清洗和文案处理结果汇总为一行一个视频的宽表。 + +必须正确理解它的能力边界： + +- content_asset 当前是结构完整、质量可标注的内容资产表。 +- 它不等于全量真实评论和真实语音文案均已完成。 +- 真实评论仍依赖 CDP，当前真实评论 CDP 采集仍为 pending。 +- 真实 ASR 仍依赖 Whisper/ASR，当前全量真实 ASR 仍为 pending。 +- 标题或描述生成的 fallback 文案不能表述为真实语音文案。 + +## 2. 全链路流程 + +1. `POST /scrape/search` 创建 search task，生成 `search_result.csv` 和 `search_title_clean.csv`。 +2. 可选：创建 comments task，生成 `comments_clean.csv`。 +3. 可选：创建 scripts task，生成 `script_sources.csv`、`script_raw.csv` 和 `script_clean.csv`。 +4. `POST /scrape/merge` 输入 `search_task_id`，并按需输入 `comments_task_id`、`scripts_task_id`。 +5. merge task 生成 `content_asset.jsonl` 和 `content_asset.csv`。 +6. `GET /scrape/status/{merge_task_id}` 查看任务状态和结果信息。 +7. `GET /scrape/result/{merge_task_id}` 下载主结果。 +8. `GET /scrape/data/preview/{merge_task_id}` 预览主结果。 +9. `GET /scrape/data/export?task_id={merge_task_id}` 原样下载主结果。 +10. 在前端 DataPage 或 TaskDetailPage 查看和下载内容资产表。 + +`search_task_id` 模式用于生成 `content_asset`。不传 task id、改传旧 JSONL 路径参数时，`/scrape/merge` 仍保留旧合并模式；两种模式不要混淆。 + +## 3. 输入 task_id + +| 参数 | 是否必需 | 用途 | +|---|---|---| +| `search_task_id` | 是 | 提供 `search_result.csv` 和 `search_title_clean.csv`，触发 content_asset 构建模式 | +| `comments_task_id` | 否 | 提供 `comments_clean.csv` | +| `scripts_task_id` | 否 | 提供 `script_sources.csv`、`script_raw.csv`、`script_clean.csv` | +| `merge_task_id` | merge 创建后返回 | 用于 status、result、preview、export 和前端访问 | + +输入约束： + +- search task 必须已完成，且必须存在 `search_result.csv`。 +- 没有 `comments_task_id` 不应导致构建失败；`comment_data_status` 会标记为 `pending_cdp`。 +- 没有 `scripts_task_id` 不应导致构建失败；`asr_data_status` 会根据现有输入标记为 `pending_asr` 或 `missing`。 +- 提供 comments/scripts task id 时，对应任务必须已完成。 + +## 4. 输出文件 + +merge task 的输出位于自己的 workspace： + +```text +workspaces//outputs/content_asset.jsonl +workspaces//outputs/content_asset.csv +``` + +格式要求： + +- CSV 使用 UTF-8-SIG 编码，文件头带 BOM，表头顺序固定。 +- JSONL 使用 UTF-8 编码和 `ensure_ascii=False`，中文不转义。 +- 缺失的文本字段写空字符串，缺失的计数字段写 `0`。 +- 可选输入文件缺失或读取失败时，详情写入 `content_asset_stats.errors`。 + +## 5. 字段数据字典 + +| 字段 | 来源 | 含义 | 空值/默认值 | 备注 | +|---|---|---|---|---| +| `source_keyword` | `search_result.csv` | 产生该视频的搜索关键词 | 空字符串 | 原始搜索上下文 | +| `platform` | `search_result.csv` | 内容平台 | `douyin` | 输入为空时由 builder 补默认值 | +| `video_id` | `search_result.csv` | 标准视频 ID | 空字符串 | 缺失时回退使用 `aweme_id` | +| `aweme_id` | `search_result.csv` | 抖音作品 ID | 空字符串 | 关联评论和文案的优先键 | +| `aweme_url` | `search_result.csv` | 作品页面地址 | 空字符串 | 也可作为标题/文案关联兜底键 | +| `raw_title` | `search_result.csv`、`search_title_clean.csv` | 原始标题 | 空字符串 | 优先 `search_result.title`，再取 `raw_title` | +| `clean_title` | `search_title_clean.csv`、`search_result.csv` | 清洗后的标题 | 空字符串 | 优先标题清洗结果 | +| `desc` | `search_result.csv`、`search_title_clean.csv` | 原始描述 | 空字符串 | 优先 `search_result.desc`，再取 `raw_desc` | +| `clean_desc` | `search_title_clean.csv`、`search_result.csv` | 清洗后的描述 | 空字符串 | 优先标题清洗结果 | +| `topic` | `search_title_clean.csv` | 规则提取的主题 | 空字符串 | 不是 AI/LLM 生成字段 | +| `pain_point` | `search_title_clean.csv` | 规则提取的痛点 | 空字符串 | 依赖标题清洗输入质量 | +| `teaching_angle` | `search_title_clean.csv` | 规则提取的教学角度 | 空字符串 | 依赖标题清洗输入质量 | +| `nickname` | `search_result.csv` | 作者昵称 | 空字符串 | 原始搜索元数据 | +| `liked_count` | `search_result.csv` | 点赞数 | `0` | 非法或空值归零 | +| `collected_count` | `search_result.csv` | 收藏数 | `0` | 非法或空值归零 | +| `comment_count` | `search_result.csv` | 平台展示的评论总数 | `0` | 不等于本次采集到的有效评论数 | +| `share_count` | `search_result.csv` | 分享数 | `0` | 非法或空值归零 | +| `total_engagement` | `search_result.csv`、builder 派生 | 总互动数 | `0` | 输入为空或为 0 时按点赞、收藏、评论、分享求和 | +| `valid_comment_count` | `comments_clean.csv`、builder 派生 | 本次清洗后有效评论数 | `0` | 只统计 `is_valid=true` | +| `top_valid_comments` | `comments_clean.csv`、builder 派生 | 代表性有效评论 | 空字符串 | 最多取前 3 条 `clean_content`，使用竖线拼接 | +| `comment_pain_tags` | `comments_clean.csv`、builder 派生 | 有效评论中的痛点标签 | 空字符串 | 去重后使用竖线拼接 | +| `script_source_status` | `script_sources.csv` | 文案来源准备状态 | 空字符串 | 常见值为 `available`、`planned`、`missing` | +| `script_source_quality` | `script_sources.csv` | 文案来源基础质量 | 空字符串 | 常见值为 `medium`、`weak`、`low`、`missing` | +| `script_clean_text` | `script_clean.csv` | 当前可用的清洗文案 | 空字符串 | 可能是真实 ASR，也可能是 fallback，必须结合 source/status 判断 | +| `script_clean_source` | `script_clean.csv` | 清洗文案来源 | 空字符串 | `asr_raw` 才表示真实 ASR | +| `script_clean_quality` | `script_clean.csv` | 清洗文案质量标记 | 空字符串 | 上游文案清洗结果的透传值 | +| `comment_data_status` | builder 派生 | 评论数据可用状态 | `pending_cdp`、`empty` 或 `available` | 由评论文件是否存在及有效评论数决定 | +| `asr_data_status` | `script_raw.csv`、`script_clean.csv`、builder 派生 | ASR 或 fallback 使用状态 | `missing` 等 | 明确区分真实 ASR、fallback 和依赖缺失 | +| `asset_quality` | builder 派生 | 当前资产完整度/可用度 | `low` 等 | 不是绝对内容质量评分 | +| `created_at` | builder 派生 | 资产记录生成时间 | UTC 时间 | 同一批构建记录使用同一时间，格式为 ISO 风格 UTC 字符串 | + +记录关联优先使用 `aweme_id`，再使用 `video_id`；标题和文案数据还可使用 `aweme_url` 兜底。评论聚合仅按 `aweme_id`、`video_id` 关联。 + +## 6. 状态字段说明 + +### comment_data_status + +| 值 | 含义 | +|---|---| +| `available` | 存在 `comments_clean.csv`，且该视频有至少一条有效评论 | +| `empty` | 存在 `comments_clean.csv`，但该视频没有有效评论 | +| `pending_cdp` | 未提供 comments task，或缺少 `comments_clean.csv` | + +`comment_count` 是搜索结果中的平台展示值；`valid_comment_count` 和 `comment_data_status` 描述的是本次评论采集与清洗结果，二者不能互相替代。 + +### asr_data_status + +实际判断按以下顺序执行： + +| 值 | 含义 | +|---|---| +| `dependency_missing` | `script_raw.asr_status` 或 `script_clean.asr_status` 为 `dependency_missing` | +| `pending_asr` | 缺少 `script_clean.csv`，但 `script_sources.csv` 标记 `source_asr_planned=true` | +| `missing` | 缺少可用文案，或缺少 scripts task 且没有可判定的 ASR 计划 | +| `available` | `script_clean_source=asr_raw` 且 `script_clean_text` 非空，表示使用真实 ASR 文案 | +| `fallback_title` | `script_clean_source=source_clean_title` 且文本非空 | +| `fallback_desc` | `script_clean_source=source_title_desc` 且文本非空 | + +### script_clean_source + +| 值 | 含义 | +|---|---| +| `asr_raw` | 真实 ASR 文案 | +| `source_clean_title` | 清洗标题/描述组成的 fallback | +| `source_title_desc` | 原始标题/描述组成的 fallback | +| `missing` | 无可用文案 | + +不得只根据 `script_clean_text` 非空就宣称存在真实 ASR。真实 ASR 必须同时满足 `script_clean_source=asr_raw` 和 `asr_data_status=available`。 + +## 7. asset_quality 实际优先级 + +`asset_quality` 不是绝对内容质量评分，而是当前资产完整度/可用度标记。代码按下列顺序从上到下判断，命中后立即返回： + +| 优先级 | 值 | 实际条件 | +|---:|---|---| +| 1 | `high` | `valid_comment_count > 0`，且 `script_clean_source=asr_raw`，且文案非空 | +| 2 | `missing` | 原始/清洗标题和描述全部为空，且文案为空 | +| 3 | `medium` | 存在 `clean_title` 或 `clean_desc`，且文案来源为 `source_clean_title` 或 `source_title_desc`，且文案非空 | +| 4 | `partial` | 文案非空，且评论状态不是 `available`，或 ASR 状态不是 `available` | +| 5 | `low` | 未命中以上条件 | + +当前实现中，medium 优先于 partial。因此，有清洗标题或清洗描述、并使用标题类 fallback 文案时，即使缺少真实 ASR，通常也会先标记为 `medium`。`partial` 用于未达到 `medium` 条件、但仍有文案且评论或真实 ASR 不完整的情况。 + +边界示例： + +- 有有效评论和真实 ASR：`high`。 +- 有清洗标题和标题 fallback：`medium`。 +- 无任何标题/描述且无文案：`missing`。 +- 有真实 ASR 文案但评论不可用：`partial`。 +- 只有搜索标题等基础信息、没有文案：`low`。 + +## 8. 真实数据与 fallback 边界 + +| 数据 | 可视为真实采集 | 说明 | +|---|---:|---| +| `search_result.csv` 中的作品和互动元数据 | 是 | 来自搜索采集结果，但仍受平台返回值和采集质量影响 | +| `comments_clean.csv` 中的评论 | 是 | 前提是 comments task 实际通过 CDP 完成评论采集；清洗字段属于规则派生 | +| `script_clean_source=asr_raw` 的文案 | 是 | 表示使用真实 ASR 转写结果 | +| `source_clean_title` 文案 | 否 | 清洗标题/描述 fallback | +| `source_title_desc` 文案 | 否 | 原始标题/描述 fallback | +| `topic`、`pain_point`、`teaching_angle` | 派生字段 | 当前为规则清洗/提取结果，不是原始平台字段 | +| `asset_quality` 和状态字段 | 派生字段 | 用于显式标注数据完整度和来源质量 | + +即使 content_asset 的 schema 完整，也不能表述为“全量真实评论和真实语音文案已经完成”。应结合 `comment_data_status`、`asr_data_status`、`script_clean_source` 和 `content_asset_stats` 判断实际质量。 + +## 9. API 使用方式 + +以下示例使用 `$API_BASE_URL` 表示当前环境配置的 API 地址： + +```powershell +$API_BASE_URL = "http://localhost:" +``` + +### POST /scrape/merge + +task-id 模式： + +```powershell +curl.exe -X POST "$API_BASE_URL/scrape/merge" ` + -H "Content-Type: application/json" ` + -d '{"search_task_id":"","comments_task_id":"","scripts_task_id":""}' +``` + +只有 search task 时： + +```powershell +curl.exe -X POST "$API_BASE_URL/scrape/merge" ` + -H "Content-Type: application/json" ` + -d '{"search_task_id":""}' +``` + +`comments_task_id` 和 `scripts_task_id` 可选。响应中的 `task_id` 是后续使用的 `merge_task_id`。 + +### GET /scrape/status/{task_id} + +```powershell +curl.exe "$API_BASE_URL/scrape/status/" +``` + +任务完成后，返回对象的 `result` 中应包含： + +```text +content_asset_jsonl +content_asset_csv +content_asset_stats +``` + +### GET /scrape/result/{task_id} + +```powershell +curl.exe -o content_asset.csv "$API_BASE_URL/scrape/result/" +``` + +merge task 存在 `content_asset.csv` 时，主结果选择器会优先返回该文件。 + +### GET /scrape/data/preview/{task_id} + +```powershell +curl.exe "$API_BASE_URL/scrape/data/preview/?limit=20" +``` + +返回结构： + +```text +task_id +file_name +format +total_rows +rows +``` + +正常情况下 `file_name` 为 `content_asset.csv`。 + +### GET /scrape/data/export?task_id={task_id} + +```powershell +curl.exe -o content_asset_export.csv "$API_BASE_URL/scrape/data/export?task_id=" +``` + +GET export 是原始主结果文件下载，保留 `content_asset.csv` 的完整 schema 和原文件 BOM。 + +### POST /scrape/data/export + +```powershell +curl.exe -X POST "$API_BASE_URL/scrape/data/export" ` + -H "Content-Type: application/json" ` + -d '{"task_ids":[""],"format":"csv","limit":200}' ` + -o content_asset_legacy_export.csv +``` + +POST export 是批量旧七字段归一化导出，输出字段为： + +```text +video_id +platform +script_text +likes +favorites +shares +comments +``` + +它不是 content_asset 完整字段导出。映射关系包括： + +```text +script_text <- script_clean_text +likes <- liked_count +favorites <- collected_count +shares <- share_count +comments <- comment_count +``` + +## 10. 前端使用方式 + +### DataPage + +- 后端 `/scrape/data/list` 仍返回 task workspace 中的多个结果文件。 +- DataPage 按优先级为每个 task 选择一个主结果文件展示。 +- merge task 的 `content_asset.csv` 显示为“内容资产表”。 +- 预览默认展示核心列，避免宽表横向过长。 +- 可使用“查看全部字段 / 只看核心字段”切换。 +- “下载原始文件”使用 GET export，保留完整 `content_asset.csv`。 +- “批量导出旧七字段”使用 POST export，输出旧兼容格式。 + +### TaskDetailPage + +- merge task 显示“主结果：内容资产表”和文件名 `content_asset.csv`。 +- 下载按钮显示为“下载内容资产表”。 +- `content_asset_stats` 以结构化方式展示，包括输入/输出行数、评论、文案、ASR、fallback 和错误信息。 + +当前前端创建 merge 任务的页面尚未提供 `search_task_id`、`comments_task_id`、`scripts_task_id` 输入；task-id 模式请直接调用 API。这不影响 DataPage 和 TaskDetailPage 的查看、预览和下载能力。 + +## 11. API 验收清单 + +### 生成与状态 + +先准备一个已完成的 search task；comments/scripts task 可选。调用 merge 后记录返回的 `merge_task_id`： + +```powershell +curl.exe -X POST "$API_BASE_URL/scrape/merge" ` + -H "Content-Type: application/json" ` + -d '{"search_task_id":"","comments_task_id":"","scripts_task_id":""}' +``` + +轮询状态： + +```powershell +curl.exe "$API_BASE_URL/scrape/status/" +``` + +验收点： + +- task 状态为 `completed`。 +- `result.content_asset_jsonl`、`result.content_asset_csv`、`result.content_asset_stats` 存在。 +- merge task workspace 的 `outputs/` 中存在两个 content_asset 文件。 + +### result、preview 与 export + +```powershell +curl.exe -o result.csv "$API_BASE_URL/scrape/result/" +curl.exe "$API_BASE_URL/scrape/data/preview/?limit=20" +curl.exe -o export.csv "$API_BASE_URL/scrape/data/export?task_id=" +``` + +验收点： + +- result 下载文件为 `content_asset.csv`。 +- preview 返回 `file_name=content_asset.csv`，并包含 `format`、`total_rows`、`rows`。 +- GET export 与主结果原文件内容一致，保留完整字段和 BOM。 + +### BOM 检查 + +PowerShell 可执行命令： + +```powershell +@' +from pathlib import Path +for name in ["result.csv", "export.csv"]: + p = Path(name) + print(name, p.read_bytes()[:3]) +'@ | python - +``` + +期望输出： + +```text +b'\xef\xbb\xbf' +``` + +### 本地测试与静态检查 + +```powershell +python -m pytest douyin_scraper\tests\test_content_asset.py -q +python -m pytest douyin_scraper\tests\test_content_asset_api.py -q +python -m pytest douyin_scraper\tests\test_task_result_selection.py -q +python -m py_compile api\tasks.py api\routes.py douyin_scraper\content_asset.py +``` + +## 12. 前端验收清单 + +### 构建 + +```powershell +cd web +npm run build +``` + +项目当前没有独立的 `lint` 或 `typecheck` npm 脚本；类型检查已包含在 `npm run build` 的 `tsc -b` 中。bundle size warning 属既有警告，不阻断 T017-5。 + +### 浏览器检查 + +打开当前环境的前端地址，检查： + +- DataPage 每个 task 只展示一个主结果。 +- merge task 显示“内容资产表”和 `content_asset.csv`。 +- content_asset 预览默认显示核心列。 +- 可切换查看全部字段。 +- “下载原始文件”下载完整 content_asset schema。 +- “批量导出旧七字段”文案明确，导出七字段兼容格式。 +- TaskDetailPage 显示“主结果：内容资产表 content_asset.csv”。 +- 下载按钮为“下载内容资产表”。 +- `content_asset_stats` 不显示为 `[object Object]`，而是结构化信息。 + +## 13. Pending 风险 + +| 风险 | 是否阻断 T017-5 功能封版 | 后续处理 | +|---|---:|---| +| CDP 真实评论采集未完成 | 否 | 独立评论采集任务 | +| 真实 ASR 未完成 | 否 | 独立 Whisper/ASR 任务 | +| POST export 保持旧七字段契约 | 否 | 作为兼容契约保留，若升级需独立版本设计 | +| content_asset 的 comment/asr 状态依赖已有输入质量 | 否 | 通过状态字段和数据质量文档持续管理 | +| 后端 `/data/list` 仍返回多文件，前端做展示过滤 | 否 | 可在后续统一后端主结果语义 | +| 前端创建 merge 尚未提供 task_id 输入 | 否 | 独立前端创建流程任务 | + +这些风险不阻断 T017-5 功能封版，但必须作为后续独立任务管理，不能把 pending 能力描述成已经完成。 diff --git a/docs/index.md b/docs/index.md index 332d0c54b..6e9318d38 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,6 +3,7 @@ ## 项目文档 - [项目架构文档](项目架构文档.md) - 系统架构、模块设计、数据流向（含 Mermaid 图表） +- [Content Asset 数据字典与验收说明](CONTENT_ASSET.md) - 内容资产字段、状态、API、前端使用与验收清单 ## 推荐：使用 uv 管理依赖 From 5f4bce09e5325066a9715e7feb6d8bb2fefa7ad4 Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 15:40:48 +0800 Subject: [PATCH 12/32] docs(tasks): archive T017-5 release isolation notes --- docs/tasks/T017-5-1-content-asset.md | 425 +++++++++++++++ docs/tasks/T017-5-2-1.md | 311 +++++++++++ docs/tasks/T017-5-3-1.md | 462 ++++++++++++++++ docs/tasks/T017-5-4-1.md | 581 +++++++++++++++++++++ docs/tasks/T017-5-Release-API-1.md | 184 +++++++ docs/tasks/T017-5-Release-Builder-API-0.md | 238 +++++++++ docs/tasks/T017-5-Release-Isolation-10.md | 398 ++++++++++++++ docs/tasks/T017-5-Release-Isolation-11.md | 371 +++++++++++++ docs/tasks/T017-5-Release-Isolation-11F.md | 239 +++++++++ docs/tasks/T017-5-Release-Isolation-12.md | 327 ++++++++++++ docs/tasks/T017-5-Release-Isolation-4.md | 414 +++++++++++++++ docs/tasks/T017-5-Release-Isolation-5.md | 303 +++++++++++ docs/tasks/T017-5-Release-Isolation-6.md | 307 +++++++++++ docs/tasks/T017-5-Release-Isolation-6r.md | 297 +++++++++++ docs/tasks/T017-5-Release-Isolation-7.md | 326 ++++++++++++ docs/tasks/T017-5-Release-Isolation-8.md | 316 +++++++++++ docs/tasks/T017-5-Release-Isolation-9.md | 311 +++++++++++ docs/tasks/T017-5-Release-PreCommit-A-D.md | 198 +++++++ docs/tasks/T017-5-Release-Web-1.md | 290 ++++++++++ 19 files changed, 6298 insertions(+) create mode 100644 docs/tasks/T017-5-1-content-asset.md create mode 100644 docs/tasks/T017-5-2-1.md create mode 100644 docs/tasks/T017-5-3-1.md create mode 100644 docs/tasks/T017-5-4-1.md create mode 100644 docs/tasks/T017-5-Release-API-1.md create mode 100644 docs/tasks/T017-5-Release-Builder-API-0.md create mode 100644 docs/tasks/T017-5-Release-Isolation-10.md create mode 100644 docs/tasks/T017-5-Release-Isolation-11.md create mode 100644 docs/tasks/T017-5-Release-Isolation-11F.md create mode 100644 docs/tasks/T017-5-Release-Isolation-12.md create mode 100644 docs/tasks/T017-5-Release-Isolation-4.md create mode 100644 docs/tasks/T017-5-Release-Isolation-5.md create mode 100644 docs/tasks/T017-5-Release-Isolation-6.md create mode 100644 docs/tasks/T017-5-Release-Isolation-6r.md create mode 100644 docs/tasks/T017-5-Release-Isolation-7.md create mode 100644 docs/tasks/T017-5-Release-Isolation-8.md create mode 100644 docs/tasks/T017-5-Release-Isolation-9.md create mode 100644 docs/tasks/T017-5-Release-PreCommit-A-D.md create mode 100644 docs/tasks/T017-5-Release-Web-1.md diff --git a/docs/tasks/T017-5-1-content-asset.md b/docs/tasks/T017-5-1-content-asset.md new file mode 100644 index 000000000..7a41fd163 --- /dev/null +++ b/docs/tasks/T017-5-1-content-asset.md @@ -0,0 +1,425 @@ +T017-5-1：content_asset.csv schema 与 builder 第一版实现。 + +本轮允许改代码，但只做 content_asset 第一版生成能力。不要改 preview/export/result，不接 AI，不处理 whisper/CDP，不改 run_all 主流程，不混修端口。 + +## 当前基线 + +T017 主线已完成到： + +```text +search_result.csv/jsonl +search_title_clean.csv/jsonl +comments_clean.csv/jsonl +script_sources.csv/jsonl +script_raw.csv/jsonl +script_clean.csv/jsonl +``` + +T017-5-0 查询结论： + +```text +merge_to_csv.py 是旧合并雏形，不适合作为主实现 +run_all.py 是旧流程，不作为 T017-5 主入口 +content_asset.csv 建议归属 merge_task workspace +推荐新增 douyin_scraper/content_asset.py +由 DouyinScraper 增加薄 helper 调用 +``` + +## 固定端口规则 + +MediaCrawler 固定端口： + +```text +API 宿主：http://localhost:18080 +API 容器内部：8000 +前端 dev：15173 +Chrome/CDP：19222 +``` + +本轮不要改端口。 + +## 禁止 + +```text +禁止接 AI/LLM +禁止处理 whisper/ASR 依赖 +禁止处理 CDP 真实评论采集 +禁止改 preview/export/result +禁止改 run_all.py 主流程 +禁止直接复用旧 merge_to_csv.py 作为主实现 +禁止新增 /scrape/content-asset-v2 之类重复接口 +禁止混修端口 +禁止删除或覆盖原始 CSV/JSONL +``` + +## 开发前先查 + +先检查并输出： + +```text +douyin_scraper/core.py +douyin_scraper/content_asset.py 是否存在 +merge_to_csv.py +run_all.py +api/routes.py +api/tasks.py +douyin_scraper/tests/ +``` + +回答： + +```text +1. 当前 /scrape/merge 在哪里？ +2. DouyinScraper 当前 merge 入口在哪里？ +3. merge task workspace 如何创建？ +4. 当前 task result 如何返回 merge 结果？ +5. content_asset.py 是否已存在？ +6. 本次新增/修改哪些文件？ +``` + +没有完成查询，不要写代码。 + +## 目标 + +基于多个 task 的 outputs，生成： + +```text +workspace//outputs/content_asset.jsonl +workspace//outputs/content_asset.csv +``` + +输入建议： + +```json +{ + "search_task_id": "", + "comments_task_id": "", + "scripts_task_id": "" +} +``` + +如当前 `/scrape/merge` schema 不支持这些字段，可最小范围扩展现有 merge request。不要新增新接口。 + +## 输入文件优先级 + +从 task outputs 读取： + +### search task + +```text +search_result.csv +search_title_clean.csv +``` + +### comments task，可选 + +```text +comments_clean.csv +``` + +没有 comments task 或没有 comments_clean.csv 时，不失败，标记： + +```text +comment_data_status=pending_cdp 或 missing +``` + +### scripts task，可选 + +```text +script_sources.csv +script_raw.csv +script_clean.csv +``` + +没有 scripts task 或没有 script_clean.csv 时，不失败，标记： + +```text +asr_data_status=missing 或 pending_asr +``` + +## 关联规则 + +按以下优先级关联： + +```text +aweme_id +video_id +aweme_url +``` + +要求： + +```text +search_result 是主表 +title_clean 按 aweme_id/video_id/aweme_url 补充 +comments_clean 按 aweme_id/video_id 聚合 +script_clean 按 aweme_id/video_id/aweme_url 补充 +缺失不崩溃 +``` + +## content_asset 字段 + +第一版 CSV/JSONL 至少包含： + +```text +source_keyword +platform +video_id +aweme_id +aweme_url + +raw_title +clean_title +desc +clean_desc +topic +pain_point +teaching_angle + +nickname +liked_count +collected_count +comment_count +share_count +total_engagement + +valid_comment_count +top_valid_comments +comment_pain_tags + +script_source_status +script_source_quality +script_clean_text +script_clean_source +script_clean_quality + +comment_data_status +asr_data_status +asset_quality +created_at +``` + +## 聚合规则 + +### comments_clean 聚合 + +只统计： + +```text +is_valid=true +``` + +输出： + +```text +valid_comment_count +top_valid_comments +comment_pain_tags +``` + +规则： + +```text +top_valid_comments 取前 3 条 clean_content，用 | 拼接 +comment_pain_tags 去重后用 | 拼接 +如果没有 comments_clean.csv：valid_comment_count=0，comment_data_status=pending_cdp +如果有文件但无有效评论：comment_data_status=empty +如果有有效评论：comment_data_status=available +``` + +### script_clean 合并 + +优先读取： + +```text +script_clean_text +script_clean_source +script_clean_quality +script_clean_status +``` + +`asr_data_status` 规则： + +```text +script_clean_source=asr_raw -> available +script_clean_source=source_clean_title -> fallback_title +script_clean_source=source_title_desc -> fallback_desc +script_clean_source=missing 或 script_clean_text 空 -> missing +如果 script_raw 中 asr_status=dependency_missing -> dependency_missing +``` + +### asset_quality 规则 + +建议： + +```text +high：有有效评论 + script_clean_source=asr_raw +medium：有 title_clean + 有 fallback script_clean_text +low：只有 search/title 基础信息 +partial：缺评论或缺真实 ASR，但有 fallback 文案 +missing：关键标题和文案都缺失 +``` + +注意：不能把 fallback 文案伪装成真实 ASR 文案。 + +## 新增文件建议 + +允许新增： + +```text +douyin_scraper/content_asset.py +``` + +职责： + +```text +读取标准 CSV +按 aweme_id/video_id/aweme_url 合并 +聚合 comments_clean +合并 script_clean +计算 comment_data_status/asr_data_status/asset_quality +写 content_asset.jsonl/csv +``` + +`core.py` 只保留薄 helper 调用，不要继续膨胀过多。 + +## status 返回 + +扩展现有 merge task result，返回： + +```text +content_asset_jsonl +content_asset_csv +content_asset_stats +``` + +`content_asset_stats` 至少包含： + +```text +rows_in +rows_out +comments_available +scripts_available +valid_comments_total +asr_available +fallback_script_total +missing_script_total +content_asset_csv_generated +errors +``` + +不要改 `/scrape/result` 显式优先级，不改 preview/export。本轮只保证 status result 有字段，文件真实生成。 + +## 输出要求 + +```text +JSONL ensure_ascii=False +CSV utf-8-sig +CSV 表头顺序固定 +缺字段填空字符串或 0 +输入文件缺失不崩溃，写入 stats.errors +``` + +## 测试要求 + +至少覆盖： + +```text +仅 search_result + title_clean 也能生成 content_asset +comments_clean 存在时聚合 valid_comment_count/top_valid_comments/comment_pain_tags +comments_clean 缺失时 comment_data_status=pending_cdp +script_clean 存在时填 script_clean_text/script_clean_source/script_clean_quality +script_raw asr_status=dependency_missing 时 asr_data_status=dependency_missing +fallback 文案时 asset_quality 不标 high +按 aweme_id 优先关联 +按 video_id 兜底关联 +CSV utf-8-sig +JSONL ensure_ascii=False +``` + +## 本地验收命令 + +```powershell +python -m py_compile douyin_scraper\core.py douyin_scraper\content_asset.py api\routes.py +python -m pytest douyin_scraper\tests\test_core.py -q +``` + +如果新增独立测试文件，也一起跑： + +```powershell +python -m pytest douyin_scraper\tests\ -q +``` + +## Docker/API 验收 + +统一使用： + +```text +宿主：http://localhost:18080 +容器内部：http://localhost:8000 +``` + +验收流程： + +```text +1. 准备已有 search task / comments task / scripts task +2. 调用 /scrape/merge，传 search_task_id/comments_task_id/scripts_task_id +3. 确认 merge task completed +4. status 返回 content_asset_jsonl/content_asset_csv/content_asset_stats +5. 检查 outputs/content_asset.jsonl +6. 检查 outputs/content_asset.csv +7. 检查 CSV BOM = ef bb bf +8. 检查 comment_data_status / asr_data_status / asset_quality +``` + +示例检查： + +```powershell +curl.exe http://localhost:18080/scrape/status/ + +docker exec mediacrawler-api-1 sh -lc "ls -lh /app/workspaces//outputs/" +docker exec mediacrawler-api-1 sh -lc "head -n 3 /app/workspaces//outputs/content_asset.csv" +docker exec mediacrawler-api-1 sh -lc "python - <<'PY' +from pathlib import Path +p = Path('/app/workspaces//outputs/content_asset.csv') +print(p.read_bytes()[:3]) +PY" +``` + +期望： + +```text +b'\xef\xbb\xbf' +``` + +## 最终报告 + +输出： + +```text +1. 开发前查询结果 +2. 新增/修改文件 +3. content_asset 字段说明 +4. 聚合规则说明 +5. 数据质量状态说明 +6. 测试结果 +7. Docker/API 验收结果 +8. 是否允许进入 T017-5-2 +``` + +## 通过标准 + +```text +生成 content_asset.jsonl +生成 content_asset.csv +CSV UTF-8-SIG +JSONL 中文正常 +status 返回 content_asset_jsonl/content_asset_csv/content_asset_stats +保留 comment_data_status/asr_data_status/asset_quality +不改 preview/export/result +不接 AI +不处理 whisper/CDP +不新增重复接口 +测试通过 +Docker/API 验收通过 +``` diff --git a/docs/tasks/T017-5-2-1.md b/docs/tasks/T017-5-2-1.md new file mode 100644 index 000000000..6837bd904 --- /dev/null +++ b/docs/tasks/T017-5-2-1.md @@ -0,0 +1,311 @@ +T017-5-2-1：content_asset result / preview / export 后端最小接入。 + +本轮允许改代码，但只做后端最小接入：让 `content_asset.csv` 可以被 `/scrape/result`、`/scrape/data/preview`、`/scrape/data/export` 稳定识别和返回。 + +## 当前基线 + +T017-5-1 已完成： + +```text +content_asset.jsonl +content_asset.csv +content_asset_stats +``` + +Docker/API 验收已 PASS。 + +T017-5-2-0 查询已完成，结论： + +```text +允许进入 T017-5-2-1 +只改 result / preview / export 后端接入 +不改 content_asset builder +不改前端 +不新增接口 +``` + +## 固定端口 + +MediaCrawler 固定端口： + +```text +API 宿主：http://localhost:18080 +API 容器内部：8000 +前端 dev：15173 +Chrome/CDP：19222 +``` + +不要使用宿主 `localhost:8000` 做验收。 + +## 禁止 + +```text +禁止改 content_asset builder 主逻辑 +禁止接 AI/LLM +禁止处理 whisper/ASR +禁止处理 CDP +禁止改 run_all.py +禁止改前端 web/src/* +禁止新增重复接口 +禁止混修端口 +禁止破坏 T016 search_result.csv 行为 +禁止破坏 comments_clean 行为 +禁止破坏 script_clean 行为 +``` + +## 开发前先查 + +先检查并输出： + +```text +api/tasks.py +api/routes.py +api/tests.py +douyin_scraper/content_asset.py +douyin_scraper/core.py +``` + +回答： + +```text +1. 当前 /scrape/result 文件选择逻辑在哪里？ +2. 当前 /scrape/data/preview 文件选择逻辑在哪里？ +3. 当前 /scrape/data/export GET/POST 分别在哪里？ +4. 当前 search/comments/scripts/merge 的 task_type 如何判断？ +5. 本次具体修改哪些文件？ +``` + +没有完成查询，不要写代码。 + +## 一、/scrape/result 接入 + +修改 `api/tasks.py` 的结果文件选择逻辑。 + +不要做全局扁平优先级，要按 task_type 感知，避免破坏 T016。 + +推荐优先级： + +```text +merge: +content_asset.csv +> content_asset.jsonl +> douyin_koubo_data.csv +> latest csv/jsonl fallback + +scripts: +script_clean.csv +> script_clean.jsonl +> script_raw.csv +> script_raw.jsonl +> script_sources.csv +> script_sources.jsonl +> latest fallback + +comments: +comments_clean.csv +> comments_clean.jsonl +> comments_raw.csv +> comments_raw.jsonl + +search: +search_result.csv +> search_result.jsonl +``` + +要求： + +```text +merge task 存在 content_asset.csv 时，/scrape/result/ 必须稳定返回 content_asset.csv +search task 仍返回 search_result.csv +comments task 仍返回 comments_clean.csv +scripts task 优先返回 script_clean.csv +``` + +## 二、preview 接入 + +修改 `/scrape/data/preview/{task_id}`。 + +要求： + +```text +复用或对齐 TaskManager 的结果文件选择逻辑 +content_asset.csv 存在时可预览 +comments_clean.csv 可预览 +script_clean.csv 可预览 +search_result.csv 行为不回归 +``` + +返回结构保持不变： + +```text +task_id +file_name +format +total_rows +rows +``` + +保留现有 `limit` 逻辑，不新增 offset。 + +修复已有 JSONL fallback 中 `json` 未定义问题。如果代码中实际导入为 `_json`，统一使用 `_json`。 + +## 三、export 接入 + +### GET export + +修改当前 GET `/scrape/data/export?task_id={task_id}` 路由。 + +要求： + +```text +使用统一结果选择器 +merge task 原样导出 content_asset.csv +保留 content_asset.csv 完整 schema +保留原文件 BOM +search/comments/scripts 行为不回归 +``` + +### POST export + +POST 批量 export 保持旧七字段契约： + +```text +video_id +platform +script_text +likes +favorites +shares +comments +``` + +但补充 content_asset 字段别名映射： + +```text +script_text <- script_clean_text +likes <- liked_count +favorites <- collected_count +shares <- share_count +comments <- comment_count +``` + +要求： + +```text +不要把 content_asset.csv 强制裁成旧字段用于 GET 原文件下载 +只有 POST 批量归一化 export 继续输出旧七字段 +``` + +如 POST 当前无 BOM，可在不破坏旧行为的前提下补充 UTF-8-SIG。 + +## 四、测试要求 + +至少覆盖： + +```text +merge task 有 content_asset.csv -> result 返回 content_asset.csv +merge task 无 content_asset.csv -> fallback 不崩溃 +search task -> result 仍返回 search_result.csv +comments task -> result 仍返回 comments_clean.csv +scripts task -> result 优先返回 script_clean.csv + +preview 能读取 content_asset.csv +preview 能读取 search_result.csv +preview JSONL fallback 不再因 json 未定义报错 + +GET export merge task 原样导出 content_asset.csv +POST export 可将 content_asset 映射到旧七字段 +CSV BOM 保持 +``` + +优先补充 `api/tests.py` 或现有 API 测试；如 TestClient/httpx 兼容问题仍存在，可增加纯函数/路由辅助函数测试，并在报告中说明。 + +## 五、本地验收 + +```powershell +python -m py_compile api\tasks.py api\routes.py api\tests.py +python -m pytest api\tests.py -q +``` + +如果 `api/tests.py` 因既有 TestClient/httpx 兼容问题失败，不要混修依赖；补充可运行的最小测试，或说明非本轮逻辑失败。 + +同时跑核心测试： + +```powershell +python -m pytest douyin_scraper\tests\test_content_asset.py -q +``` + +## 六、Docker/API 验收 + +统一使用： + +```text +宿主：http://localhost:18080 +容器内部：http://localhost:8000 +``` + +使用已有 merge task，例如： + +```text +merge_task_id=fa1b4aea6251 +``` + +或重新创建一个 merge task。 + +检查： + +```powershell +curl.exe -I http://localhost:18080/scrape/result/ +curl.exe http://localhost:18080/scrape/data/preview/ +curl.exe -o content_asset_download.csv "http://localhost:18080/scrape/data/export?task_id=" +``` + +也检查： + +```powershell +curl.exe -o result.csv http://localhost:18080/scrape/result/ +``` + +必须确认： + +```text +/scrape/result 返回 content_asset.csv +preview 返回 file_name=content_asset.csv +GET export 原样导出 content_asset.csv +CSV BOM = ef bb bf +content_asset 字段完整 +``` + +同时抽查 search/comments/scripts 任务，确认不回归。 + +## 七、最终报告 + +输出： + +```text +1. 开发前查询结果 +2. 修改文件 +3. result 优先级说明 +4. preview 接入说明 +5. export 接入说明 +6. 测试结果 +7. Docker/API 验收结果 +8. 是否允许进入 T017-5-3 +``` + +## 通过标准 + +```text +merge task 的 /scrape/result 稳定返回 content_asset.csv +preview 能读取 content_asset.csv +GET export 能原样导出 content_asset.csv +POST export 能兼容 content_asset 到旧七字段 +search_result 行为不回归 +comments_clean 行为不回归 +script_clean 行为不回归 +CSV BOM 保持 +不改 builder +不改前端 +不新增接口 +Docker/API 验收通过 +``` diff --git a/docs/tasks/T017-5-3-1.md b/docs/tasks/T017-5-3-1.md new file mode 100644 index 000000000..ab14f7a7d --- /dev/null +++ b/docs/tasks/T017-5-3-1.md @@ -0,0 +1,462 @@ +T017-5-3-1：前端 content_asset 展示与下载体验最小优化。 + +本轮允许改代码，但只做前端展示与下载语义优化。不改后端，不改 builder，不接 AI，不处理 CDP/whisper/ASR，不做图表看板。 + +## 当前基线 + +后端已完成： + +```text +T017-5-1：content_asset.csv builder PASS +T017-5-2-1：content_asset result / preview / export 后端接入 PASS +T017-5-3-0：前端只读查询 PASS +``` + +当前后端已支持： + +```text +/scrape/result/ 返回 content_asset.csv +/scrape/data/preview/ 可预览 content_asset.csv +/scrape/data/export?task_id= 可原样导出 content_asset.csv +POST /scrape/data/export 保持旧七字段批量导出 +``` + +## 固定端口 + +MediaCrawler 固定端口： + +```text +API 宿主：http://localhost:18080 +API 容器内部：8000 +前端 dev：http://localhost:15173 +CDP：19222 +``` + +禁止使用宿主 `localhost:8000` 做验收。 + +## 禁止 + +```text +禁止改后端 api/tasks.py +禁止改后端 api/routes.py +禁止改 douyin_scraper/content_asset.py +禁止改 douyin_scraper/core.py +禁止改 run_all.py +禁止接 AI/LLM +禁止处理 whisper/ASR +禁止处理 CDP +禁止做图表看板 +禁止做内容评分 +禁止重构后端 /data/list +禁止新增后端接口 +禁止混修端口 +禁止提交 commit +``` + +## 开发前先查 + +先检查并输出： + +```text +web/src/pages/DataPage.tsx +web/src/pages/TaskDetailPage.tsx +web/src/api/client.ts +web/src/api/types.ts +web/src/utils/constants.ts +web/vite.config.ts +``` + +回答： + +```text +1. DataPage 当前如何展示数据文件列表？ +2. DataPage 当前 preview/export/download 分别调用什么？ +3. TaskDetailPage 当前如何显示 task.result？ +4. TaskDetailPage 当前下载按钮调用什么？ +5. client.ts 是否已有 GET 原始文件导出 helper？ +6. types.ts 是否已有 DataPreviewResponse.format？ +7. 本次具体修改哪些文件？ +``` + +未完成查询前不要写代码。 + +## 一、DataPage 优化目标 + +### 1. 列表只突出主结果文件 + +当前 `/scrape/data/list` 会返回同一 task 的多个文件，包括： + +```text +content_asset.csv +content_asset.jsonl +search_result.csv +search_title_clean.csv +comments_clean.csv +script_clean.csv +execution_log.jsonl +``` + +本轮前端最小处理： + +```text +过滤 execution_log.jsonl +过滤明显辅助产物 +同一 task 优先展示主结果文件 +使用 task_id + file_path 或 task_id + file_name 作为 React key +避免同一 task 多行同时选中 +``` + +主结果文件优先级建议： + +```text +content_asset.csv +script_clean.csv +comments_clean.csv +search_result.csv +content_asset.jsonl +script_clean.jsonl +comments_clean.jsonl +search_result.jsonl +其他 CSV +其他 JSONL +``` + +注意：这是前端列表展示优化，不改后端 `/data/list`。 + +### 2. 显示更清楚的文件信息 + +DataPage 列表建议显示： + +```text +任务 ID +任务类型 +主结果文件名 +格式 csv/jsonl +行数 +大小 +完成时间 +``` + +如果 `file_name=content_asset.csv` 或 `task_type=merge`，展示中文标签： + +```text +内容资产表 +``` + +如果是： + +```text +search_result.csv -> 搜索结果表 +comments_clean.csv -> 评论清洗表 +script_clean.csv -> 文案清洗表 +``` + +### 3. Preview 默认行数优化 + +当前默认 100 行。content_asset 是宽表，本轮建议： + +```text +默认 preview limit 降为 50 或 20 +client.previewData 支持可选 limit +DataPage 调用 previewData(taskId, limit) +``` + +推荐默认： + +```text +20 +``` + +### 4. content_asset 核心列展示 + +content_asset.csv 约 30 列，默认展示核心列，避免横向过宽。 + +默认核心列建议： + +```text +source_keyword +platform +aweme_id +clean_title +topic +pain_point +teaching_angle +nickname +liked_count +collected_count +comment_count +share_count +total_engagement +valid_comment_count +comment_data_status +asr_data_status +asset_quality +``` + +隐藏但可查看全部字段： + +```text +raw_title +desc +clean_desc +top_valid_comments +comment_pain_tags +script_source_status +script_source_quality +script_clean_text +script_clean_source +script_clean_quality +created_at +``` + +要求： + +```text +提供“查看全部字段 / 只看核心字段”切换 +非 content_asset 文件仍按原逻辑动态展示全部字段 +不要丢失数据，只是前端展示筛选 +``` + +### 5. 长文本列宽处理 + +对以下字段增加较宽展示或截断 tooltip： + +```text +script_text +script_clean_text +top_valid_comments +comment_pain_tags +desc +clean_desc +``` + +要求： + +```text +不要让表格撑爆页面 +保留横向滚动 +长文本可截断显示 +``` + +### 6. 区分两类导出 + +当前 DataPage 只有 POST `/scrape/data/export`，它是旧七字段批量归一化导出。 + +本轮要在文案上区分： + +```text +下载原始文件：GET /scrape/data/export?task_id= +批量导出旧格式：POST /scrape/data/export +``` + +新增前端 helper： + +```text +getDataExportUrl(taskId) +``` + +对应后端已有： + +```text +GET /scrape/data/export?task_id= +``` + +DataPage 对单个文件/任务提供： + +```text +下载原始文件 +``` + +批量按钮保留，但文案改成： + +```text +批量导出旧七字段 +``` + +说明文案： + +```text +“下载原始文件”会保留 content_asset.csv 完整字段； +“批量导出旧七字段”会导出 video_id/platform/script_text/likes/favorites/shares/comments。 +``` + +## 二、TaskDetailPage 优化目标 + +### 1. merge task 显示内容资产表 + +如果 task 类型是 merge，或 task.result 包含： + +```text +content_asset_csv +content_asset_jsonl +content_asset_stats +``` + +则显示： + +```text +主结果：内容资产表 content_asset.csv +``` + +下载按钮文案改为： + +```text +下载内容资产表 +``` + +其他任务保持： + +```text +下载结果文件 +``` + +### 2. content_asset_stats 结构化显示 + +当前对象会显示为： + +```text +[object Object] +``` + +本轮要把 `content_asset_stats` 展示为简洁结构化信息。 + +建议展示字段： + +```text +rows_in +rows_out +comments_available +scripts_available +valid_comments_total +asr_available +fallback_script_total +missing_script_total +content_asset_csv_generated +errors +``` + +如字段不存在，不报错。 + +展示方式可以是小表格或 key-value 列表。 + +### 3. 保持其他任务不回归 + +search/comments/scripts 不做大改，只保持现有通用 result 展示和下载。 + +## 三、API 封装与类型 + +修改 `web/src/api/client.ts`： + +新增： + +```ts +getDataExportUrl(taskId: string): string +``` + +用于 GET 原始文件导出。 + +扩展： + +```ts +previewData(taskId: string, limit?: number) +``` + +拼接： + +```text +/scrape/data/preview/{task_id}?limit={limit} +``` + +修改 `web/src/api/types.ts`： + +为 `DataPreviewResponse` 增加： + +```ts +format?: 'csv' | 'jsonl' | string +``` + +如有必要，可增加 content asset stats 类型，但不要过度复杂化。 + +## 四、测试 / 构建要求 + +先查 package.json，使用项目已有命令。 + +至少执行： + +```powershell +cd web +npm run build +``` + +如项目有 lint/typecheck，也执行： + +```powershell +npm run lint +npm run typecheck +``` + +如果命令不存在，不要新增脚本，在报告中说明。 + +## 五、浏览器/前端验收建议 + +如果前端 dev 可运行，使用固定端口： + +```text +http://localhost:15173 +``` + +验收页面： + +```text +DataPage +TaskDetailPage +``` + +重点检查： + +```text +DataPage 不再大量重复展示同一 task 的辅助文件 +execution_log.jsonl 不再混入主要列表 +merge task 显示 content_asset.csv / 内容资产表 +content_asset 预览默认核心列 +可以切换查看全部字段 +下载原始文件指向 GET /scrape/data/export?task_id=... +批量导出旧七字段文案清晰 +TaskDetailPage 显示内容资产表 +TaskDetailPage 下载按钮文案正确 +content_asset_stats 不再显示 [object Object] +``` + +## 六、最终报告 + +输出： + +```text +1. 开发前查询结果 +2. 修改文件 +3. DataPage 优化说明 +4. TaskDetailPage 优化说明 +5. API 封装与类型修改 +6. 构建/测试结果 +7. 前端验收结果 +8. 是否允许进入 T017-5-4 +``` + +## 通过标准 + +```text +DataPage 主列表不再明显混入 execution_log +同一 task 重复选中问题改善 +content_asset 展示为“内容资产表” +content_asset 默认核心列可读 +可切换查看全部字段 +下载原始文件与批量旧格式导出语义区分 +TaskDetailPage merge task 显示“内容资产表” +TaskDetailPage 下载按钮为“下载内容资产表” +content_asset_stats 结构化展示 +npm run build 通过 +不改后端 +不改 builder +不新增后端接口 +不改端口 +``` diff --git a/docs/tasks/T017-5-4-1.md b/docs/tasks/T017-5-4-1.md new file mode 100644 index 000000000..89d0c99b1 --- /dev/null +++ b/docs/tasks/T017-5-4-1.md @@ -0,0 +1,581 @@ +T017-5-4-1：content_asset 文档固化与验收清单落地。 + +本轮允许修改文档，但禁止修改业务代码、前端逻辑、后端逻辑、builder、测试逻辑和端口配置。不新增脚本，不提交 commit。 + +## 当前基线 + +已完成并验收： + +```text +T017-5-0：content_asset 查询与复用判断 PASS +T017-5-1：content_asset schema 与 builder 第一版 PASS +T017-5-2-1：content_asset result / preview / export 后端接入 PASS +T017-5-3-1：前端 content_asset 展示与下载体验优化 PASS +T017-5-4-0：全链路封版验收与文档固化查询 PASS +``` + +当前功能链路已经具备封版条件，但正式文档封版尚未完成。 + +## 本轮目标 + +完成 content_asset 的正式文档收口： + +```text +1. 固化 content_asset 数据字典 +2. 固化字段来源、状态枚举、asset_quality 实际优先级 +3. 固化真实评论 / 真实 ASR / fallback 边界 +4. 固化 task-id 全链路 API 使用方式 +5. 固化 result / preview / GET export / POST export 区别 +6. 固化前端 DataPage / TaskDetailPage 使用说明 +7. 固化 Docker/API/前端验收清单 +8. 固化 pending 风险清单 +``` + +完成后用于正式关闭 T017-5。 + +## 固定端口 + +文档中所有宿主访问必须使用： + +```text +API 宿主：http://localhost:18080 +API 容器内部：http://localhost:8000 +前端 dev：http://localhost:15173 +CDP：19222 +``` + +禁止在文档中继续使用宿主 `localhost:8000` 作为 MediaCrawler API 验收地址。 + +## 禁止 + +```text +禁止修改业务代码 +禁止修改 api/tasks.py +禁止修改 api/routes.py +禁止修改 douyin_scraper/content_asset.py +禁止修改 douyin_scraper/core.py +禁止修改 web/src/* +禁止修改 run_all.py +禁止新增脚本 +禁止改测试逻辑 +禁止处理 CDP +禁止处理 Whisper / ASR +禁止修 TestClient/httpx +禁止混修端口 +禁止提交 commit +``` + +## 开始前先查 + +先只读检查： + +```text +README.md +api/README.md +docs/ +docs/tasks/ +douyin_scraper/content_asset.py +api/routes.py +api/tasks.py +web/src/pages/DataPage.tsx +web/src/pages/TaskDetailPage.tsx +web/src/api/client.ts +Makefile +docker-compose.yml +.env.example +``` + +回答： + +```text +1. 当前已有哪些 content_asset 相关文档？ +2. 哪些文档有旧路径或错误示例？ +3. README 是否需要补充？ +4. api/README 是否需要补充？ +5. docs 下是否已有适合新增数据字典的位置？ +6. 本次计划新增/修改哪些文档文件？ +``` + +完成查询后再改文档。 + +## 建议新增文档 + +优先新增： + +```text +docs/CONTENT_ASSET.md +``` + +如项目已有更合适的 docs 结构，可以选择同级命名，但必须说明原因。 + +## 一、docs/CONTENT_ASSET.md 内容要求 + +文档标题建议： + +```markdown +# Content Asset 数据字典与验收说明 +``` + +至少包含以下章节： + +```markdown +## 1. 定位 +## 2. 全链路流程 +## 3. 输入 task_id +## 4. 输出文件 +## 5. 字段数据字典 +## 6. 状态字段说明 +## 7. asset_quality 实际优先级 +## 8. 真实数据与 fallback 边界 +## 9. API 使用方式 +## 10. 前端使用方式 +## 11. Docker/API 验收清单 +## 12. 前端验收清单 +## 13. Pending 风险 +``` + +## 二、必须写清楚定位 + +必须明确写入： + +```text +content_asset 当前是结构完整、质量可标注的内容资产表。 +它不等于全量真实评论和真实语音文案均已完成。 +真实评论仍依赖 CDP。 +真实 ASR 仍依赖 Whisper/ASR。 +标题/描述 fallback 不能表述为真实语音文案。 +``` + +## 三、全链路流程必须写清楚 + +写入流程： + +```text +1. POST /scrape/search 生成 search_result.csv 和 search_title_clean.csv +2. 可选：comments task 生成 comments_clean.csv +3. 可选：scripts task 生成 script_sources.csv / script_raw.csv / script_clean.csv +4. POST /scrape/merge 输入 search_task_id、comments_task_id、scripts_task_id +5. merge task 生成 content_asset.jsonl / content_asset.csv +6. GET /scrape/status/{merge_task_id} 查看状态 +7. GET /scrape/result/{merge_task_id} 下载主结果 +8. GET /scrape/data/preview/{merge_task_id} 预览 +9. GET /scrape/data/export?task_id={merge_task_id} 原样下载 +10. 前端 DataPage / TaskDetailPage 查看和下载 +``` + +## 四、输入 task_id 必须写清楚 + +```text +search_task_id：必需 +comments_task_id：可选 +scripts_task_id：可选 +merge_task_id：merge 创建后返回，用于 status/result/preview/export/前端访问 +``` + +说明： + +```text +没有 comments_task_id 不应失败，comment_data_status 会标记 pending_cdp。 +没有 scripts_task_id 不应失败，asr_data_status 会标记 pending_asr 或 missing。 +``` + +## 五、输出文件必须写清楚 + +```text +workspaces//outputs/content_asset.jsonl +workspaces//outputs/content_asset.csv +``` + +容器路径： + +```text +/app/workspaces//outputs/ +``` + +宿主路径： + +```text +G:\MediaCrawler\workspaces\\outputs\ +``` + +输出要求： + +```text +CSV：UTF-8-SIG，带 BOM +JSONL：ensure_ascii=False，中文正常 +``` + +## 六、字段数据字典必须包含 30 个字段 + +必须记录以下字段： + +```text +source_keyword +platform +video_id +aweme_id +aweme_url +raw_title +clean_title +desc +clean_desc +topic +pain_point +teaching_angle +nickname +liked_count +collected_count +comment_count +share_count +total_engagement +valid_comment_count +top_valid_comments +comment_pain_tags +script_source_status +script_source_quality +script_clean_text +script_clean_source +script_clean_quality +comment_data_status +asr_data_status +asset_quality +created_at +``` + +建议表格格式： + +```markdown +| 字段 | 来源 | 含义 | 空值/默认值 | 备注 | +|---|---|---|---|---| +``` + +字段来源必须区分： + +```text +search_result.csv +search_title_clean.csv +comments_clean.csv +script_sources.csv +script_raw.csv +script_clean.csv +builder 派生 +``` + +## 七、状态字段必须写清楚 + +### comment_data_status + +写入枚举： + +```text +available：存在 comments_clean.csv 且有有效评论 +empty：存在 comments_clean.csv 但无有效评论 +pending_cdp：未提供 comments_task_id 或缺 comments_clean.csv +``` + +### asr_data_status + +写入枚举： + +```text +available：script_clean_source=asr_raw，表示使用真实 ASR 文案 +fallback_title：使用 source_clean_title fallback +fallback_desc：使用 source_title_desc fallback +dependency_missing：script_raw.asr_status=dependency_missing +pending_asr：缺少 scripts task 或 ASR 未完成 +missing：无可用文案 +``` + +### script_clean_source + +写清楚： + +```text +asr_raw：真实 ASR 文案 +source_clean_title：标题/清洗标题 fallback +source_title_desc：标题/描述 fallback +missing：无可用文案 +``` + +## 八、asset_quality 实际优先级必须按代码写 + +必须查询 `douyin_scraper/content_asset.py` 中实际实现，以代码为准。 + +文档中要明确： + +```text +asset_quality 不是绝对内容质量评分，而是当前资产完整度/可用度标记。 +``` + +需要解决 T017-5-4-0 发现的语义重叠： + +```text +文档曾同时描述： +- 有 title_clean + fallback 为 medium +- 缺真实 ASR 但有 fallback 为 partial + +本轮要按实际代码优先级写清楚，避免歧义。 +``` + +如果实际实现是先 medium 后 partial，就写成： + +```text +当前实现中，medium 优先于 partial。 +有 title_clean 且有 fallback 文案时，通常会被标记为 medium。 +partial 用于更低完整度但仍有 fallback 文案的情况。 +``` + +但必须以代码为准，不要凭空写。 + +## 九、API 使用方式必须写清楚 + +### POST /scrape/merge + +必须写 task-id 模式示例： + +```bash +curl.exe -X POST http://localhost:18080/scrape/merge ^ + -H "Content-Type: application/json" ^ + -d "{\"search_task_id\":\"\",\"comments_task_id\":\"\",\"scripts_task_id\":\"\"}" +``` + +说明 comments/scripts 可选。 + +### GET /scrape/status/{task_id} + +```bash +curl.exe http://localhost:18080/scrape/status/ +``` + +说明应返回： + +```text +content_asset_jsonl +content_asset_csv +content_asset_stats +``` + +### GET /scrape/result/{task_id} + +```bash +curl.exe -o content_asset.csv http://localhost:18080/scrape/result/ +``` + +说明 merge task 会优先返回 `content_asset.csv`。 + +### GET /scrape/data/preview/{task_id} + +```bash +curl.exe "http://localhost:18080/scrape/data/preview/?limit=20" +``` + +说明返回： + +```text +task_id +file_name +format +total_rows +rows +``` + +### GET /scrape/data/export?task_id={task_id} + +必须使用真实路径： + +```bash +curl.exe -o content_asset_export.csv "http://localhost:18080/scrape/data/export?task_id=" +``` + +明确： + +```text +GET export 是原始文件下载，保留 content_asset.csv 完整 schema 和 BOM。 +``` + +### POST /scrape/data/export + +写清楚： + +```text +POST export 是批量旧七字段归一化导出。 +输出字段为 video_id/platform/script_text/likes/favorites/shares/comments。 +它不是 content_asset 完整字段导出。 +``` + +## 十、清理错误旧路径 + +全仓库文档中的 GET export 示例应统一为： + +```text +/scrape/data/export?task_id= +``` + +注意：只改文档，不改代码。 + +## 十一、前端使用方式 + +记录： + +```text +DataPage: +- 每个 task 显示一个主结果文件 +- content_asset 显示为“内容资产表” +- 默认展示核心列 +- 可切换查看全部字段 +- “下载原始文件”保留完整 content_asset.csv +- “批量导出旧七字段”输出旧兼容格式 + +TaskDetailPage: +- merge task 显示“主结果：内容资产表 content_asset.csv” +- 下载按钮为“下载内容资产表” +- content_asset_stats 结构化展示 +``` + +## 十二、验收清单 + +写入 Docker/API 验收命令。 + +### 后端验收 + +```powershell +curl.exe http://localhost:18080/health +curl.exe http://localhost:18080/scrape/status/ +curl.exe -o result.csv http://localhost:18080/scrape/result/ +curl.exe "http://localhost:18080/scrape/data/preview/?limit=20" +curl.exe -o export.csv "http://localhost:18080/scrape/data/export?task_id=" +``` + +### BOM 检查 + +```powershell +python - <<'PY' +from pathlib import Path +for name in ["result.csv", "export.csv"]: + p = Path(name) + print(name, p.read_bytes()[:3]) +PY +``` + +期望： + +```text +b'\xef\xbb\xbf' +``` + +### 测试命令 + +写入现有可用命令： + +```powershell +python -m pytest douyin_scraper\tests\test_content_asset.py -q +python -m pytest douyin_scraper\tests\test_core.py -q +python -m py_compile api\tasks.py api\routes.py douyin_scraper\content_asset.py +``` + +如写 api tests，必须说明： + +```text +完整 api/tests.py 当前存在既有 TestClient/httpx 兼容问题，不作为 T017-5 阻断项。 +``` + +### 前端构建 + +```powershell +cd web +npm run build +``` + +说明： + +```text +项目当前无独立 lint/typecheck 脚本。 +bundle size warning 属既有警告，不阻断 T017-5。 +``` + +## 十三、README / api/README 补充 + +### README.md + +增加简短入口： + +```text +Content Asset 内容资产表 +``` + +至少说明： + +```text +content_asset 是 T017-5 输出 +完整文档位置 docs/CONTENT_ASSET.md +API 宿主端口 localhost:18080 +前端 localhost:15173 +``` + +### api/README.md + +补充 API 说明： + +```text +/scrape/merge task-id 模式 +/scrape/result/{task_id} +/scrape/data/preview/{task_id} +/scrape/data/export?task_id={task_id} +POST /scrape/data/export +``` + +并链接 `docs/CONTENT_ASSET.md`。 + +## 十四、Pending 风险清单 + +必须写入： + +```text +CDP 真实评论采集未完成 +真实 ASR 未完成 +TestClient/httpx 兼容问题 +health degraded 因 CDP 未启动 +POST export 保持旧七字段契约 +content_asset 的 comment/asr 状态依赖输入质量 +后端 /data/list 仍返回多文件，前端做展示过滤 +前端创建 merge 尚未提供 task_id 输入 +``` + +并说明： + +```text +这些不阻断 T017-5 功能封版，但应作为后续独立任务管理。 +``` + +## 十五、验收方式 + +本轮完成后输出报告： + +```text +1. 开发前查询结果 +2. 新增/修改文档文件 +3. docs/CONTENT_ASSET.md 内容概览 +4. README / api/README 更新说明 +5. 错误旧路径修正情况 +6. 文档中的验收清单 +7. 是否允许正式封版 T017-5 +``` + +## 通过标准 + +```text +新增或更新正式 content_asset 文档 +字段数据字典完整 +状态枚举完整 +asset_quality 实际优先级无歧义 +API 示例使用 localhost:18080 +GET export 示例使用 ?task_id= +前端使用方式清楚 +验收清单可执行 +pending 风险写清楚 +不改业务代码 +不改前端逻辑 +不改后端逻辑 +不新增脚本 +不提交 commit +``` diff --git a/docs/tasks/T017-5-Release-API-1.md b/docs/tasks/T017-5-Release-API-1.md new file mode 100644 index 000000000..402efe2e0 --- /dev/null +++ b/docs/tasks/T017-5-Release-API-1.md @@ -0,0 +1,184 @@ +T017-5-Release-API-1：Content Asset API result / preview / export 实际拆分与提交。 + +本轮允许实际暂存并提交 Content Asset API 接入内容，但只允许提交 result selector、preview、GET export、POST export、字段映射和对应 API 测试。禁止混入 Builder、Web、Docs、runtime、api/webui、CDP、ASR。 + +## 当前状态 + +```text +A-D 前序成果已提交 +Builder 已提交 +Builder commit: 55199ea5 feat(content-asset): add content asset builder +暂存区为空 +未 push +``` + +## 目标 commit + +```text +feat(api): expose content asset result preview and export +``` + +## 允许范围 + +只允许包含： + +```text +api/tasks.py 中 merge/content_asset result selector +api/routes.py 中 content_asset preview 相关 hunk +api/routes.py 中 GET /scrape/data/export?task_id=... 原始主结果下载 +api/routes.py 中 POST export 对 content_asset 的旧七字段映射 +api/routes.py 中 _normalize_row 的 content_asset 字段别名映射 +Content Asset API 测试 +``` + +建议将 API 测试迁移或提交到独立测试文件： + +```text +douyin_scraper/tests/test_content_asset_api.py +douyin_scraper/tests/test_task_result_selection.py 中 merge/content_asset selector 部分 +``` + +## 禁止范围 + +```text +禁止提交 Builder 代码 +禁止提交 douyin_scraper/content_asset.py +禁止提交 douyin_scraper/tests/test_content_asset.py +禁止提交 Web +禁止提交 Docs +禁止提交 api/webui +禁止提交 runtime port +禁止提交 crawl_comments_v2.py +禁止提交 pyproject.toml +禁止处理 CDP +禁止处理真实 ASR/Whisper +禁止 git add . +禁止整文件暂存 routes.py +禁止混入 A-D 已提交内容 +``` + +## 执行前检查 + +```powershell +git status --short +git diff --cached --name-status +git diff --check +python -m py_compile api\routes.py api\tasks.py +python -m pytest douyin_scraper\tests\test_content_asset_api.py -q +python -m pytest douyin_scraper\tests\test_task_result_selection.py -q +``` + +如果 `test_content_asset_api.py` 尚不存在，请先从混杂的 `api/tests.py` 中迁移 Content Asset API 相关测试到该文件，再执行测试。 + +## 暂存方式 + +`api/tasks.py` 可以只暂存 merge/content_asset selector hunk： + +```powershell +git add -p api/tasks.py +``` + +`api/routes.py` 必须 patch 暂存： + +```powershell +git add -p api/routes.py +``` + +只选择 API hunk： + +```text +_normalize_row 的 content_asset 字段别名 +preview_data 对 content_asset 的主结果选择 +GET export 原始主结果下载 +POST export 对 content_asset 的旧七字段映射 +``` + +测试文件如已拆清，可整文件暂存： + +```powershell +git add douyin_scraper/tests/test_content_asset_api.py +git add -p douyin_scraper/tests/test_task_result_selection.py +``` + +## 暂存后检查 + +```powershell +git diff --cached --stat +git diff --cached --name-status +git diff --cached --check +git diff --cached -- api/routes.py api/tasks.py +``` + +必须确认 staged diff 不包含： + +```text +Builder +Web +Docs +runtime port +api/webui +CDP +ASR +pyproject.toml +``` + +## 测试 + +```powershell +python -m py_compile api\routes.py api\tasks.py +python -m pytest douyin_scraper\tests\test_content_asset_api.py -q +python -m pytest douyin_scraper\tests\test_task_result_selection.py -q +python -m pytest douyin_scraper\tests\test_content_asset.py -q +``` + +## 提交 + +只有 staged diff 和测试均通过后，执行： + +```powershell +git commit -m "feat(api): expose content asset result preview and export" +``` + +## 提交后验证 + +```powershell +git status --short +git log --oneline -8 +python -m pytest douyin_scraper\tests\test_content_asset_api.py -q +python -m pytest douyin_scraper\tests\test_task_result_selection.py -q +``` + +## 最终报告 + +输出： + +```markdown +# T017-5-Release-API-1 报告 + +## 1. 执行结论 + +## 2. Commit hash + +## 3. 暂存文件范围 + +## 4. staged diff 检查结果 + +## 5. 测试结果 + +## 6. 剩余工作区状态 + +## 7. 是否允许进入 Web 三组提交阶段 +``` + +## 通过标准 + +```text +只提交 Content Asset API +不包含 Builder/Web/Docs/runtime/api-webui +Content Asset API 测试 PASS +task result selector 测试 PASS +py_compile PASS +commit 创建成功 +暂存区最终为空 +未 push +``` diff --git a/docs/tasks/T017-5-Release-Builder-API-0.md b/docs/tasks/T017-5-Release-Builder-API-0.md new file mode 100644 index 000000000..faf59fb2e --- /dev/null +++ b/docs/tasks/T017-5-Release-Builder-API-0.md @@ -0,0 +1,238 @@ +T017-5-Release-Builder-API-0：Content Asset Builder / API / Web / Docs 剩余差异重新审查。 + +本轮只做查询和分组判断，不提交 commit。目标是在 A-D 四个前序 commit 已完成后，重新审查剩余工作区，确认 T017-5 Builder、API、Web、Docs 的提交边界。 + +## 当前状态 + +```text +A-D 前序成果已实际提交 +分支 ahead 4 +暂存区为空 +未 push +T017-5 Release-1 尚未开启 +``` + +已完成 commit： + +```text +be8fbcec387328ef7e754c90290ad646c853e245 feat(scraper): standardize search outputs in task workspaces +00a4946cc5dbe31a163b78d4bfa3f85ba5885ee0 feat(scraper): add raw and cleaned comment outputs +8bee152f028c48a301be8d4f2cbd4d2bddc17a98 feat(scraper): add cleaned search title outputs +2d7afaf8b61ea43977858560ef1745ca5aad5f09 feat(scraper): add script source raw and clean outputs +``` + +## 本轮目标 + +重新审查剩余 diff，拆分后续提交： + +```text +1. Content Asset Builder +2. Content Asset API result / preview / export +3. Web baseline +4. Runtime web port patch +5. T017-5 frontend UX +6. Docs +7. api/webui build 产物 +8. 暂缓项 +``` + +## 禁止 + +```text +禁止 git commit +禁止 git add . +禁止 git add web/ +禁止 push +禁止处理真实 CDP +禁止处理真实 Whisper/ASR +禁止接 AI/LLM +禁止提交 api/webui +禁止提交 crawl_comments_v2.py +禁止提交 pyproject.toml +禁止混入 runtime port 全局契约 +``` + +## 允许 + +```text +允许 git status / git diff 查询 +允许 git diff --cached 查询 +允许临时 git add -p 演练并 reset +允许运行测试 +禁止 commit +``` + +## 一、执行前检查 + +```powershell +git status --short +git diff --cached --name-status +git diff --stat +git log --oneline -8 +git diff --check +``` + +要求： + +```text +暂存区为空 +A-D commit 已在 log 中 +git diff --check PASS +``` + +## 二、剩余工作区分类 + +请输出剩余文件分组： + +### 1. T017-5 Builder + +候选范围： + +```text +douyin_scraper/content_asset.py +douyin_scraper/tests/test_content_asset.py +douyin_scraper/core.py 中 build_content_asset() +douyin_scraper/core.py get_paths() content_asset_jsonl/content_asset_csv/content_asset_stats +api/routes.py MergeRequest 三个 task-id 字段 +api/routes.py merge task-id content_asset 分支 +``` + +判断： + +```text +是否可形成独立 commit？ +是否仍依赖 A-D？ +是否有越界内容？ +建议 commit message？ +``` + +建议 message： + +```text +feat(content-asset): add content asset builder +``` + +### 2. T017-5 API + +候选范围： + +```text +api/tasks.py merge/content_asset selector +api/routes.py preview_data content_asset +api/routes.py GET export +api/routes.py POST export content_asset 映射 +Content Asset API 测试 +``` + +判断： + +```text +是否应先迁移 API 测试到 douyin_scraper/tests/test_content_asset_api.py？ +是否可独立提交？ +是否与 Builder 分开？ +``` + +建议 message： + +```text +feat(api): expose content asset result preview and export +``` + +### 3. Web baseline + +候选范围： + +```text +39 个 checkpoint web baseline 文件 +``` + +判断： + +```text +是否仍可按 Isolation-8 的三组顺序提交？ +是否仍需要先恢复 baseline？ +是否有 web/node_modules / tsbuildinfo 混入？ +``` + +建议 message： + +```text +feat(web): establish Vite frontend baseline +``` + +### 4. Runtime web port patch + +候选范围： + +```text +web/src/utils/constants.ts +web/vite.config.ts +``` + +建议 message： + +```text +chore(web): align frontend API and dev ports +``` + +### 5. T017-5 frontend + +候选范围： + +```text +web/src/api/client.ts +web/src/api/types.ts +web/src/pages/DataPage.tsx +web/src/pages/TaskDetailPage.tsx +``` + +建议 message： + +```text +feat(web): add content asset preview and download UX +``` + +### 6. Docs + +候选范围： + +```text +docs/CONTENT_ASSET.md +docs/index.md +README.md Content Asset hunk +api/README.md Content Asset hunk +docs/tasks/T017-5-* 可选归档 +``` + +建议拆成： + +```text +docs(content-asset): document schema workflow and acceptance +docs(tasks): archive T017-5 implementation notes +``` + +### 7. api/webui + +当前只审查，不提交。 + +判断： + +```text +已跟踪文件是否干净？ +未跟踪新 bundle 是否仍存在？ +是否继续暂缓到最后 build(webui)? +``` + +### 8. 暂缓项 + +明确列出： + +```text +crawl_comments_v2.py +pyproject.toml +CDP 真实采集 +真实 Whisper/ASR +全局 runtime port 契约 +api/webui build 产物 +未 +``` diff --git a/docs/tasks/T017-5-Release-Isolation-10.md b/docs/tasks/T017-5-Release-Isolation-10.md new file mode 100644 index 000000000..c514d4fa5 --- /dev/null +++ b/docs/tasks/T017-5-Release-Isolation-10.md @@ -0,0 +1,398 @@ +T017-5-Release-Isolation-10：手工生成 A-D 独立 patch、拆分 search/get_paths/selector、迁移四组测试。 + +本轮允许修改文件以完成前序 A-D 的可提交拆分，但禁止 commit。允许新增测试文件、迁移测试、手工拆分混合 hunk。完成后必须保持暂存区为空，输出报告。 + +## 当前状态 + +```text +T017-5：功能已 CLOSED +T017-5 Release-1：继续 NO-GO +T017-5-Release-Isolation-9：PASS +``` + +Isolation-9 结论： + +```text +A-D 功能测试通过，但不能直接形成完整独立提交。 +阻塞点： +1. search() 混合 A/C/D +2. get_paths() 混合 A/B/C/D/Content Asset +3. api/tasks.py selector 混合 search/comments/scripts/merge +4. 标准 18 字段转换、comments_clean、title_clean、scripts 实现处于大混合 hunk +5. 测试仍在综合 test_core.py / api/tests.py 中，未拆分 +``` + +## 本轮目标 + +把前序 A-D 变成后续可独立提交的结构： + +```text +A. T016 search_result.csv 标准化与 workspace 输出 +B. T017-2 comments_raw/comments_clean 输出 +C. T017-3 search_title_clean 输出 +D. T017-4 script_sources/script_raw/script_clean 输出 +``` + +本轮允许修改，但不提交 commit。 + +## 禁止 + +```text +禁止 git commit +禁止 git add . +禁止整文件提交 core.py / routes.py +禁止处理 api/webui +禁止处理 T017-5 content_asset +禁止处理 CDP 真实采集修复 +禁止处理真实 Whisper/ASR +禁止接 AI/LLM +禁止端口契约修改 +禁止格式化无关文件 +``` + +## 允许 + +```text +允许修改 douyin_scraper/core.py +允许修改 api/routes.py +允许修改 api/tasks.py +允许新增/拆分测试文件 +允许从 test_core.py 迁移测试 +允许从 api/tests.py 迁移 selector 测试 +允许运行 pytest +允许生成临时 patch +禁止 commit +``` + +## 一、执行前检查 + +```powershell +git status --short +git diff --cached --name-status +git diff --stat +``` + +要求： + +```text +暂存区为空 +api/webui 不纳入 +web 三组状态保持不处理 +``` + +## 二、拆分原则 + +### 1. `search()` 拆分 + +将混合逻辑整理为可独立理解的 helper，避免 A/C/D 混在同一 hunk 中。 + +建议结构： + +```text +search() + - 负责搜索任务主流程 + - 生成标准 search_result.jsonl/csv + - 可调用后续增强 helper，但每个 helper 独立 + +A helper: + - _convert_jsonl_to_standard_csv() + - 标准 18 字段 + - UTF-8-SIG + - 去重 + - total_engagement + +C helper: + - _do_clean_search_titles() + +D helper: + - _build_script_sources() +``` + +### 2. `get_paths()` 拆分 + +避免一个大 dict 混合全部输出字段。建议整理成分组 helper 或至少保持分块清晰： + +```text +base paths +search paths +comments paths +title paths +script paths +content_asset paths +``` + +Content Asset 三项必须保留，但不要混入 A-D patch。 + +### 3. `api/tasks.py` selector 拆分 + +建议从单一混合 selector 改成清晰的 task type priority map： + +```text +search/run_all selector +comments selector +scripts selector +merge/content_asset selector +fallback selector +``` + +后续提交时可以分别暂存： + +```text +A: search/run_all +B: comments +D: scripts +T017-5: merge/content_asset +``` + +### 4. `api/routes.py` 拆分 + +保持 endpoint 分区清楚： + +```text +search endpoint +comments endpoint +scripts endpoint +merge/content_asset endpoint +preview/export endpoint +``` + +不要把 T017-5 merge/export 与 A-D 混在同一个新增块里。 + +## 三、测试文件迁移 + +新增或整理以下测试文件： + +```text +douyin_scraper/tests/test_search_outputs.py +douyin_scraper/tests/test_comments_outputs.py +douyin_scraper/tests/test_title_clean.py +douyin_scraper/tests/test_script_outputs.py +``` + +### A：`test_search_outputs.py` + +迁移/新增： + +```text +搜索 workspace 输出测试 +search_result.jsonl 生成测试 +search_result.csv 生成测试 +标准 18 字段测试 +UTF-8-SIG 测试 +aweme_id / aweme_url 去重测试 +total_engagement 测试 +``` + +### B：`test_comments_outputs.py` + +迁移： + +```text +test_comments_raw_csv_export +test_comments_clean_rule_export +test_fetch_comments_writes_workspace_outputs +test_fetch_comments_subprocess_failure_keeps_outputs +``` + +### C：`test_title_clean.py` + +迁移/新增： + +```text +标题清洗规则测试 +hashtag 提取 +topic / pain_point / teaching_angle +从混合 search 测试中拆出 title-clean 断言 +``` + +### D：`test_script_outputs.py` + +迁移： + +```text +test_script_sources_export_from_search_csv +test_script_sources_fallback_to_search_jsonl +test_script_raw_from_script_sources_jsonl_success_and_skips +test_script_raw_fallback_to_csv_download_failed_and_empty_asr +test_script_raw_dependency_missing_does_not_download +test_script_clean_priority_from_raw_sources_and_title_clean +``` + +### API selector 测试 + +从 `api/tests.py` 迁移 search/comments/scripts selector 相关测试。 + +建议新增： + +```text +douyin_scraper/tests/test_task_result_selection.py +``` + +内容： + +```text +search selector +comments selector +scripts selector +fallback selector +``` + +merge/content_asset selector 留给 T017-5 API 独立提交，不混入 A-D。 + +## 四、A-D patch 验证 + +完成修改后，分别验证 A-D 是否可独立暂存。 + +### A 组验证 + +```powershell +git add -p -- douyin_scraper/core.py api/routes.py api/tasks.py douyin_scraper/tests/test_search_outputs.py douyin_scraper/tests/test_task_result_selection.py +git diff --cached --stat +git diff --cached -- douyin_scraper/core.py api/routes.py api/tasks.py +``` + +要求： + +```text +只包含 search_result.jsonl/csv、workspace、18字段、UTF-8-SIG、去重、search selector +不包含 comments +不包含 title +不包含 scripts +不包含 content_asset +``` + +验证后 reset： + +```powershell +git reset -- douyin_scraper/core.py api/routes.py api/tasks.py douyin_scraper/tests/test_search_outputs.py douyin_scraper/tests/test_task_result_selection.py +``` + +### B 组验证 + +```powershell +git add -p -- douyin_scraper/core.py api/routes.py api/tasks.py douyin_scraper/tests/test_comments_outputs.py +git diff --cached --stat +``` + +要求： + +```text +只包含 comments_raw/comments_clean、comments endpoint、comments selector、comments tests +不包含 title/scripts/content_asset/CDP真实修复 +``` + +验证后 reset。 + +### C 组验证 + +```powershell +git add -p -- douyin_scraper/core.py api/routes.py douyin_scraper/tests/test_title_clean.py +git diff --cached --stat +``` + +要求： + +```text +只包含 search_title_clean、title helper、title path、title tests +不包含 scripts/content_asset +``` + +验证后 reset。 + +### D 组验证 + +```powershell +git add -p -- douyin_scraper/core.py api/routes.py api/tasks.py douyin_scraper/tests/test_script_outputs.py +git diff --cached --stat +``` + +要求： + +```text +只包含 script_sources/script_raw/script_clean、dependency_missing 降级、scripts endpoint、scripts selector、scripts tests +不包含 content_asset builder +不安装 whisper +``` + +验证后 reset。 + +## 五、测试运行 + +运行： + +```powershell +python -m py_compile douyin_scraper\core.py api\routes.py api\tasks.py + +python -m pytest douyin_scraper\tests\test_search_outputs.py -q +python -m pytest douyin_scraper\tests\test_comments_outputs.py -q +python -m pytest douyin_scraper\tests\test_title_clean.py -q +python -m pytest douyin_scraper\tests\test_script_outputs.py -q +python -m pytest douyin_scraper\tests\test_task_result_selection.py -q +``` + +如有失败： + +```text +只修复与 A-D 拆分直接相关的问题。 +不要顺手修复 CDP/ASR/T017-5。 +``` + +## 六、最终状态检查 + +```powershell +git diff --cached --name-status +git status --short +git diff --stat +``` + +要求： + +```text +暂存区为空 +未 commit +api/webui 未处理 +A-D 测试文件已拆分 +工作区改动仅限本轮允许范围 +``` + +## 七、最终报告 + +输出： + +```markdown +# T017-5-Release-Isolation-10 报告 + +## 1. 执行结论 + +## 2. 代码拆分结果 + +## 3. 测试迁移结果 + +## 4. A 组暂存验证结果 + +## 5. B 组暂存验证结果 + +## 6. C 组暂存验证结果 + +## 7. D 组暂存验证结果 + +## 8. 测试运行结果 + +## 9. 暂存区最终状态 + +## 10. 是否允许进入 Isolation-11 / Release-1 +``` + +## 通过标准 + +```text +A-D 逻辑边界清晰 +测试文件已拆分 +A-D 均可独立暂存验证 +py_compile PASS +A-D 对应测试 PASS +暂存区最终为空 +未 commit +api/webui 未处理 +Release-1 继续 NO-GO，除非 A-D 已完全可执行提交 +``` diff --git a/docs/tasks/T017-5-Release-Isolation-11.md b/docs/tasks/T017-5-Release-Isolation-11.md new file mode 100644 index 000000000..0d2be9199 --- /dev/null +++ b/docs/tasks/T017-5-Release-Isolation-11.md @@ -0,0 +1,371 @@ +T017-5-Release-Isolation-11：实际形成 A-D 顺序提交候选并最终复核。 + +本轮允许实际暂存 A-D 四组候选提交内容，但禁止 commit。每组暂存后必须检查 staged diff、运行对应测试、记录结果，然后 reset，确保最终暂存区为空。 + +## 当前状态 + +```text +T017-5：功能已 CLOSED +T017-5 Release-1：继续 NO-GO +T017-5-Release-Isolation-10：PASS +``` + +Isolation-10 已完成： + +```text +A-D 代码边界已拆分 +五个独立测试文件已建立 +py_compile PASS +A-D + selector：19 passed +test_core.py：27 passed +api/tests.py::TestTaskManager：10 passed +git diff --check：PASS +暂存区为空 +未 commit +``` + +## 本轮目标 + +实际形成并复核四组候选提交： + +```text +A. T016 标准搜索输出 +B. T017-2 评论 raw/clean 输出 +C. T017-3 标题清洗输出 +D. T017-4 script sources/raw/clean 输出 +``` + +本轮只做暂存与复核，不提交。 + +## 禁止 + +```text +禁止 git commit +禁止 git add . +禁止 git add web/ +禁止处理 api/webui +禁止处理 T017-5 content_asset +禁止处理 CDP 真实采集修复 +禁止处理真实 Whisper/ASR +禁止接 AI/LLM +禁止端口契约修改 +禁止格式化无关文件 +``` + +## 允许 + +```text +允许按文件/patch 暂存 A-D 候选内容 +允许 git diff --cached 检查 +允许运行对应 pytest +允许 git reset 取消暂存 +禁止 commit +``` + +## 一、执行前检查 + +```powershell +git status --short +git diff --cached --name-status +git diff --check +python -m py_compile douyin_scraper\core.py api\routes.py api\tasks.py +``` + +要求： + +```text +暂存区为空 +git diff --check PASS +py_compile PASS +api/webui 不纳入 +``` + +## 二、A 组候选提交复核 + +Commit message： + +```text +feat(scraper): standardize search outputs in task workspaces +``` + +目标范围： + +```text +douyin_scraper/core.py +api/routes.py +api/tasks.py +douyin_scraper/tests/test_search_outputs.py +douyin_scraper/tests/test_task_result_selection.py +``` + +只能包含： + +```text +search_result.jsonl +search_result.csv +workspace 输出 +CSV UTF-8-SIG +标准 18 字段 +aweme_id / aweme_url 去重 +total_engagement +search/run_all selector +A 组测试 +``` + +不得包含： + +```text +comments +title_clean +scripts +content_asset +runtime port +api/webui +``` + +执行暂存后检查： + +```powershell +git diff --cached --stat +git diff --cached --name-status +git diff --cached -- douyin_scraper/core.py api/routes.py api/tasks.py +python -m pytest douyin_scraper\tests\test_search_outputs.py -q +python -m pytest douyin_scraper\tests\test_task_result_selection.py -q +``` + +记录： + +```text +staged 文件数 +insertions/deletions +测试结果 +是否存在越界内容 +``` + +然后 reset： + +```powershell +git reset -- douyin_scraper/core.py api/routes.py api/tasks.py douyin_scraper/tests/test_search_outputs.py douyin_scraper/tests/test_task_result_selection.py +git diff --cached --name-status +``` + +## 三、B 组候选提交复核 + +Commit message： + +```text +feat(scraper): add raw and cleaned comment outputs +``` + +目标范围： + +```text +douyin_scraper/core.py +api/routes.py +api/tasks.py +douyin_scraper/tests/test_comments_outputs.py +``` + +如 `crawl_comments_v2.py` 已经被拆清且不含 CDP/9222 越界内容，可列入；否则继续暂缓，不纳入 B。 + +只能包含： + +```text +comments_raw.jsonl/csv +comments_clean.jsonl/csv +comments endpoint +comments selector +comments tests +``` + +不得包含： + +```text +真实 CDP 修复 +9222/19222 端口契约调整 +title_clean +scripts +content_asset +``` + +执行： + +```powershell +git diff --cached --stat +git diff --cached --name-status +git diff --cached -- douyin_scraper/core.py api/routes.py api/tasks.py +python -m pytest douyin_scraper\tests\test_comments_outputs.py -q +``` + +然后 reset 对应文件，确保暂存区为空。 + +## 四、C 组候选提交复核 + +Commit message： + +```text +feat(scraper): add cleaned search title outputs +``` + +目标范围： + +```text +douyin_scraper/core.py +api/routes.py +douyin_scraper/tests/test_title_clean.py +``` + +只能包含： + +```text +search_title_clean.jsonl/csv +title clean helper +hashtags +topic +pain_point +teaching_angle +title path +search response title payload +title tests +``` + +不得包含： + +```text +scripts +content_asset +comments +runtime port +``` + +执行： + +```powershell +git diff --cached --stat +git diff --cached --name-status +git diff --cached -- douyin_scraper/core.py api/routes.py +python -m pytest douyin_scraper\tests\test_title_clean.py -q +``` + +然后 reset，暂存区清空。 + +## 五、D 组候选提交复核 + +Commit message： + +```text +feat(scraper): add script source raw and clean outputs +``` + +目标范围： + +```text +douyin_scraper/core.py +api/routes.py +api/tasks.py +douyin_scraper/tests/test_script_outputs.py +``` + +`pyproject.toml` 仅在可单独抽取 whisper 可选依赖 hunk 且不重写项目元数据时纳入；否则暂缓。 + +只能包含： + +```text +script_sources.jsonl/csv +script_raw.jsonl/csv +script_clean.jsonl/csv +dependency_missing 降级 +scripts endpoint +scripts selector +scripts tests +``` + +不得包含： + +```text +content_asset builder +真实 Whisper/ASR 集成 +安装依赖动作 +runtime port +api/webui +``` + +执行： + +```powershell +git diff --cached --stat +git diff --cached --name-status +git diff --cached -- douyin_scraper/core.py api/routes.py api/tasks.py +python -m pytest douyin_scraper\tests\test_script_outputs.py -q +``` + +然后 reset，暂存区清空。 + +## 六、总体验证 + +```powershell +python -m py_compile douyin_scraper\core.py api\routes.py api\tasks.py +python -m pytest douyin_scraper\tests\test_search_outputs.py douyin_scraper\tests\test_comments_outputs.py douyin_scraper\tests\test_title_clean.py douyin_scraper\tests\test_script_outputs.py douyin_scraper\tests\test_task_result_selection.py -q +python -m pytest douyin_scraper\tests\test_core.py -q +python -m pytest api\tests.py::TestTaskManager -q +git diff --check +git diff --cached --name-status +``` + +要求： + +```text +py_compile PASS +A-D + selector PASS +test_core.py PASS +api/tests.py::TestTaskManager PASS +git diff --check PASS +最终暂存区为空 +``` + +## 七、最终报告 + +输出： + +```markdown +# T017-5-Release-Isolation-11 报告 + +## 1. 执行结论 + +## 2. A 组候选提交复核 + +## 3. B 组候选提交复核 + +## 4. C 组候选提交复核 + +## 5. D 组候选提交复核 + +## 6. 总体验证结果 + +## 7. 暂存区最终状态 + +## 8. 是否允许进入 Isolation-12 / Release-1 +``` + +## 通过标准 + +```text +A-D 均可形成候选提交 +每组 staged diff 无越界内容 +每组对应测试 PASS +总体验证 PASS +暂存区最终为空 +未 commit +api/webui 未处理 +Release-1 仍需等待最终提交授权 +``` + +## 下一步建议 + +若 Isolation-11 PASS： + +```text +Isolation-12：A-D 实际提交授权前最终检查 +``` + +Release-1 仍不要直接开启，除非明确授权开始逐组 commit。 diff --git a/docs/tasks/T017-5-Release-Isolation-11F.md b/docs/tasks/T017-5-Release-Isolation-11F.md new file mode 100644 index 000000000..bfe25541d --- /dev/null +++ b/docs/tasks/T017-5-Release-Isolation-11F.md @@ -0,0 +1,239 @@ +T017-5-Release-Isolation-11F：api/webui diff-check 污染隔离/豁免方案确认。 + +本轮只处理全局 `git diff --check` 被既有 `api/webui/index.html` 尾随空格阻断的问题。目标是确认是否恢复、暂缓、还是在 A-D 提交流程中使用范围限定 diff-check。禁止提交 commit。 + +## 当前状态 + +```text +T017-5：功能已 CLOSED +T017-5 Release-1：继续 NO-GO +T017-5-Release-Isolation-11：条件性 NO-GO +``` + +Isolation-11 结果： + +```text +A-D 候选提交复核全部通过 +A-D 范围 git diff --check：PASS +全局 git diff --check：FAIL +失败仅涉及既有 api/webui/index.html 尾随空格 +api/webui 当前禁止处理、未暂存、未提交 +最终暂存区为空 +``` + +## 本轮目标 + +确认以下问题： + +```text +1. api/webui/index.html 的尾随空格是否来自构建产物既有污染 +2. 是否可以恢复 api/webui 到 HEAD 来解除全局 diff-check 阻断 +3. 如果不能恢复，是否允许 A-D 提交使用范围限定 git diff --check +4. 是否需要建立“api/webui 暂缓区”规则 +5. 是否允许重新进入 Isolation-12 +``` + +## 禁止 + +```text +禁止 git commit +禁止处理 A-D 业务代码 +禁止处理 T017-5 content_asset +禁止处理 web 源码 +禁止处理 CDP +禁止处理 ASR +禁止接 AI/LLM +禁止 git add . +禁止提交 api/webui +``` + +## 允许 + +```text +允许只读检查 api/webui +允许 git diff --check -- api/webui/index.html +允许 git diff --name-status -- api/webui +允许判断是否恢复 api/webui 到 HEAD +如果明确选择恢复，允许只恢复 api/webui 到 HEAD，但不得暂存或提交 +``` + +## 一、执行前检查 + +```powershell +git status --short +git diff --cached --name-status +git diff --check +git diff --check -- api/webui/index.html +git diff --name-status -- api/webui +``` + +记录： + +```text +全局 diff-check 是否只因 api/webui/index.html 失败 +api/webui 当前有哪些修改/删除/新增 +暂存区是否为空 +``` + +## 二、判断 api/webui 是否应恢复 + +api/webui 当前是构建产物区，Isolation-8/6R 已多次确认： + +```text +旧 bundle 删除 +新 bundle 未跟踪 +index.html / vite.svg 修改 +5 个 Logo 删除 +当前不允许提交 +``` + +请判断： + +```text +1. api/webui 当前变化是否全部属于构建产物/静态资源变化？ +2. 当前阶段是否应恢复 api/webui 到 HEAD？ +3. 恢复 api/webui 是否会影响 A-D、web baseline、runtime patch、frontend patch？ +4. 恢复 api/webui 是否会删除新 bundle 未跟踪文件？ +5. 是否需要额外清理未跟踪新 bundle？本轮默认不删除，除非明确确认。 +``` + +## 三、方案选择 + +输出三种方案并推荐一种。 + +### 方案 A：恢复 api/webui 到 HEAD + +用途： + +```text +解除全局 git diff --check 阻断 +保证 Release 隔离阶段不被构建产物污染 +``` + +建议命令： + +```powershell +git restore --worktree -- api/webui +``` + +然后检查： + +```powershell +git status --short api/webui +git diff --check +git diff --cached --name-status +``` + +注意： + +```text +此命令只恢复已跟踪的 api/webui 文件。 +未跟踪的新 bundle 可能仍存在。 +本轮不要删除未跟踪新 bundle，除非报告确认它们仍干扰 status 且需要清理。 +``` + +### 方案 B：不恢复 api/webui，A-D 使用范围限定 diff-check + +用途： + +```text +保留当前构建产物状态，A-D 提交时只对 A-D 范围执行 git diff --check +``` + +建议命令模式： + +```powershell +git diff --check -- douyin_scraper/core.py api/routes.py api/tasks.py douyin_scraper/tests/test_search_outputs.py douyin_scraper/tests/test_comments_outputs.py douyin_scraper/tests/test_title_clean.py douyin_scraper/tests/test_script_outputs.py douyin_scraper/tests/test_task_result_selection.py +``` + +风险： + +```text +全局 diff-check 仍失败 +提交前门禁不干净 +容易遗漏非 A-D 污染 +``` + +### 方案 C：将 api/webui 作为独立污染区暂缓 + +用途： + +```text +明确 api/webui 不参与 A-D/T017-5 源码提交 +只在最后 build(webui) 阶段统一处理 +``` + +可与方案 A 或 B 组合。 + +## 四、推荐判断 + +优先推荐： + +```text +如果 api/webui 当前变化全部是构建产物污染，执行方案 A：恢复已跟踪 api/webui 到 HEAD。 +未跟踪新 bundle 暂不删除，只记录。 +如果恢复后全局 diff-check PASS，则允许进入 Isolation-12。 +``` + +如果不能恢复： + +```text +明确采用方案 B + C： +A-D 提交只使用范围限定 diff-check +api/webui 保持暂缓 +Release-1 仍需更严格审批 +``` + +## 五、最终检查 + +```powershell +git diff --cached --name-status +git status --short +git diff --check +git status --short api/webui +``` + +要求： + +```text +暂存区为空 +未 commit +A-D 业务文件未改变 +api/webui 处理策略明确 +``` + +## 六、最终报告 + +输出： + +```markdown +# T017-5-Release-Isolation-11F 报告 + +## 1. 执行结论 + +## 2. diff-check 失败来源确认 + +## 3. api/webui 当前状态 + +## 4. 方案选择与理由 + +## 5. 是否恢复 api/webui 到 HEAD + +## 6. 最终 git diff --check 结果 + +## 7. 暂存区最终状态 + +## 8. 是否允许进入 Isolation-12 / Release-1 +``` + +## 通过标准 + +```text +确认 diff-check 阻断来源 +明确 api/webui 是恢复还是豁免 +暂存区最终为空 +未 commit +A-D 候选提交不被破坏 +允许或不允许进入 Isolation-12 有明确依据 +Release-1 继续等待最终授权 +``` diff --git a/docs/tasks/T017-5-Release-Isolation-12.md b/docs/tasks/T017-5-Release-Isolation-12.md new file mode 100644 index 000000000..4961a7a5b --- /dev/null +++ b/docs/tasks/T017-5-Release-Isolation-12.md @@ -0,0 +1,327 @@ +T017-5-Release-Isolation-12：A-D 实际提交授权前最终检查。 + +本轮只做最终检查和提交授权前确认，不执行 commit。目标是确认 A-D 四个前序候选提交是否已经可以进入实际逐组 commit 阶段。 + +## 当前状态 + +```text +T017-5：功能已 CLOSED +T017-5 Release-1：继续 NO-GO +T017-5-Release-Isolation-10：PASS +T017-5-Release-Isolation-11：A-D 候选复核通过 +T017-5-Release-Isolation-11F：PASS +``` + +Isolation-11F 已完成： + +```text +api/webui 已跟踪文件已恢复到 HEAD +全局 git diff --check PASS +未跟踪新 bundle 暂缓 +A-D 业务 diff 未受影响 +暂存区为空 +未 commit +``` + +## 本轮目标 + +最终确认以下内容： + +```text +1. A-D 四个候选提交边界是否仍然有效 +2. A-D 对应测试是否仍然通过 +3. git diff --check 是否通过 +4. api/webui 是否仍保持暂缓、不纳入 +5. 未跟踪新 bundle 是否仍不纳入 +6. 是否允许进入实际逐组 commit 阶段 +``` + +## 禁止 + +```text +禁止 git commit +禁止 git add . +禁止 git add web/ +禁止处理 api/webui +禁止删除未跟踪新 bundle +禁止处理 T017-5 content_asset +禁止处理 CDP 真实采集修复 +禁止处理真实 Whisper/ASR +禁止接 AI/LLM +禁止端口契约修改 +禁止格式化无关文件 +``` + +## 允许 + +```text +允许 git status / git diff 检查 +允许临时暂存 A-D 候选并检查 staged diff +允许 git reset 清空暂存区 +允许运行测试 +禁止 commit +``` + +## 一、执行前检查 + +```powershell +git status --short +git diff --cached --name-status +git diff --check +git status --short api/webui +git status --ignored --short +``` + +要求： + +```text +暂存区为空 +git diff --check PASS +api/webui 已跟踪文件无修改/删除 +未跟踪新 bundle 仍暂缓 +web/node_modules、web/tsconfig.tsbuildinfo、workspaces 正常 ignored +``` + +## 二、确认 A-D 候选提交边界 + +### A 组 + +Commit message： + +```text +feat(scraper): standardize search outputs in task workspaces +``` + +范围： + +```text +标准 search_result.jsonl/csv +workspace 输出 +18 字段 +UTF-8-SIG/BOM +去重 +total_engagement +search/run_all selector +test_search_outputs.py +test_task_result_selection.py 的 search/run_all 部分 +``` + +不得包含： + +```text +comments +title_clean +scripts +content_asset +runtime port +api/webui +``` + +### B 组 + +Commit message： + +```text +feat(scraper): add raw and cleaned comment outputs +``` + +范围： + +```text +comments_raw.jsonl/csv +comments_clean.jsonl/csv +comments endpoint +comments selector +test_comments_outputs.py +``` + +不得包含： + +```text +真实 CDP 修复 +9222/19222 端口契约 +title_clean +scripts +content_asset +``` + +`crawl_comments_v2.py` 仍不纳入，除非已经被拆清且不含 CDP/9222 越界内容。 + +### C 组 + +Commit message： + +```text +feat(scraper): add cleaned search title outputs +``` + +范围： + +```text +search_title_clean.jsonl/csv +title clean helper +hashtags +topic +pain_point +teaching_angle +title path +search response title payload +test_title_clean.py +``` + +不得包含： + +```text +scripts +content_asset +comments +runtime port +``` + +### D 组 + +Commit message： + +```text +feat(scraper): add script source raw and clean outputs +``` + +范围： + +```text +script_sources.jsonl/csv +script_raw.jsonl/csv +script_clean.jsonl/csv +dependency_missing 降级 +scripts endpoint +scripts selector +test_script_outputs.py +``` + +不得包含： + +```text +content_asset builder +真实 Whisper/ASR 集成 +安装依赖动作 +runtime port +api/webui +``` + +`pyproject.toml` 仍暂缓，除非 Whisper 可选依赖 hunk 已能独立抽取且不重写项目元数据。 + +## 三、重新运行总体验证 + +```powershell +python -m py_compile douyin_scraper\core.py api\routes.py api\tasks.py + +python -m pytest douyin_scraper\tests\test_search_outputs.py douyin_scraper\tests\test_comments_outputs.py douyin_scraper\tests\test_title_clean.py douyin_scraper\tests\test_script_outputs.py douyin_scraper\tests\test_task_result_selection.py -q + +python -m pytest douyin_scraper\tests\test_core.py -q + +python -m pytest api\tests.py::TestTaskManager -q + +git diff --check +``` + +要求： + +```text +py_compile PASS +A-D + selector PASS +test_core.py PASS +api/tests.py::TestTaskManager PASS +git diff --check PASS +``` + +完整 `api/tests.py` 路由部分如果仍受 Starlette/httpx TestClient 版本影响，继续记录为既有问题，不在本轮处理。 + +## 四、A-D staged diff 最终抽查 + +本轮不 commit，只抽查可暂存性。 + +对每组临时暂存后检查： + +```powershell +git diff --cached --stat +git diff --cached --name-status +git diff --cached --check +``` + +每组检查完必须 reset： + +```powershell +git reset -- <本组文件> +git diff --cached --name-status +``` + +最终要求： + +```text +每组 staged diff 无越界内容 +每组 staged diff --check PASS +每组测试 PASS +最终暂存区为空 +``` + +## 五、api/webui 暂缓确认 + +检查： + +```powershell +git status --short api/webui +git diff --name-status -- api/webui +``` + +要求： + +```text +已跟踪 api/webui 文件不应参与 A-D +未跟踪新 bundle 不纳入 +api/webui 保持 build(webui) 阶段再处理 +``` + +## 六、最终报告 + +输出： + +```markdown +# T017-5-Release-Isolation-12 报告 + +## 1. 执行结论 + +## 2. A-D 候选提交边界最终确认 + +## 3. 测试运行结果 + +## 4. staged diff 最终抽查结果 + +## 5. api/webui 暂缓确认 + +## 6. 暂存区最终状态 + +## 7. 是否允许进入实际逐组 commit / Release-1 +``` + +## 通过标准 + +```text +A-D 候选提交边界仍准确 +测试全部通过 +git diff --check PASS +api/webui 不纳入 +暂存区最终为空 +未 commit +可以明确判断是否允许进入实际逐组 commit +``` + +## 下一步 + +如果 Isolation-12 PASS： + +```text +下一步不是自动 Release-1，而是等待明确授权： +“开始按 A-D 顺序实际提交” +``` + +只有收到明确授权后，才允许逐组 commit。 diff --git a/docs/tasks/T017-5-Release-Isolation-4.md b/docs/tasks/T017-5-Release-Isolation-4.md new file mode 100644 index 000000000..1534e843f --- /dev/null +++ b/docs/tasks/T017-5-Release-Isolation-4.md @@ -0,0 +1,414 @@ +T017-5-Release-Isolation-4：精确暂存清单、checkpoint 恢复步骤和逐 commit patch manifest。 + +本轮只做只读查询和方案输出，不修改文件，不暂存，不提交 commit。 + +## 当前状态 + +```text +T017-5：功能已 CLOSED +T017-5 Release-1：继续 NO-GO +T017-5-Release-Isolation-1：PASS +T017-5-Release-Isolation-2：PASS +T017-5-Release-Isolation-3：PASS +``` + +Isolation-3 发现可恢复前功能基线： + +```text +Codex checkpoint tree: +aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 + +时间： +2026-06-17 13:38:16 UTC+8 +``` + +注意：这是内部 tree ref，不是普通 Git commit。必须先验证可访问性和恢复方式。本轮只查询，不执行恢复。 + +## 本轮目标 + +输出可执行但本轮不执行的 release 隔离 manifest： + +```text +1. checkpoint tree 可访问性验证方案 +2. web baseline 39 文件恢复方案 +3. api/README.md baseline 恢复方案 +4. api/tests.py baseline / 测试迁移方案 +5. 前序 A-D patch 暂存 manifest +6. T017-5 builder/API/frontend/docs patch manifest +7. api/webui 构建产物处理方案 +8. 是否允许进入 Isolation-5 +9. Release-1 是否仍 NO-GO +``` + +## 禁止 + +```text +禁止修改文件 +禁止 git add +禁止 git commit +禁止 git checkout / restore 实际恢复文件 +禁止删除文件 +禁止格式化 +禁止改业务代码 +禁止改前端 +禁止改后端 +禁止改 api/webui +禁止处理 CDP +禁止处理 ASR +禁止接 AI/LLM +禁止改端口 +``` + +## 必须只读查询 + +```bash +git status --short +git diff --stat +git cat-file -t aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 +git ls-tree -r --name-only aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 -- web +git ls-tree -r --name-only aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 -- api/README.md +git ls-tree -r --name-only aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 -- api/tests.py +git status --short web +git status --short api/webui +``` + +PowerShell 可用同等命令。 + +## 一、checkpoint 可访问性与恢复方案 + +回答： + +```text +1. checkpoint tree 是否可访问？ +2. git cat-file 返回类型是什么？ +3. checkpoint 中是否包含 web/ 39 个基线文件？ +4. checkpoint 中是否包含 api/README.md？ +5. checkpoint 中是否包含 api/tests.py？ +6. 推荐如何从 checkpoint 恢复指定路径？ +7. 恢复前是否需要保存当前 T017-5 前端文件补丁？ +8. 是否建议先生成 patch 文件而不是直接 checkout？ +``` + +只输出命令建议，不执行。 + +建议给出两种安全方案： + +### 方案 A：生成 patch 文件 + +```bash +git diff -- web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx web/src/api/client.ts web/src/api/types.ts > /tmp/t017-5-frontend.patch +git diff -- web/src/utils/constants.ts web/vite.config.ts > /tmp/runtime-port-web.patch +``` + +### 方案 B：使用 git checkout tree -- path 恢复基线 + +```bash +git checkout aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 -- web/ +``` + +注意：本轮不要执行。 + +## 二、web baseline manifest + +基于 checkpoint，输出 web baseline 文件清单。 + +必须包含： + +```text +package.json +package-lock.json +vite.config.ts +tsconfig*.json +tailwind/postcss 配置 +index.html +public/ +src/ +``` + +必须排除： + +```text +web/node_modules/ +*.tsbuildinfo +``` + +回答： + +```text +1. baseline 应提交哪些文件？ +2. 当前 T017-5 前端改动如何保存并后续恢复？ +3. 当前 runtime port 改动 constants/vite.config 如何保存并后续恢复？ +4. web baseline commit 是否应包含 T017-5 前端改动？ +5. 建议 commit message +``` + +目标 commit： + +```text +feat(web): establish Vite frontend baseline +``` + +## 三、api/README.md manifest + +回答： + +```text +1. checkpoint 中是否有 api/README.md 前功能版本？ +2. 是否建议先恢复并提交 api/README.md baseline？ +3. 当前 api/README.md 中哪些内容属于 runtime port？ +4. 当前 api/README.md 中哪些内容属于 content_asset？ +5. 后续拆分 commit message 是什么？ +``` + +建议 commit： + +```text +docs(api): establish API usage guide baseline +docs(api): document content asset workflow +``` + +端口内容如单独提交： + +```text +docs(api): align API port and CDP guidance +``` + +## 四、api/tests.py manifest + +回答： + +```text +1. checkpoint 中是否有 api/tests.py 前功能版本？ +2. 当前 api/tests.py 新增 content_asset 测试有哪些函数？ +3. 是否建议恢复 api/tests.py baseline？ +4. 是否建议迁移 content_asset API 测试到 douyin_scraper/tests/？ +5. 建议新测试文件名和测试函数清单 +6. 迁移前是否阻断 Release-1？ +``` + +建议目标文件： + +```text +douyin_scraper/tests/test_content_asset_api.py +douyin_scraper/tests/test_task_result_selection.py +``` + +注意：本轮只列方案，不迁移。 + +## 五、前序 A-D patch manifest + +为以下前序提交列出精确 hunk manifest。 + +```text +A. T016 search_result.csv 标准化与 workspace 输出 +B. T017-2 comments_clean 输出 +C. T017-3 search_title_clean 输出 +D. T017-4 script_sources / script_raw / script_clean 输出 +``` + +每组输出： + +```markdown +### A. T016 search_result.csv 标准化与 workspace 输出 + +Commit: +feat(scraper): standardize search outputs in task workspaces + +Files: +- ... + +Hunks: +- core.py: ... +- routes.py: ... +- api/tasks.py: ... +- tests: ... + +Suggested staging: +- whole file: +- git add -p: +- manual hunk edit required: + +Validation: +- commands to run: +``` + +不要执行任何 staging。 + +## 六、runtime port patch manifest + +因为 Isolation-3 发现端口仍有冲突： + +```text +其他配置/文档使用 19222 +douyin_scraper/config.py 默认仍是 9222 +``` + +输出： + +```text +1. 当前所有 9222 / 19222 残留位置 +2. 哪些属于运行契约提交 +3. 是否必须先修一致性再提交 runtime port +4. 建议 commit message +5. 本轮是否允许处理：不允许，只给方案 +``` + +建议 commit： + +```text +chore(runtime): align API web and CDP port contracts +``` + +## 七、T017-5 patch manifest + +分四组： + +```text +Builder +API +Frontend +Docs +``` + +### Builder + +```text +douyin_scraper/content_asset.py +douyin_scraper/tests/test_content_asset.py +core.py build_content_asset() +core.py get_paths() content_asset 三项 +routes.py MergeRequest task id 字段 +routes.py merge task-id 分支 +``` + +### API + +```text +api/tasks.py merge selector +routes.py preview/result/export content_asset 相关 hunk +content_asset API tests，建议迁移后提交 +``` + +### Frontend + +```text +DataPage.tsx +TaskDetailPage.tsx +client.ts +types.ts +``` + +前提： + +```text +web baseline 已提交 +runtime port web 两文件已处理 +``` + +### Docs + +```text +docs/CONTENT_ASSET.md +docs/index.md +README Content Asset hunk +api/README Content Asset hunk +docs/tasks/T017-5-* 可选归档 +``` + +每组输出： + +```text +文件范围 +是否整文件 +是否 git add -p +依赖哪个前序 commit +建议 commit message +``` + +## 八、api/webui manifest + +输出： + +```text +1. 当前 api/webui 删除/新增/修改列表 +2. 哪些删除需要恢复 +3. 是否等待前端源码提交后重新 build +4. 是否需要保留 logo +5. 最终提交文件范围 +6. 建议 commit message +``` + +建议： + +```text +build(webui): rebuild static frontend bundle +``` + +但明确： + +```text +api/webui 仍暂缓，不能在 Release-Isolation 阶段提交。 +``` + +## 九、最终判断 + +回答： + +```text +是否允许进入 T017-5-Release-Isolation-5？ +是否允许进入 T017-5-Release-1？ +``` + +建议： + +```text +Isolation-5 可作为实际“patch 文件生成与恢复演练”阶段，但仍不提交。 +Release-1 继续 NO-GO，直到： +- web baseline 已恢复并提交 +- 前序 A-D 生产链路已提交 +- API 测试已迁移或明确处理 +- runtime port 契约一致 +- api/webui 暂缓或干净重建策略明确 +``` + +## 最终报告格式 + +```markdown +# T017-5-Release-Isolation-4 查询报告 + +## 1. 查询结论 + +## 2. checkpoint 可访问性与恢复方案 + +## 3. web baseline manifest + +## 4. api/README.md manifest + +## 5. api/tests.py manifest + +## 6. 前序 A-D patch manifest + +## 7. runtime port patch manifest + +## 8. T017-5 patch manifest + +## 9. api/webui manifest + +## 10. 是否允许进入 Isolation-5 / Release-1 +``` + +## 通过标准 + +```text +完成只读查询 +确认 checkpoint 是否可访问 +明确 web baseline 恢复方案 +明确 api/README.md / api/tests.py 恢复或拆分方案 +明确前序 A-D patch manifest +明确 runtime port 一致性问题 +明确 T017-5 builder/API/frontend/docs manifest +明确 api/webui 暂缓方案 +未修改文件 +未暂存 +未提交 commit +``` diff --git a/docs/tasks/T017-5-Release-Isolation-5.md b/docs/tasks/T017-5-Release-Isolation-5.md new file mode 100644 index 000000000..6dab384d6 --- /dev/null +++ b/docs/tasks/T017-5-Release-Isolation-5.md @@ -0,0 +1,303 @@ +T017-5-Release-Isolation-5：临时补丁生成与 checkpoint 恢复演练。 + +本轮允许生成临时 patch 文件和临时目录验证，但禁止提交 commit，禁止正式暂存，禁止整目录 checkout，禁止污染真实 Git index。 + +## 当前状态 + +```text +T017-5：功能已 CLOSED +T017-5 Release-1：继续 NO-GO +T017-5-Release-Isolation-4：PASS +``` + +Isolation-4 已确认： + +```text +checkpoint tree 可访问： +aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 + +checkpoint 类型： +tree + +web/ 中包含历史 node_modules，因此禁止： +git checkout aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 -- web/ +``` + +## 本轮目标 + +只做恢复演练，不进入提交阶段： + +```text +1. 使用临时索引生成 patch +2. 验证 T017-5 前端 patch 可保存 +3. 验证 runtime port web patch 可保存 +4. 验证 api/README.md patch 可保存 +5. 验证 api/tests.py patch 可保存 +6. 验证 checkpoint 中 39 个 web baseline 文件可显式恢复 +7. 输出下一步 Isolation-6 是否可以进入 +``` + +## 禁止 + +```text +禁止 git add +禁止 git commit +禁止 git checkout -- web/ +禁止 git restore 整个 web/ +禁止污染真实 Git index +禁止修改业务代码 +禁止修改前端逻辑 +禁止修改后端逻辑 +禁止修改 api/webui +禁止删除 node_modules +禁止删除 workspaces +禁止处理 CDP +禁止处理 ASR +禁止接 AI/LLM +禁止改端口 +``` + +## 允许 + +仅允许： + +```text +生成临时 patch 文件 +使用临时 GIT_INDEX_FILE +使用临时目录验证 +只读检查 Git 对象 +``` + +临时文件建议放到仓库外，例如： + +```text +C:\tmp\t0175-release\ +``` + +## 一、准备临时目录 + +```powershell +New-Item -ItemType Directory -Force C:\tmp\t0175-release | Out-Null +``` + +## 二、验证 checkpoint + +```powershell +git cat-file -t aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 +git ls-tree -r --name-only aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 -- web | Measure-Object +git ls-tree -r --name-only aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 -- api/README.md +git ls-tree -r --name-only aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 -- api/tests.py +``` + +确认： + +```text +checkpoint 类型为 tree +web/ 中文件很多，但合法 baseline 只取显式 39 个路径 +api/README.md 存在 +api/tests.py 存在 +``` + +## 三、使用临时索引生成 patch + +注意：必须使用临时索引，不污染真实 index。 + +```powershell +$env:GIT_INDEX_FILE='C:\tmp\t0175-release\checkpoint.index' +git read-tree aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 +``` + +生成 patch： + +```powershell +git diff -- web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx web/src/api/client.ts web/src/api/types.ts > C:\tmp\t0175-release\t017-5-frontend.patch + +git diff -- web/src/utils/constants.ts web/vite.config.ts > C:\tmp\t0175-release\runtime-port-web.patch + +git diff -- api/README.md > C:\tmp\t0175-release\api-readme.patch + +git diff -- api/tests.py > C:\tmp\t0175-release\api-tests.patch +``` + +清理临时索引环境变量： + +```powershell +Remove-Item Env:GIT_INDEX_FILE +``` + +验证真实暂存区没有被污染： + +```powershell +git diff --cached --name-status +git status --short +``` + +要求： + +```text +git diff --cached --name-status 为空 +真实工作区除已有改动外，不应新增 staged 内容 +``` + +## 四、验证 patch 文件 + +检查 patch 是否存在且非空： + +```powershell +Get-Item C:\tmp\t0175-release\*.patch | Select-Object Name,Length +``` + +要求： + +```text +t017-5-frontend.patch 非空 +runtime-port-web.patch 非空 +api-readme.patch 非空 +api-tests.patch 非空 +``` + +如果某个 patch 为空，说明当前文件与 checkpoint 无差异，需在报告中说明。 + +## 五、显式 39 路径恢复演练方案 + +本轮不要实际恢复仓库文件，只输出命令清单。 + +必须使用显式路径，不允许整目录恢复。 + +39 个 web baseline 路径使用 Isolation-4 确认清单： + +```text +web/index.html +web/package.json +web/package-lock.json +web/postcss.config.js +web/public/vite.svg +web/tailwind.config.ts +web/tsconfig.json +web/tsconfig.node.json +web/vite.config.ts +web/src/App.tsx +web/src/index.css +web/src/main.tsx +web/src/vite-env.d.ts +web/src/api/client.ts +web/src/api/types.ts +web/src/hooks/usePolling.ts +web/src/hooks/useWebSocket.ts +web/src/store/useSettingsStore.ts +web/src/store/useTaskStore.ts +web/src/theme/theme.ts +web/src/utils/constants.ts +web/src/utils/format.ts +web/src/pages/AnalyticsPage.tsx +web/src/pages/CreateTaskPage.tsx +web/src/pages/DashboardPage.tsx +web/src/pages/DataPage.tsx +web/src/pages/SettingsPage.tsx +web/src/pages/TaskDetailPage.tsx +web/src/pages/TaskListPage.tsx +web/src/components/dashboard/HealthPanel.tsx +web/src/components/dashboard/RecentTasks.tsx +web/src/components/dashboard/StatsCards.tsx +web/src/components/layout/AppLayout.tsx +web/src/components/shared/ConnectionIndicator.tsx +web/src/components/shared/ErrorAlert.tsx +web/src/components/shared/LoadingOverlay.tsx +web/src/components/task/DeleteConfirmDialog.tsx +web/src/components/task/StatusBadge.tsx +web/src/components/task/TaskTypeIcon.tsx +``` + +后续真正恢复时，应使用： + +```powershell +git restore --source=aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 --worktree -- <39个明确路径> +``` + +禁止： + +```powershell +git checkout aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 -- web/ +git restore --source=aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 --worktree -- web/ +``` + +## 六、patch 应用演练方案 + +本轮不要应用到真实工作区。只输出后续建议命令。 + +后续顺序建议： + +```text +1. 保存 patch +2. 显式 39 路径恢复 web baseline +3. 提交 web baseline +4. 应用 runtime-port-web.patch +5. 提交 runtime port web 部分 +6. 应用 t017-5-frontend.patch +7. 提交 T017-5 frontend +``` + +patch 检查命令建议： + +```powershell +git apply --check C:\tmp\t0175-release\t017-5-frontend.patch +git apply --check C:\tmp\t0175-release\runtime-port-web.patch +``` + +注意：只有在恢复 baseline 后再执行 `git apply --check` 才有意义。 + +## 七、api/README.md 与 api/tests.py 后续方案 + +输出建议： + +```text +api/README.md: +1. 先从 checkpoint 恢复 baseline +2. 提交 API README baseline +3. 应用 runtime/API 文档 patch +4. 应用 content_asset 文档 patch + +api/tests.py: +1. 不建议直接应用综合 patch +2. 建议迁移 content_asset API 测试到 douyin_scraper/tests/test_content_asset_api.py +3. 建议迁移 task result selector 测试到 douyin_scraper/tests/test_task_result_selection.py +4. api/tests.py baseline 是否提交，后续单独决定 +``` + +## 八、最终报告 + +输出： + +```markdown +# T017-5-Release-Isolation-5 报告 + +## 1. 执行结论 + +## 2. checkpoint 验证结果 + +## 3. 临时 patch 生成结果 + +## 4. 真实 Git index 是否被污染 + +## 5. 39 路径恢复命令清单 + +## 6. patch 应用顺序建议 + +## 7. api/README.md 与 api/tests.py 后续处理 + +## 8. 是否允许进入 Isolation-6 / Release-1 +``` + +## 判断标准 + +```text +临时 patch 已生成 +真实 Git index 未污染 +未执行 git add +未执行 git commit +未整目录 checkout web/ +未修改 api/webui +明确 39 路径恢复方案 +明确后续 patch 应用顺序 +Release-1 继续 NO-GO +``` diff --git a/docs/tasks/T017-5-Release-Isolation-6.md b/docs/tasks/T017-5-Release-Isolation-6.md new file mode 100644 index 000000000..4df15c681 --- /dev/null +++ b/docs/tasks/T017-5-Release-Isolation-6.md @@ -0,0 +1,307 @@ +T017-5-Release-Isolation-6：实际恢复 web baseline 与 patch 应用验证。 + +本轮允许实际恢复显式 39 个 web baseline 文件，允许应用已生成并验证通过的 frontend/runtime patch，但禁止 commit，禁止处理 api/webui，禁止整目录 checkout/restore。 + +## 当前状态 + +```text +T017-5：功能已 CLOSED +T017-5 Release-1：继续 NO-GO +T017-5-Release-Isolation-5：PASS +``` + +Isolation-5 已完成： + +```text +checkpoint tree 可访问： +aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 + +四个 patch 已生成并 apply-check PASS： +C:\tmp\t0175-release\t017-5-frontend.patch +C:\tmp\t0175-release\runtime-port-web.patch +C:\tmp\t0175-release\api-readme.patch +C:\tmp\t0175-release\api-tests.patch + +39 个 web baseline 路径已验证存在。 +临时 baseline 无 node_modules、无 tsbuildinfo。 +真实 Git index 未污染。 +``` + +## 本轮目标 + +```text +1. 显式恢复 39 个 web baseline 文件 +2. 验证 web baseline 状态 +3. 应用 runtime-port-web.patch +4. 验证 runtime web patch +5. 应用 t017-5-frontend.patch +6. 验证 frontend patch +7. 不提交 commit +8. 判断是否允许进入 Isolation-7 +``` + +## 禁止 + +```text +禁止 git commit +禁止 git checkout -- web/ +禁止 git restore --source= --worktree -- web/ +禁止整目录恢复 web/ +禁止修改 api/webui/ +禁止处理后端业务代码 +禁止处理 CDP +禁止处理 ASR +禁止接 AI/LLM +禁止改端口以外文件 +禁止删除 node_modules +禁止删除 workspaces +``` + +## 允许 + +```text +允许 git restore 显式 39 路径 +允许 git apply runtime-port-web.patch +允许 git apply t017-5-frontend.patch +允许运行 npm build 验证 +允许 git status / git diff 检查 +禁止 commit +``` + +## 一、执行前检查 + +```powershell +git status --short +git diff --cached --name-status +Test-Path C:\tmp\t0175-release\t017-5-frontend.patch +Test-Path C:\tmp\t0175-release\runtime-port-web.patch +git apply --check C:\tmp\t0175-release\runtime-port-web.patch +git apply --check C:\tmp\t0175-release\t017-5-frontend.patch +``` + +要求： + +```text +暂存区为空 +patch 文件存在 +apply-check 通过 +``` + +如果 apply-check 在当前工作区失败，先停止并报告，不要强行 apply。 + +## 二、显式恢复 39 个 web baseline 文件 + +必须使用显式路径变量，不允许恢复整个 web 目录。 + +```powershell +$webBaseline = @( + 'web/index.html', + 'web/package.json', + 'web/package-lock.json', + 'web/postcss.config.js', + 'web/public/vite.svg', + 'web/tailwind.config.ts', + 'web/tsconfig.json', + 'web/tsconfig.node.json', + 'web/vite.config.ts', + 'web/src/App.tsx', + 'web/src/index.css', + 'web/src/main.tsx', + 'web/src/vite-env.d.ts', + 'web/src/api/client.ts', + 'web/src/api/types.ts', + 'web/src/hooks/usePolling.ts', + 'web/src/hooks/useWebSocket.ts', + 'web/src/store/useSettingsStore.ts', + 'web/src/store/useTaskStore.ts', + 'web/src/theme/theme.ts', + 'web/src/utils/constants.ts', + 'web/src/utils/format.ts', + 'web/src/pages/AnalyticsPage.tsx', + 'web/src/pages/CreateTaskPage.tsx', + 'web/src/pages/DashboardPage.tsx', + 'web/src/pages/DataPage.tsx', + 'web/src/pages/SettingsPage.tsx', + 'web/src/pages/TaskDetailPage.tsx', + 'web/src/pages/TaskListPage.tsx', + 'web/src/components/dashboard/HealthPanel.tsx', + 'web/src/components/dashboard/RecentTasks.tsx', + 'web/src/components/dashboard/StatsCards.tsx', + 'web/src/components/layout/AppLayout.tsx', + 'web/src/components/shared/ConnectionIndicator.tsx', + 'web/src/components/shared/ErrorAlert.tsx', + 'web/src/components/shared/LoadingOverlay.tsx', + 'web/src/components/task/DeleteConfirmDialog.tsx', + 'web/src/components/task/StatusBadge.tsx', + 'web/src/components/task/TaskTypeIcon.tsx' +) + +git restore --source=aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 --worktree -- $webBaseline +``` + +恢复后检查： + +```powershell +git status --short web +git ls-files web +Get-ChildItem web -Recurse -File | Where-Object { $_.FullName -match 'node_modules|tsbuildinfo' } | Select-Object -First 20 +``` + +要求： + +```text +web baseline 文件出现在未跟踪列表中 +不能出现 web/node_modules 普通未跟踪 +不能出现 tsbuildinfo +``` + +## 三、应用 runtime-port-web.patch + +```powershell +git apply --check C:\tmp\t0175-release\runtime-port-web.patch +git apply C:\tmp\t0175-release\runtime-port-web.patch +``` + +验证： + +```powershell +git diff -- web/src/utils/constants.ts web/vite.config.ts +git status --short web/src/utils/constants.ts web/vite.config.ts +``` + +要求： + +```text +只影响 constants.ts 和 vite.config.ts +端口应符合： +API 宿主 18080 +前端 dev 15173 +不要引入 8000 宿主访问 +``` + +## 四、应用 T017-5 frontend patch + +```powershell +git apply --check C:\tmp\t0175-release\t017-5-frontend.patch +git apply C:\tmp\t0175-release\t017-5-frontend.patch +``` + +验证： + +```powershell +git diff -- web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx web/src/api/client.ts web/src/api/types.ts +git status --short web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx web/src/api/client.ts web/src/api/types.ts +``` + +要求： + +```text +只影响四个 T017-5 前端文件 +DataPage / TaskDetailPage / client / types 恢复为已验收版本 +``` + +## 五、前端构建验证 + +```powershell +cd web +npm run build +cd .. +``` + +如果缺依赖： + +```text +不要重新安装 node_modules，先报告。 +``` + +如果 build 通过，记录结果。 + +注意： + +```text +npm run build 可能刷新 api/webui。 +如果刷新 api/webui，本轮不要提交 api/webui。 +只记录变化。 +``` + +## 六、确认 api/webui 未被本轮纳入 + +```powershell +git status --short api/webui +git diff --name-status -- api/webui +``` + +要求： + +```text +api/webui 可有 build 产生的变化，但本轮不得提交。 +如 build 刷新了 api/webui，只记录，不处理。 +``` + +## 七、最终状态检查 + +```powershell +git diff --cached --name-status +git status --short +git status --ignored --short +``` + +要求： + +```text +暂存区仍为空 +web/node_modules 仍 ignored +workspaces 仍 ignored +无 git add +无 git commit +``` + +## 八、最终报告 + +输出： + +```markdown +# T017-5-Release-Isolation-6 报告 + +## 1. 执行结论 + +## 2. 39 路径恢复结果 + +## 3. runtime-port-web.patch 应用结果 + +## 4. t017-5-frontend.patch 应用结果 + +## 5. 前端构建结果 + +## 6. api/webui 变化情况 + +## 7. git status / 暂存区状态 + +## 8. 是否允许进入 Isolation-7 / Release-1 +``` + +## 判断标准 + +```text +39 个 web baseline 文件已显式恢复 +未恢复整个 web/ +未带入 node_modules +未带入 tsbuildinfo +runtime web patch 应用成功 +T017-5 frontend patch 应用成功 +npm run build 通过或明确失败原因 +暂存区为空 +未 commit +api/webui 未被提交 +Release-1 继续 NO-GO +``` + +## 下一步建议 + +如果 Isolation-6 PASS，Isolation-7 建议定位为： + +```text +T017-5-Release-Isolation-7：web baseline / runtime patch / frontend patch 暂存分组演练 +``` + +仍然先不 commit。 diff --git a/docs/tasks/T017-5-Release-Isolation-6r.md b/docs/tasks/T017-5-Release-Isolation-6r.md new file mode 100644 index 000000000..7fac6dbb6 --- /dev/null +++ b/docs/tasks/T017-5-Release-Isolation-6r.md @@ -0,0 +1,297 @@ +T017-5-Release-Isolation-6R：修订版 web baseline 恢复与 patch 应用验证。 + +本轮允许显式恢复 39 个 web baseline 文件，并重新应用已生成 patch，但必须先确认当前文件已是 patch 后版本。禁止 commit，禁止整目录 checkout/restore，禁止处理 api/webui。 + +## 当前状态 + +```text +T017-5-Release-Isolation-6：STOP +原因：patch 正向 apply-check 失败，反向 apply-check 通过 +判断：当前 web 文件已经处于 patch 应用后的已验收版本 +``` + +上一轮未执行： + +```text +39路径恢复 +patch 应用 +npm run build +git add +git commit +``` + +暂存区仍为空。 + +## 本轮目标 + +修订执行顺序： + +```text +1. 确认当前六个 web 文件是 patch 后版本 +2. 保存当前状态备份信息 +3. 显式恢复 39 个 web baseline 文件 +4. 恢复后检查 patch 正向 apply-check +5. 应用 runtime-port-web.patch +6. 应用 t017-5-frontend.patch +7. 运行 npm run build +8. 确认暂存区为空 +9. 判断是否允许进入 Isolation-7 +``` + +## 禁止 + +```text +禁止 git commit +禁止 git add +禁止 git checkout -- web/ +禁止 git restore --source= --worktree -- web/ +禁止整目录恢复 web/ +禁止修改 api/webui +禁止处理后端业务代码 +禁止处理 CDP +禁止处理 ASR +禁止接 AI/LLM +禁止删除 node_modules +禁止删除 workspaces +``` + +## 允许 + +```text +允许 git restore 显式 39 路径 +允许 git apply runtime-port-web.patch +允许 git apply t017-5-frontend.patch +允许运行 npm run build +允许 git status / git diff 检查 +禁止 commit +``` + +## 一、执行前确认当前状态 + +```powershell +git status --short +git diff --cached --name-status +git apply --reverse --check C:\tmp\t0175-release\runtime-port-web.patch +git apply --reverse --check C:\tmp\t0175-release\t017-5-frontend.patch +``` + +要求： + +```text +暂存区为空 +两个 patch 的 reverse check 通过 +``` + +如果 reverse check 不通过，停止并报告。 + +## 二、记录当前已应用状态 + +输出这六个文件当前状态： + +```powershell +git status --short web/src/utils/constants.ts web/vite.config.ts +git status --short web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx web/src/api/client.ts web/src/api/types.ts +``` + +说明： + +```text +这些文件当前应是 patch 后版本。 +接下来恢复 baseline 会暂时覆盖它们。 +patch 文件已保存在 C:\tmp\t0175-release\。 +``` + +## 三、显式恢复 39 个 web baseline 文件 + +必须使用显式路径变量，不允许恢复整个 web 目录。 + +```powershell +$webBaseline = @( + 'web/index.html', + 'web/package.json', + 'web/package-lock.json', + 'web/postcss.config.js', + 'web/public/vite.svg', + 'web/tailwind.config.ts', + 'web/tsconfig.json', + 'web/tsconfig.node.json', + 'web/vite.config.ts', + 'web/src/App.tsx', + 'web/src/index.css', + 'web/src/main.tsx', + 'web/src/vite-env.d.ts', + 'web/src/api/client.ts', + 'web/src/api/types.ts', + 'web/src/hooks/usePolling.ts', + 'web/src/hooks/useWebSocket.ts', + 'web/src/store/useSettingsStore.ts', + 'web/src/store/useTaskStore.ts', + 'web/src/theme/theme.ts', + 'web/src/utils/constants.ts', + 'web/src/utils/format.ts', + 'web/src/pages/AnalyticsPage.tsx', + 'web/src/pages/CreateTaskPage.tsx', + 'web/src/pages/DashboardPage.tsx', + 'web/src/pages/DataPage.tsx', + 'web/src/pages/SettingsPage.tsx', + 'web/src/pages/TaskDetailPage.tsx', + 'web/src/pages/TaskListPage.tsx', + 'web/src/components/dashboard/HealthPanel.tsx', + 'web/src/components/dashboard/RecentTasks.tsx', + 'web/src/components/dashboard/StatsCards.tsx', + 'web/src/components/layout/AppLayout.tsx', + 'web/src/components/shared/ConnectionIndicator.tsx', + 'web/src/components/shared/ErrorAlert.tsx', + 'web/src/components/shared/LoadingOverlay.tsx', + 'web/src/components/task/DeleteConfirmDialog.tsx', + 'web/src/components/task/StatusBadge.tsx', + 'web/src/components/task/TaskTypeIcon.tsx' +) + +git restore --source=aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 --worktree -- $webBaseline +``` + +禁止执行： + +```powershell +git checkout aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 -- web/ +git restore --source=aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 --worktree -- web/ +``` + +## 四、恢复 baseline 后检查 + +```powershell +git status --short web +git apply --check C:\tmp\t0175-release\runtime-port-web.patch +git apply --check C:\tmp\t0175-release\t017-5-frontend.patch +``` + +要求： + +```text +两个 patch 的正向 apply-check 必须通过 +``` + +如果不通过，停止，不要强行 apply。 + +## 五、应用 runtime-port-web.patch + +```powershell +git apply C:\tmp\t0175-release\runtime-port-web.patch +``` + +验证： + +```powershell +git diff -- web/src/utils/constants.ts web/vite.config.ts +git apply --reverse --check C:\tmp\t0175-release\runtime-port-web.patch +``` + +要求： + +```text +只影响 constants.ts 和 vite.config.ts +reverse check 通过 +``` + +## 六、应用 T017-5 frontend patch + +```powershell +git apply C:\tmp\t0175-release\t017-5-frontend.patch +``` + +验证： + +```powershell +git diff -- web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx web/src/api/client.ts web/src/api/types.ts +git apply --reverse --check C:\tmp\t0175-release\t017-5-frontend.patch +``` + +要求： + +```text +只影响四个 T017-5 前端文件 +reverse check 通过 +``` + +## 七、前端构建验证 + +```powershell +cd web +npm run build +cd .. +``` + +如果缺依赖，不要安装，停止并报告。 + +注意： + +```text +npm run build 可能刷新 api/webui。 +本轮不要提交 api/webui。 +只记录 api/webui 状态。 +``` + +## 八、最终状态检查 + +```powershell +git diff --cached --name-status +git status --short +git status --ignored --short +git status --short api/webui +``` + +要求： + +```text +暂存区为空 +未 commit +web/node_modules 仍 ignored +workspaces 仍 ignored +api/webui 如有变化只记录，不处理 +``` + +## 九、最终报告 + +输出： + +```markdown +# T017-5-Release-Isolation-6R 报告 + +## 1. 执行结论 + +## 2. 执行前 reverse check 结果 + +## 3. 39 路径 baseline 恢复结果 + +## 4. patch 正向检查结果 + +## 5. runtime-port-web.patch 应用结果 + +## 6. t017-5-frontend.patch 应用结果 + +## 7. npm run build 结果 + +## 8. api/webui 变化情况 + +## 9. git status / 暂存区状态 + +## 10. 是否允许进入 Isolation-7 / Release-1 +``` + +## 通过标准 + +```text +执行前 reverse check 通过 +39 个 web baseline 文件显式恢复 +未恢复整个 web/ +未带入 node_modules +未带入 tsbuildinfo +runtime patch 正向检查并应用成功 +frontend patch 正向检查并应用成功 +npm run build 通过或明确失败原因 +暂存区为空 +未 commit +api/webui 未提交 +Release-1 继续 NO-GO +``` diff --git a/docs/tasks/T017-5-Release-Isolation-7.md b/docs/tasks/T017-5-Release-Isolation-7.md new file mode 100644 index 000000000..2eaae79b2 --- /dev/null +++ b/docs/tasks/T017-5-Release-Isolation-7.md @@ -0,0 +1,326 @@ +T017-5-Release-Isolation-7：web baseline / runtime patch / frontend patch 暂存分组演练。 + +本轮允许进行暂存分组演练，但禁止 commit。目标是验证 web baseline、runtime web patch、T017-5 frontend patch 能否被安全拆成 3 个提交组。 + +## 当前状态 + +```text +T017-5：功能已 CLOSED +T017-5 Release-1：继续 NO-GO +T017-5-Release-Isolation-6R：PASS +``` + +Isolation-6R 已完成： + +```text +39 个 web baseline 文件已显式恢复 +runtime-port-web.patch 已应用 +t017-5-frontend.patch 已应用 +npm run build PASS +暂存区为空 +web/ 仍为未跟踪 +api/webui 有构建变化但未处理 +``` + +## 本轮目标 + +验证以下 3 组能否安全暂存： + +```text +1. web baseline +2. runtime web port patch +3. T017-5 frontend patch +``` + +本轮只做 staged diff 演练，不提交 commit。 + +## 禁止 + +```text +禁止 git commit +禁止处理 api/webui +禁止提交 api/webui +禁止修改后端 +禁止修改 builder +禁止处理 CDP +禁止处理 ASR +禁止接 AI/LLM +禁止整目录 git add web/ +禁止 git add . +禁止删除文件 +禁止改端口以外内容 +``` + +## 允许 + +```text +允许 git add -N 指定 web baseline 文件 +允许 git add -p 指定文件 +允许 git diff --cached 检查 staged 内容 +允许 git reset 取消暂存 +允许 git status 检查 +禁止 commit +``` + +## 一、执行前检查 + +```powershell +git status --short +git diff --cached --name-status +git status --short api/webui +git status --ignored --short +``` + +要求： + +```text +暂存区为空 +web/node_modules ignored +web/tsconfig.tsbuildinfo ignored +workspaces ignored +api/webui 有变化但本轮不处理 +``` + +## 二、定义 39 个 web baseline 文件 + +使用 Isolation-6R 的 39 路径： + +```powershell +$webBaseline = @( + 'web/index.html', + 'web/package.json', + 'web/package-lock.json', + 'web/postcss.config.js', + 'web/public/vite.svg', + 'web/tailwind.config.ts', + 'web/tsconfig.json', + 'web/tsconfig.node.json', + 'web/vite.config.ts', + 'web/src/App.tsx', + 'web/src/index.css', + 'web/src/main.tsx', + 'web/src/vite-env.d.ts', + 'web/src/api/client.ts', + 'web/src/api/types.ts', + 'web/src/hooks/usePolling.ts', + 'web/src/hooks/useWebSocket.ts', + 'web/src/store/useSettingsStore.ts', + 'web/src/store/useTaskStore.ts', + 'web/src/theme/theme.ts', + 'web/src/utils/constants.ts', + 'web/src/utils/format.ts', + 'web/src/pages/AnalyticsPage.tsx', + 'web/src/pages/CreateTaskPage.tsx', + 'web/src/pages/DashboardPage.tsx', + 'web/src/pages/DataPage.tsx', + 'web/src/pages/SettingsPage.tsx', + 'web/src/pages/TaskDetailPage.tsx', + 'web/src/pages/TaskListPage.tsx', + 'web/src/components/dashboard/HealthPanel.tsx', + 'web/src/components/dashboard/RecentTasks.tsx', + 'web/src/components/dashboard/StatsCards.tsx', + 'web/src/components/layout/AppLayout.tsx', + 'web/src/components/shared/ConnectionIndicator.tsx', + 'web/src/components/shared/ErrorAlert.tsx', + 'web/src/components/shared/LoadingOverlay.tsx', + 'web/src/components/task/DeleteConfirmDialog.tsx', + 'web/src/components/task/StatusBadge.tsx', + 'web/src/components/task/TaskTypeIcon.tsx' +) +``` + +## 三、暂存组 1：web baseline 演练 + +由于 web/ 当前是未跟踪，不能直接 `git add web/`。 + +建议方式： + +```powershell +git add -N -- $webBaseline +``` + +然后只暂存 baseline 版本，排除 6 个已应用 patch 文件中的后续改动。 + +需要检查： + +```powershell +git diff --cached --name-status -- web +git diff --cached --stat -- web +``` + +目标 staged 内容： + +```text +39 个 web baseline 文件作为新增文件 +但不能包含 runtime port patch 和 T017-5 frontend patch 的改动 +``` + +注意：如果 `git add -N` 后无法把 baseline 和 patch 后内容拆开，则报告：web baseline 与后续 patch 在当前工作区仍需通过重新恢复 baseline 后单独暂存，而不是在当前状态直接暂存。 + +建议验证方式： + +```powershell +git diff --cached -- web/src/utils/constants.ts web/vite.config.ts +git diff --cached -- web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx web/src/api/client.ts web/src/api/types.ts +``` + +要求： + +```text +baseline commit 不应包含 18080/15173 runtime patch +baseline commit 不应包含 content_asset 前端 UX patch +``` + +如果 staged diff 已包含这些内容，立即 `git reset -- web`，并报告当前状态不适合直接拆 baseline。 + +## 四、取消暂存 + +无论成功与否，组 1 演练结束后都取消暂存： + +```powershell +git reset -- web +git diff --cached --name-status +``` + +要求暂存区为空。 + +## 五、暂存组 2：runtime web patch 演练 + +这组文件： + +```text +web/src/utils/constants.ts +web/vite.config.ts +``` + +前提：web baseline 已能作为独立提交存在。当前只是演练。 + +命令建议： + +```powershell +git add -p -- web/src/utils/constants.ts web/vite.config.ts +``` + +检查 staged diff： + +```powershell +git diff --cached -- web/src/utils/constants.ts web/vite.config.ts +``` + +目标： + +```text +只包含 18080 / 15173 等 runtime web port 改动 +不包含 content_asset 前端 UX +不包含无关格式化 +``` + +演练后取消暂存： + +```powershell +git reset -- web/src/utils/constants.ts web/vite.config.ts +git diff --cached --name-status +``` + +## 六、暂存组 3：T017-5 frontend patch 演练 + +这组文件： + +```text +web/src/pages/DataPage.tsx +web/src/pages/TaskDetailPage.tsx +web/src/api/client.ts +web/src/api/types.ts +``` + +前提：web baseline 和 runtime patch 已能独立提交。当前只是演练。 + +命令建议： + +```powershell +git add -p -- web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx web/src/api/client.ts web/src/api/types.ts +``` + +检查 staged diff： + +```powershell +git diff --cached -- web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx web/src/api/client.ts web/src/api/types.ts +``` + +目标： + +```text +只包含 content_asset 展示、下载、preview limit、核心列/全部字段切换、类型/API helper +不包含 runtime port +不包含无关前端基线 +``` + +演练后取消暂存： + +```powershell +git reset -- web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx web/src/api/client.ts web/src/api/types.ts +git diff --cached --name-status +``` + +## 七、最终检查 + +```powershell +git diff --cached --name-status +git status --short +git status --ignored --short +``` + +要求： + +```text +暂存区为空 +未 commit +web/node_modules ignored +web/tsconfig.tsbuildinfo ignored +workspaces ignored +api/webui 未被暂存 +``` + +## 八、最终报告 + +输出： + +```markdown +# T017-5-Release-Isolation-7 报告 + +## 1. 执行结论 + +## 2. web baseline 暂存演练结果 + +## 3. runtime web patch 暂存演练结果 + +## 4. T017-5 frontend patch 暂存演练结果 + +## 5. api/webui 是否保持未处理 + +## 6. 暂存区最终状态 + +## 7. 是否允许进入 Isolation-8 / Release-1 +``` + +## 判断标准 + +```text +明确 web baseline 是否可独立暂存 +明确 runtime web patch 是否可独立暂存 +明确 T017-5 frontend patch 是否可独立暂存 +所有演练后暂存区为空 +未 commit +api/webui 未处理 +Release-1 继续 NO-GO +``` + +## 下一步建议 + +如果 Isolation-7 PASS： + +```text +Isolation-8：实际 web baseline / runtime web / frontend 三组提交前最终验证 +``` + +仍然不进入 Release-1，直到前序 A-D 和 API 测试拆分也完成。 diff --git a/docs/tasks/T017-5-Release-Isolation-8.md b/docs/tasks/T017-5-Release-Isolation-8.md new file mode 100644 index 000000000..bba247fca --- /dev/null +++ b/docs/tasks/T017-5-Release-Isolation-8.md @@ -0,0 +1,316 @@ +T017-5-Release-Isolation-8：web baseline / runtime web / T017-5 frontend 三组提交前最终验证。 + +本轮只做最终验证和提交前清单确认，不执行 commit。允许临时暂存并 reset，但禁止提交。 + +## 当前状态 + +```text +T017-5：功能已 CLOSED +T017-5 Release-1：继续 NO-GO +T017-5-Release-Isolation-7：PASS +``` + +Isolation-7 已确认： + +```text +web baseline 可独立暂存，但必须先恢复 checkpoint baseline，再显式暂存 39 路径 +runtime web patch 可独立暂存 +T017-5 frontend patch 可独立暂存 +api/webui 未处理、未暂存 +真实暂存区最终为空 +``` + +## 本轮目标 + +最终确认以下三组未来 commit 的安全边界： + +```text +1. web baseline commit +2. runtime web port commit +3. T017-5 frontend commit +``` + +本轮只验证，不提交。 + +## 禁止 + +```text +禁止 git commit +禁止 git add . +禁止 git add web/ +禁止提交 api/webui +禁止修改后端 +禁止修改 builder +禁止处理 CDP +禁止处理 ASR +禁止接 AI/LLM +禁止删除 node_modules +禁止删除 workspaces +禁止修改业务逻辑 +``` + +## 允许 + +```text +允许 git status / git diff 检查 +允许 git add -N / git add -p 演练 +允许 git diff --cached 检查 +允许 git reset 取消暂存 +允许 npm run build 验证 +禁止 commit +``` + +## 一、执行前检查 + +```powershell +git status --short +git diff --cached --name-status +git status --ignored --short +git status --short api/webui +``` + +要求： + +```text +暂存区为空 +web/node_modules ignored +web/tsconfig.tsbuildinfo ignored +workspaces ignored +api/webui 不纳入本轮 +``` + +## 二、确认三组未来 commit + +### Commit 1：web baseline + +```text +feat(web): establish Vite frontend baseline +``` + +文件范围：39 个 checkpoint baseline 文件。 + +要求： + +```text +不包含 runtime 18080 / 15173 改动 +不包含 content_asset 前端 UX +不包含 api/webui +不包含 node_modules +不包含 tsbuildinfo +``` + +### Commit 2：runtime web port + +```text +chore(web): align frontend API and dev ports +``` + +文件范围： + +```text +web/src/utils/constants.ts +web/vite.config.ts +``` + +要求： + +```text +仅包含 API/WS 18080、Web dev 15173、Vite 环境变量化 +不包含 content_asset 前端 UX +``` + +### Commit 3：T017-5 frontend + +```text +feat(web): add content asset preview and download UX +``` + +文件范围： + +```text +web/src/api/client.ts +web/src/api/types.ts +web/src/pages/DataPage.tsx +web/src/pages/TaskDetailPage.tsx +``` + +要求： + +```text +仅包含 Content Asset 展示、下载、preview limit、核心列切换、API/type helper +不包含 runtime port +不包含 api/webui +``` + +## 三、验证 web baseline 文件清单 + +确认 39 个文件存在： + +```powershell +$webBaseline = @( + 'web/index.html', + 'web/package.json', + 'web/package-lock.json', + 'web/postcss.config.js', + 'web/public/vite.svg', + 'web/tailwind.config.ts', + 'web/tsconfig.json', + 'web/tsconfig.node.json', + 'web/vite.config.ts', + 'web/src/App.tsx', + 'web/src/index.css', + 'web/src/main.tsx', + 'web/src/vite-env.d.ts', + 'web/src/api/client.ts', + 'web/src/api/types.ts', + 'web/src/hooks/usePolling.ts', + 'web/src/hooks/useWebSocket.ts', + 'web/src/store/useSettingsStore.ts', + 'web/src/store/useTaskStore.ts', + 'web/src/theme/theme.ts', + 'web/src/utils/constants.ts', + 'web/src/utils/format.ts', + 'web/src/pages/AnalyticsPage.tsx', + 'web/src/pages/CreateTaskPage.tsx', + 'web/src/pages/DashboardPage.tsx', + 'web/src/pages/DataPage.tsx', + 'web/src/pages/SettingsPage.tsx', + 'web/src/pages/TaskDetailPage.tsx', + 'web/src/pages/TaskListPage.tsx', + 'web/src/components/dashboard/HealthPanel.tsx', + 'web/src/components/dashboard/RecentTasks.tsx', + 'web/src/components/dashboard/StatsCards.tsx', + 'web/src/components/layout/AppLayout.tsx', + 'web/src/components/shared/ConnectionIndicator.tsx', + 'web/src/components/shared/ErrorAlert.tsx', + 'web/src/components/shared/LoadingOverlay.tsx', + 'web/src/components/task/DeleteConfirmDialog.tsx', + 'web/src/components/task/StatusBadge.tsx', + 'web/src/components/task/TaskTypeIcon.tsx' +) + +$missing = $webBaseline | Where-Object { -not (Test-Path $_) } +$missing +``` + +要求： + +```text +missing = 空 +``` + +## 四、验证 patch 文件仍可用 + +```powershell +Test-Path C:\tmp\t0175-release\runtime-port-web.patch +Test-Path C:\tmp\t0175-release\t017-5-frontend.patch +``` + +并记录 patch 大小： + +```powershell +Get-Item C:\tmp\t0175-release\runtime-port-web.patch +Get-Item C:\tmp\t0175-release\t017-5-frontend.patch +``` + +## 五、三组提交前验证方式 + +本轮不要 commit，只输出建议命令和验证结果。 + +未来真正提交顺序应是： + +```text +1. 恢复 checkpoint baseline +2. git add 显式 39 路径 +3. commit web baseline +4. apply runtime-port-web.patch +5. git add constants.ts vite.config.ts +6. commit runtime web ports +7. apply t017-5-frontend.patch +8. git add DataPage/TaskDetailPage/client/types +9. commit T017-5 frontend +``` + +但本轮不提交。 + +## 六、前端构建验证 + +```powershell +cd web +npm run build +cd .. +``` + +记录结果。 + +注意： + +```text +构建可能刷新 api/webui。 +api/webui 不属于本轮提交。 +只记录状态，不暂存、不提交。 +``` + +## 七、最终状态检查 + +```powershell +git diff --cached --name-status +git status --short +git status --ignored --short +git status --short api/webui +``` + +要求： + +```text +最终暂存区为空 +未 commit +api/webui 未暂存 +ignored 正常 +``` + +## 八、最终报告 + +输出： + +```markdown +# T017-5-Release-Isolation-8 报告 + +## 1. 执行结论 + +## 2. 三组未来 commit 边界 + +## 3. 39 文件清单验证 + +## 4. patch 文件验证 + +## 5. npm run build 结果 + +## 6. api/webui 状态 + +## 7. 最终暂存区状态 + +## 8. 是否允许进入 Isolation-9 / Release-1 +``` + +## 判断标准 + +```text +明确三组 commit 边界 +39 文件清单完整 +patch 文件仍可用 +npm run build 通过或明确失败原因 +api/webui 未纳入 +暂存区最终为空 +Release-1 继续 NO-GO +``` + +## 下一步 + +如果 Isolation-8 PASS： + +```text +Isolation-9：前序 A-D patch manifest 转实际拆分演练 +``` + +Release-1 仍需等待前序 A-D、API 测试迁移、runtime port 全局一致性、api/webui 策略完成。 diff --git a/docs/tasks/T017-5-Release-Isolation-9.md b/docs/tasks/T017-5-Release-Isolation-9.md new file mode 100644 index 000000000..49e8c4818 --- /dev/null +++ b/docs/tasks/T017-5-Release-Isolation-9.md @@ -0,0 +1,311 @@ +T017-5-Release-Isolation-9：前序 A-D patch manifest 转实际拆分演练。 + +本轮只做前序成果拆分演练，不提交 commit。允许临时暂存并 reset，禁止实际提交。 + +## 当前状态 + +```text +T017-5：功能已 CLOSED +T017-5 Release-1：继续 NO-GO +T017-5-Release-Isolation-8：PASS +``` + +Isolation-8 已确认 web 三组未来提交边界： + +```text +1. web baseline：39 个 checkpoint 文件 +2. runtime web port：constants.ts + vite.config.ts +3. T017-5 frontend：client.ts + types.ts + DataPage.tsx + TaskDetailPage.tsx +``` + +但 Release-1 仍阻断，因为： + +```text +前序 A-D 尚未形成独立提交 +API 测试尚未迁移或明确处理 +runtime port 全局契约仍需统一 +api/webui 仍暂缓 +``` + +## 本轮目标 + +验证前序 A-D 是否可以拆成独立提交组： + +```text +A. T016 search_result.csv 标准化与 workspace 输出 +B. T017-2 comments_clean 输出 +C. T017-3 search_title_clean 输出 +D. T017-4 script_sources / script_raw / script_clean 输出 +``` + +本轮只演练，不 commit。 + +## 禁止 + +```text +禁止 git commit +禁止 git add . +禁止整文件提交 core.py / routes.py +禁止处理 api/webui +禁止修改业务逻辑 +禁止格式化 +禁止删除文件 +禁止处理 CDP +禁止处理 ASR +禁止接 AI/LLM +``` + +## 允许 + +```text +允许 git add -p 演练 +允许 git diff --cached 检查 +允许 git reset 取消暂存 +允许运行指定测试 +禁止 commit +``` + +## 一、执行前检查 + +```powershell +git status --short +git diff --cached --name-status +git diff --stat +``` + +要求： + +```text +暂存区为空 +api/webui 不纳入 +web/node_modules / workspaces ignored +``` + +## 二、A 组：T016 标准搜索输出演练 + +目标 commit： + +```text +feat(scraper): standardize search outputs in task workspaces +``` + +候选范围： + +```text +douyin_scraper/core.py +api/routes.py +api/tasks.py +搜索输出相关测试 +``` + +只演练暂存以下能力： + +```text +search_result.jsonl / search_result.csv +workspace 输出 +CSV UTF-8-SIG +标准 18 字段 +result 优先 search_result.csv +``` + +检查 staged diff： + +```powershell +git diff --cached -- douyin_scraper/core.py api/routes.py api/tasks.py +``` + +要求： + +```text +不包含 comments_clean +不包含 title_clean +不包含 script_sources/raw/clean +不包含 content_asset +``` + +演练后取消暂存： + +```powershell +git reset -- douyin_scraper/core.py api/routes.py api/tasks.py +git diff --cached --name-status +``` + +## 三、B 组：comments_clean 输出演练 + +目标 commit： + +```text +feat(scraper): add raw and cleaned comment outputs +``` + +候选范围： + +```text +douyin_scraper/core.py +api/routes.py +api/tasks.py +crawl_comments_v2.py +评论输出相关测试 +``` + +只演练暂存以下能力： + +```text +comments_raw.jsonl/csv +comments_clean.jsonl/csv +comment_data_status 前置能力 +comments result selector +``` + +要求： + +```text +不包含 title_clean +不包含 script_clean +不包含 content_asset +不处理 CDP 真实采集修复 +``` + +演练后 reset，保持暂存区为空。 + +## 四、C 组：search_title_clean 输出演练 + +目标 commit： + +```text +feat(scraper): add cleaned search title outputs +``` + +候选范围： + +```text +douyin_scraper/core.py +api/routes.py +标题清洗相关测试 +``` + +只演练暂存： + +```text +search_title_clean.jsonl/csv +topic / pain_point / teaching_angle +title noise clean +``` + +要求： + +```text +不包含 scripts +不包含 content_asset +``` + +演练后 reset。 + +## 五、D 组：script_sources / script_raw / script_clean 输出演练 + +目标 commit： + +```text +feat(scraper): add script source raw and clean outputs +``` + +候选范围： + +```text +douyin_scraper/core.py +api/routes.py +api/tasks.py +pyproject.toml 可选依赖 hunk +script 相关测试 +``` + +只演练暂存： + +```text +script_sources.jsonl/csv +script_raw.jsonl/csv +script_clean.jsonl/csv +dependency_missing 降级 +script result selector +``` + +要求： + +```text +不安装 whisper +不处理真实 ASR +不包含 content_asset builder +``` + +演练后 reset。 + +## 六、测试拆分判断 + +当前综合测试不适合直接提交。请判断并输出建议： + +```text +test_search_outputs.py +test_comments_outputs.py +test_title_clean.py +test_script_outputs.py +``` + +回答： + +```text +1. 哪些现有测试应该迁移到哪个文件？ +2. 是否必须先迁移测试再进入 Release-1？ +3. 哪些测试可以先暂缓？ +``` + +本轮不迁移，只判断。 + +## 七、最终检查 + +```powershell +git diff --cached --name-status +git status --short +``` + +要求： + +```text +暂存区为空 +未 commit +工作区不因演练产生新改动 +``` + +## 八、最终报告 + +输出： + +```markdown +# T017-5-Release-Isolation-9 报告 + +## 1. 执行结论 + +## 2. A 组 T016 暂存演练结果 + +## 3. B 组 comments 暂存演练结果 + +## 4. C 组 title 暂存演练结果 + +## 5. D 组 scripts 暂存演练结果 + +## 6. 测试拆分建议 + +## 7. 暂存区最终状态 + +## 8. 是否允许进入 Isolation-10 / Release-1 +``` + +## 判断标准 + +```text +明确 A-D 是否可独立暂存 +明确哪些 hunk 难以拆分 +明确测试拆分方案 +演练后暂存区为空 +未 commit +Release-1 继续 NO-GO，除非 A-D 和测试拆分已清楚到可执行 +``` diff --git a/docs/tasks/T017-5-Release-PreCommit-A-D.md b/docs/tasks/T017-5-Release-PreCommit-A-D.md new file mode 100644 index 000000000..2258d9475 --- /dev/null +++ b/docs/tasks/T017-5-Release-PreCommit-A-D.md @@ -0,0 +1,198 @@ +T017-5-Release-PreCommit-A-D：按 A→B→C→D 顺序实际提交前序成果。 + +本轮允许按顺序实际创建 4 个 commit，但必须严格按 A、B、C、D 分组执行。每个 commit 前必须暂存对应范围、检查 staged diff、运行对应测试；每个 commit 后必须记录 commit hash，并确认暂存区为空后再进入下一组。 + +## 当前状态 + +```text +T017-5：功能已 CLOSED +T017-5-Release-Isolation-12：PASS +A-D 实际逐组 commit：已获准 +``` + +## 严禁 + +```text +禁止 git add . +禁止 git add web/ +禁止处理 api/webui +禁止提交 content_asset +禁止提交 web 前端 +禁止提交 runtime port +禁止提交 crawl_comments_v2.py +禁止提交 pyproject.toml +禁止处理 CDP +禁止处理真实 Whisper/ASR +禁止接 AI/LLM +禁止格式化无关文件 +``` + +## Commit A + +Message： + +```text +feat(scraper): standardize search outputs in task workspaces +``` + +范围仅限： + +```text +标准 search_result.jsonl/csv +workspace 输出 +18 字段 +UTF-8-SIG/BOM +去重 +total_engagement +search/run_all selector +test_search_outputs.py +test_task_result_selection.py 的 search/run_all 部分 +``` + +提交前验证： + +```powershell +git diff --cached --check +python -m pytest douyin_scraper\tests\test_search_outputs.py -q +python -m pytest douyin_scraper\tests\test_task_result_selection.py -q +``` + +提交后记录 commit hash。 + +## Commit B + +Message： + +```text +feat(scraper): add raw and cleaned comment outputs +``` + +范围仅限： + +```text +comments_raw.jsonl/csv +comments_clean.jsonl/csv +comments endpoint +comments selector +test_comments_outputs.py +``` + +不得包含： + +```text +crawl_comments_v2.py +真实 CDP 修复 +9222/19222 端口契约 +title_clean +scripts +content_asset +``` + +验证： + +```powershell +git diff --cached --check +python -m pytest douyin_scraper\tests\test_comments_outputs.py -q +``` + +## Commit C + +Message： + +```text +feat(scraper): add cleaned search title outputs +``` + +范围仅限： + +```text +search_title_clean.jsonl/csv +title clean helper +hashtags +topic +pain_point +teaching_angle +title path +search response title payload +test_title_clean.py +``` + +验证： + +```powershell +git diff --cached --check +python -m pytest douyin_scraper\tests\test_title_clean.py -q +``` + +## Commit D + +Message： + +```text +feat(scraper): add script source raw and clean outputs +``` + +范围仅限： + +```text +script_sources.jsonl/csv +script_raw.jsonl/csv +script_clean.jsonl/csv +dependency_missing 降级 +scripts endpoint +scripts selector +test_script_outputs.py +``` + +不得包含： + +```text +content_asset builder +真实 Whisper/ASR 集成 +pyproject.toml +安装依赖动作 +runtime port +api/webui +``` + +验证： + +```powershell +git diff --cached --check +python -m pytest douyin_scraper\tests\test_script_outputs.py -q +``` + +## 四个 commit 完成后总体验证 + +```powershell +python -m py_compile douyin_scraper\core.py api\routes.py api\tasks.py +python -m pytest douyin_scraper\tests\test_search_outputs.py douyin_scraper\tests\test_comments_outputs.py douyin_scraper\tests\test_title_clean.py douyin_scraper\tests\test_script_outputs.py douyin_scraper\tests\test_task_result_selection.py -q +python -m pytest douyin_scraper\tests\test_core.py -q +python -m pytest api\tests.py::TestTaskManager -q +git diff --check +git status --short +``` + +## 最终报告 + +输出： + +```markdown +# T017-5-Release-PreCommit-A-D 报告 + +## 1. 执行结论 + +## 2. Commit A 结果与 hash + +## 3. Commit B 结果与 hash + +## 4. Commit C 结果与 hash + +## 5. Commit D 结果与 hash + +## 6. 总体验证结果 + +## 7. 剩余工作区状态 + +## 8. 是否允许进入 T017-5 Builder/API/Web/Docs 提交阶段 +``` diff --git a/docs/tasks/T017-5-Release-Web-1.md b/docs/tasks/T017-5-Release-Web-1.md new file mode 100644 index 000000000..9fb615348 --- /dev/null +++ b/docs/tasks/T017-5-Release-Web-1.md @@ -0,0 +1,290 @@ +T017-5-Release-Web-1：web baseline / runtime web / Content Asset UX 三组实际提交。 + +本轮允许按顺序创建 3 个 Web commit，但必须严格分组。禁止提交 api/webui 构建产物，禁止提交 Docs，禁止提交后端，禁止提交 runtime 全局端口契约。 + +## 当前状态 + +```text +A-D 前序成果已提交 +Builder 已提交 +API 已提交 +API commit: adcc7f29 feat(api): expose content asset result preview and export +暂存区为空 +未 push +``` + +## 目标提交顺序 + +```text +1. feat(web): establish Vite frontend baseline +2. chore(web): align frontend API and dev ports +3. feat(web): add content asset preview and download UX +``` + +## 严禁 + +```text +禁止 git add . +禁止 git add web/ +禁止提交 api/webui +禁止提交 Docs +禁止提交后端 +禁止提交 runtime 全局端口契约 +禁止提交 crawl_comments_v2.py +禁止提交 pyproject.toml +禁止处理 CDP +禁止处理真实 Whisper/ASR +禁止接 AI/LLM +禁止 push +``` + +## Commit 1：web baseline + +Message： + +```text +feat(web): establish Vite frontend baseline +``` + +只允许提交 39 个 checkpoint baseline 文件。 + +必须先显式恢复 baseline，再显式暂存 39 路径。 + +禁止整目录 restore： + +```powershell +git checkout aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 -- web/ +git restore --source=aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 --worktree -- web/ +``` + +使用显式路径： + +```powershell +$webBaseline = @( + 'web/index.html', + 'web/package.json', + 'web/package-lock.json', + 'web/postcss.config.js', + 'web/public/vite.svg', + 'web/tailwind.config.ts', + 'web/tsconfig.json', + 'web/tsconfig.node.json', + 'web/vite.config.ts', + 'web/src/App.tsx', + 'web/src/index.css', + 'web/src/main.tsx', + 'web/src/vite-env.d.ts', + 'web/src/api/client.ts', + 'web/src/api/types.ts', + 'web/src/hooks/usePolling.ts', + 'web/src/hooks/useWebSocket.ts', + 'web/src/store/useSettingsStore.ts', + 'web/src/store/useTaskStore.ts', + 'web/src/theme/theme.ts', + 'web/src/utils/constants.ts', + 'web/src/utils/format.ts', + 'web/src/pages/AnalyticsPage.tsx', + 'web/src/pages/CreateTaskPage.tsx', + 'web/src/pages/DashboardPage.tsx', + 'web/src/pages/DataPage.tsx', + 'web/src/pages/SettingsPage.tsx', + 'web/src/pages/TaskDetailPage.tsx', + 'web/src/pages/TaskListPage.tsx', + 'web/src/components/dashboard/HealthPanel.tsx', + 'web/src/components/dashboard/RecentTasks.tsx', + 'web/src/components/dashboard/StatsCards.tsx', + 'web/src/components/layout/AppLayout.tsx', + 'web/src/components/shared/ConnectionIndicator.tsx', + 'web/src/components/shared/ErrorAlert.tsx', + 'web/src/components/shared/LoadingOverlay.tsx', + 'web/src/components/task/DeleteConfirmDialog.tsx', + 'web/src/components/task/StatusBadge.tsx', + 'web/src/components/task/TaskTypeIcon.tsx' +) + +git restore --source=aeaead1458c6187b42b9c62bd104e2fb7ff1d1b9 --worktree -- $webBaseline +git add -- $webBaseline +``` + +提交前检查： + +```powershell +git diff --cached --stat +git diff --cached --name-status +git diff --cached --check +git diff --cached -- web/src/utils/constants.ts web/vite.config.ts +git diff --cached -- web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx web/src/api/client.ts web/src/api/types.ts +``` + +要求： + +```text +只包含 39 个 web baseline 文件 +不包含 18080 / 15173 runtime port patch +不包含 Content Asset UX +不包含 api/webui +不包含 node_modules +不包含 tsbuildinfo +``` + +提交： + +```powershell +git commit -m "feat(web): establish Vite frontend baseline" +``` + +## Commit 2：runtime web port + +Message： + +```text +chore(web): align frontend API and dev ports +``` + +应用 patch： + +```powershell +git apply --check C:\tmp\t0175-release\runtime-port-web.patch +git apply C:\tmp\t0175-release\runtime-port-web.patch +``` + +只允许暂存： + +```text +web/src/utils/constants.ts +web/vite.config.ts +``` + +```powershell +git add web/src/utils/constants.ts web/vite.config.ts +``` + +提交前检查： + +```powershell +git diff --cached --stat +git diff --cached --name-status +git diff --cached --check +git diff --cached -- web/src/utils/constants.ts web/vite.config.ts +``` + +要求： + +```text +只包含 API/WS 18080、Web dev 15173、Vite 环境变量化 +不包含 Content Asset UX +不包含 api/webui +``` + +提交： + +```powershell +git commit -m "chore(web): align frontend API and dev ports" +``` + +## Commit 3：T017-5 frontend UX + +Message： + +```text +feat(web): add content asset preview and download UX +``` + +应用 patch： + +```powershell +git apply --check C:\tmp\t0175-release\t017-5-frontend.patch +git apply C:\tmp\t0175-release\t017-5-frontend.patch +``` + +只允许暂存： + +```text +web/src/api/client.ts +web/src/api/types.ts +web/src/pages/DataPage.tsx +web/src/pages/TaskDetailPage.tsx +``` + +```powershell +git add web/src/api/client.ts web/src/api/types.ts web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx +``` + +提交前检查： + +```powershell +git diff --cached --stat +git diff --cached --name-status +git diff --cached --check +git diff --cached -- web/src/api/client.ts web/src/api/types.ts web/src/pages/DataPage.tsx web/src/pages/TaskDetailPage.tsx +``` + +要求： + +```text +只包含 Content Asset 展示、下载、preview limit、核心列切换、API/type helper +不包含 runtime port +不包含 api/webui +``` + +提交： + +```powershell +git commit -m "feat(web): add content asset preview and download UX" +``` + +## 三个 commit 完成后验证 + +```powershell +cd web +npm.cmd run build +cd .. + +git status --short +git diff --check +git log --oneline -10 +``` + +注意： + +```text +npm build 可能刷新 api/webui。 +api/webui 不得提交，本轮只记录。 +``` + +## 最终报告 + +输出： + +```markdown +# T017-5-Release-Web-1 报告 + +## 1. 执行结论 + +## 2. Web baseline commit hash + +## 3. Runtime web commit hash + +## 4. Content Asset frontend commit hash + +## 5. 构建验证结果 + +## 6. api/webui 状态 + +## 7. 剩余工作区状态 + +## 8. 是否允许进入 Docs 提交阶段 +``` + +## 通过标准 + +```text +3 个 Web commit 创建成功 +baseline 不含 runtime/content_asset +runtime 不含 content_asset +frontend 不含 runtime +api/webui 未提交 +npm build PASS 或明确失败原因 +暂存区最终为空 +未 push +``` From 906266bb11c3e00e00d94f01416c8d5d88054de2 Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 16:28:44 +0800 Subject: [PATCH 13/32] fix(api): add missing API utility helpers --- api/utils.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 api/utils.py diff --git a/api/utils.py b/api/utils.py new file mode 100644 index 000000000..81cdf1fed --- /dev/null +++ b/api/utils.py @@ -0,0 +1,62 @@ +""" +douyin_scraper.api.utils — API 安全工具函数 +============================================= +提供路径遍历防护和符号链接检测，防止恶意用户通过 API 参数 +访问 workspace 外的文件或通过符号链接逃逸。 +""" + +from pathlib import Path + +from fastapi import HTTPException + + +def validate_path_in_workspace(path_str: str, workspace: Path) -> Path: + """ + 验证路径 resolve 后必须位于 workspace resolve 路径之下， + 否则抛出 HTTPException(400) 阻止路径遍历攻击。 + + Args: + path_str: 用户提供的路径字符串 + workspace: 允许的工作空间根目录 + + Returns: + resolve 后的合法 Path 对象 + + Raises: + HTTPException: 路径遍历攻击时返回 400 + """ + resolved_workspace = workspace.resolve() + target = (resolved_workspace / path_str).resolve() if not Path(path_str).is_absolute() else Path(path_str).resolve() + + # 检查 target 是否以 workspace 开头 + try: + target.relative_to(resolved_workspace) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"路径遍历攻击被阻止: '{path_str}' 不在工作空间 '{resolved_workspace}' 内", + ) + + return target + + +def validate_no_symlink(file_path: Path) -> Path: + """ + 确保文件不是符号链接，防止通过符号链接读取任意文件。 + + Args: + file_path: 要检查的文件路径 + + Returns: + 原始 Path 对象（如果检查通过） + + Raises: + HTTPException: 文件是符号链接时返回 400 + """ + file_path = Path(file_path) + if file_path.exists() and file_path.is_symlink(): + raise HTTPException( + status_code=400, + detail=f"符号链接不允许: '{file_path}' 是一个符号链接", + ) + return file_path From 98168e9c991b7f342267af8734a636d3341d5109 Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 19:14:34 +0800 Subject: [PATCH 14/32] chore(runtime): align ports and deployment scaffolding --- .dockerignore | 42 ++++++++ .env.example | 14 +++ Dockerfile | 63 ++++++++++++ README.md | 8 +- api/deploy.sh | 194 +++++++++++++++++++++++++++++++++++++ api/douyin-scraper.service | 40 ++++++++ api/main.py | 11 ++- api/requirements.txt | 8 ++ config/base_config.py | 4 +- docker-compose.yml | 30 ++++++ douyin_scraper/core.py | 2 +- web/.env.example | 3 + 12 files changed, 408 insertions(+), 11 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 api/deploy.sh create mode 100644 api/douyin-scraper.service create mode 100644 api/requirements.txt create mode 100644 docker-compose.yml create mode 100644 web/.env.example diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..ac565de0a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,42 @@ +# Python +.venv/ +__pycache__/ +*.pyc +*.pyo +*.egg-info/ +dist/ +build/ +*.egg + +# Testing +.pytest_cache/ +.mypy_cache/ +htmlcov/ +.coverage + +# Project data +data/ +workspaces/ + +# Git +.git/ +.gitignore + +# Docs +docs/ +*.md +!README.md + +# Environment +.env +.env.* + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# Docker +docker-compose*.yml +Dockerfile diff --git a/.env.example b/.env.example index ec8997033..3ff5a569d 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,17 @@ +# MediaCrawler Port Configuration +# Container-internal API port remains 8000; host access uses 18080. +DY_API_HOST=0.0.0.0 +DY_API_PORT=8000 +DY_API_PUBLIC_PORT=18080 +DY_API_BASE_URL=http://localhost:18080 + +DY_CHROME_PORT=19222 +DY_CHROME_CDP_URL=http://localhost:19222 + +WEB_DEV_PORT=15173 +VITE_API_BASE_URL=http://localhost:18080 +VITE_WS_BASE_URL=ws://localhost:18080 + # MySQL Configuration MYSQL_DB_PWD=123456 MYSQL_DB_USER=root diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..b65b4b75b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,63 @@ +# MediaCrawler — 抖音关键词批量采集工具 +# 简化构建：直接使用本地构建好的前端产物（api/webui/） +# 本地构建前端：cd web && npm run build + +FROM python:3.11-slim + +# Install system dependencies (Chromium for headless browsing, Node.js for execjs) +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + chromium \ + chromium-driver \ + nodejs \ + npm \ + libnss3 \ + libnspr4 \ + libdbus-1-3 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libcups2 \ + libdrm2 \ + libxkbcommon0 \ + libxcomposite1 \ + libxdamage1 \ + libxfixes3 \ + libxrandr2 \ + libgbm1 \ + libpango-1.0-0 \ + libcairo2 \ + libasound2 \ + fonts-liberation \ + fonts-noto-color-emoji \ + && rm -rf /var/lib/apt/lists/* + +# Set Chromium as default browser for Playwright +ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium +ENV CHROME_PATH=/usr/bin/chromium + +WORKDIR /app + +# Copy dependency files first for cache efficiency +COPY requirements.txt ./requirements.txt +COPY api/requirements.txt ./api-requirements.txt +COPY pyproject.toml ./ + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt \ + && pip install --no-cache-dir -r api-requirements.txt + +# Copy source code +COPY . . + +# Install douyin_scraper package +RUN pip install --no-cache-dir . + +# Expose API port +EXPOSE 8000 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --retries=3 \ + CMD python -c "import httpx; httpx.get('http://localhost:8000/health')" || exit 1 + +# Start API server (also serves Web UI via StaticFiles at /ui/) +CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index e09d2cdd4..eb7aa8d63 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ uv run playwright install 1. **安装最新版 Chrome 浏览器**（版本 >= 144），[下载地址](https://www.google.com/chrome/) 2. **开启远程调试功能**：在 Chrome 地址栏输入 `chrome://inspect/#remote-debugging`，勾选 **"Allow remote debugging for this browser instance"** -3. 页面显示 `Server running at: 127.0.0.1:9222` 表示已就绪 +3. 页面显示 `Server running at: 127.0.0.1:19222` 表示已就绪 > 💡 **提示**：运行爬虫后，Chrome 浏览器会弹出确认对话框，点击"接受"即可。程序会等待用户确认，60秒内操作完成即可。 > @@ -160,14 +160,14 @@ MediaCrawler 提供了基于 Web 的可视化操作界面，无需命令行也 #### 启动 WebUI 服务 ```shell -# 启动 API 服务器（默认端口 8080） -uv run uvicorn api.main:app --port 8080 --reload +# 启动 API 服务器（宿主固定端口 18080；容器内部端口为 8000） +uv run uvicorn api.main:app --port 18080 --reload # 或者使用模块方式启动 uv run python -m api.main ``` -启动成功后，访问 `http://localhost:8080` 即可打开 WebUI 界面。 +启动成功后，访问 `http://localhost:18080/ui/` 即可打开 WebUI 界面。 #### WebUI 功能特性 diff --git a/api/deploy.sh b/api/deploy.sh new file mode 100644 index 000000000..be425c847 --- /dev/null +++ b/api/deploy.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# ═══════════════════════════════════════════════════════════════ +# douyin-scraper 自动化部署脚本 +# ═══════════════════════════════════════════════════════════════ +# 用法： +# chmod +x deploy.sh +# ./deploy.sh # 交互式部署 +# ./deploy.sh --unattended # 非交互式（CI/CD） +# +# 我实际执行时踩过的坑： +# - 虚拟环境不隔离 → 系统包被覆盖 +# - ffmpeg 缺失 → 音视频处理运行时才发现 +# - 权限不对 → systemd 启动失败 +# - 端口冲突 → 服务无法绑定 +# ═══════════════════════════════════════════════════════════════ + +set -euo pipefail + +# 配置（可通过环境变量覆盖） +INSTALL_DIR="${INSTALL_DIR:-/opt/douyin-scraper}" +SERVICE_USER="${SERVICE_USER:-douyin}" +API_PORT="${API_PORT:-18080}" +CHROME_PORT="${CHROME_PORT:-19222}" +WORKSPACE_DIR="${WORKSPACE_DIR:-/opt/douyin-scraper/workspaces}" +UNATTENDED="${1:-}" + +echo "══════════════════════════════════════════════════" +echo " 抖音采集工具 API — 自动化部署" +echo "══════════════════════════════════════════════════" +echo "" + +# ─── 1. 系统依赖检查 ───────────────────────────────── +echo ">>> [1/8] 检查系统依赖..." + +check_cmd() { + if ! command -v "$1" &>/dev/null; then + echo "❌ 缺少依赖: $1" + echo " 安装: $2" + return 1 + fi + echo "✅ $1 已安装" + return 0 +} + +MISSING=0 +check_cmd python3 "apt install python3 python3-venv" || MISSING=1 +check_cmd git "apt install git" || MISSING=1 +check_cmd ffmpeg "apt install ffmpeg" || MISSING=1 + +if [ "$MISSING" -eq 1 ]; then + echo "" + echo "⚠️ 缺少系统依赖，是否自动安装？(y/n)" + if [ "$UNATTENDED" = "--unattended" ]; then + echo "非交互模式：自动安装..." + sudo apt-get update -qq + sudo apt-get install -y -qq python3 python3-venv python3-pip git ffmpeg + else + read -r answer + if [ "$answer" = "y" ]; then + sudo apt-get update -qq + sudo apt-get install -y -qq python3 python3-venv python3-pip git ffmpeg + else + echo "请手动安装后重新运行" + exit 1 + fi + fi +fi + +# ─── 2. 创建用户和目录 ───────────────────────────── +echo ">>> [2/8] 创建用户和目录..." + +if ! id "$SERVICE_USER" &>/dev/null; then + sudo useradd -r -s /bin/bash -d "$INSTALL_DIR" "$SERVICE_USER" + echo "✅ 用户 $SERVICE_USER 已创建" +else + echo "✅ 用户 $SERVICE_USER 已存在" +fi + +sudo mkdir -p "$INSTALL_DIR" +sudo mkdir -p "$WORKSPACE_DIR" +sudo chown -R "$SERVICE_USER:$SERVICE_USER" "$INSTALL_DIR" +sudo chown -R "$SERVICE_USER:$SERVICE_USER" "$WORKSPACE_DIR" + +# ─── 3. 复制代码 ──────────────────────────────────── +echo ">>> [3/8] 复制代码..." + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# 复制 douyin_scraper 模块 +sudo -u "$SERVICE_USER" cp -r "$PROJECT_ROOT/douyin_scraper" "$INSTALL_DIR/" +# 复制 API 代码 +sudo -u "$SERVICE_USER" cp -r "$PROJECT_ROOT/api" "$INSTALL_DIR/" +# 复制 pyproject.toml +sudo -u "$SERVICE_USER" cp "$PROJECT_ROOT/pyproject.toml" "$INSTALL_DIR/" 2>/dev/null || true + +echo "✅ 代码已复制到 $INSTALL_DIR" + +# ─── 4. 创建虚拟环境 ─────────────────────────────── +echo ">>> [4/8] 创建虚拟环境..." + +if [ ! -d "$INSTALL_DIR/.venv" ]; then + sudo -u "$SERVICE_USER" python3 -m venv "$INSTALL_DIR/.venv" + echo "✅ 虚拟环境已创建" +else + echo "✅ 虚拟环境已存在" +fi + +# ─── 5. 安装 Python 依赖 ────────────────────────── +echo ">>> [5/8] 安装 Python 依赖..." + +VENV_PIP="$INSTALL_DIR/.venv/bin/pip" + +sudo -u "$SERVICE_USER" "$VENV_PIP" install --upgrade pip --quiet +sudo -u "$SERVICE_USER" "$VENV_PIP" install -r "$INSTALL_DIR/api/requirements.txt" --quiet +sudo -u "$SERVICE_USER" "$VENV_PIP" install -e "$INSTALL_DIR" --quiet 2>/dev/null || \ + sudo -u "$SERVICE_USER" "$VENV_PIP" install httpx --quiet + +echo "✅ Python 依赖已安装" + +# ─── 6. 创建 .env 文件 ──────────────────────────── +echo ">>> [6/8] 创建配置文件..." + +ENV_FILE="$INSTALL_DIR/.env" +if [ ! -f "$ENV_FILE" ]; then + sudo -u "$SERVICE_USER" tee "$ENV_FILE" > /dev/null << EOF +# 抖音采集工具 API 配置 +DY_API_HOST=0.0.0.0 +DY_API_PORT=$API_PORT +DY_API_PUBLIC_PORT=$API_PORT +DY_API_BASE_URL=http://localhost:$API_PORT +DY_WORKSPACE_DIR=$WORKSPACE_DIR +DY_CHROME_PORT=$CHROME_PORT +DY_CHROME_CDP_URL=http://localhost:$CHROME_PORT +WEB_DEV_PORT=15173 +VITE_API_BASE_URL=http://localhost:$API_PORT +VITE_WS_BASE_URL=ws://localhost:$API_PORT +DY_LOG_LEVEL=INFO +DY_CORS_ORIGINS=* +DY_RELOAD=0 +EOF + echo "✅ .env 文件已创建" +else + echo "✅ .env 文件已存在" +fi + +# ─── 7. 安装 systemd 服务 ───────────────────────── +echo ">>> [7/8] 安装 systemd 服务..." + +# 更新 service 文件中的路径 +sudo cp "$INSTALL_DIR/api/douyin-scraper.service" /etc/systemd/system/ 2>/dev/null || true +sudo sed -i "s|/opt/douyin-scraper|$INSTALL_DIR|g" /etc/systemd/system/douyin-scraper.service +sudo sed -i "s|User=douyin|User=$SERVICE_USER|g" /etc/systemd/system/douyin-scraper.service +sudo sed -i "s|Group=douyin|Group=$SERVICE_USER|g" /etc/systemd/system/douyin-scraper.service +sudo systemctl daemon-reload +sudo systemctl enable douyin-scraper + +echo "✅ systemd 服务已安装并启用" + +# ─── 8. 启动服务 ────────────────────────────────── +echo ">>> [8/8] 启动服务..." + +sudo systemctl restart douyin-scraper + +# 等待服务启动 +sleep 3 + +if sudo systemctl is-active --quiet douyin-scraper; then + echo "" + echo "══════════════════════════════════════════════════" + echo " ✅ 部署成功！" + echo "══════════════════════════════════════════════════" + echo "" + echo " API 地址: http://localhost:$API_PORT" + echo " API 文档: http://localhost:$API_PORT/docs" + echo " 健康检查: http://localhost:$API_PORT/health" + echo " 工作目录: $WORKSPACE_DIR" + echo "" + echo " 常用命令:" + echo " sudo systemctl status douyin-scraper" + echo " sudo systemctl restart douyin-scraper" + echo " sudo journalctl -u douyin-scraper -f" + echo "" +else + echo "" + echo "══════════════════════════════════════════════════" + echo " ❌ 服务启动失败！" + echo "══════════════════════════════════════════════════" + echo "" + echo " 排查：" + echo " sudo journalctl -u douyin-scraper -n 50 --no-pager" + echo " sudo systemctl status douyin-scraper" + exit 1 +fi diff --git a/api/douyin-scraper.service b/api/douyin-scraper.service new file mode 100644 index 000000000..1e8e8905c --- /dev/null +++ b/api/douyin-scraper.service @@ -0,0 +1,40 @@ +[Unit] +Description=Douyin Scraper API Service +After=network.target +Wants=network-online.target + +[Service] +Type=simple +User=douyin +Group=douyin +WorkingDirectory=/opt/douyin-scraper + +# 环境变量 +EnvironmentFile=/opt/douyin-scraper/.env + +# 虚拟环境中的 uvicorn +ExecStart=/opt/douyin-scraper/.venv/bin/uvicorn api.main:app \ + --host 0.0.0.0 \ + --port 18080 \ + --workers 1 \ + --log-level info \ + --access-log + +# 崩溃重启 +Restart=on-failure +RestartSec=5 +StartLimitBurst=5 +StartLimitIntervalSec=60 + +# 资源限制 +LimitNOFILE=65536 +MemoryMax=2G +CPUQuota=200% + +# 日志输出到 journald +StandardOutput=journal +StandardError=journal +SyslogIdentifier=douyin-scraper + +[Install] +WantedBy=multi-user.target diff --git a/api/main.py b/api/main.py index 9a973be39..a54439070 100644 --- a/api/main.py +++ b/api/main.py @@ -4,10 +4,12 @@ v6 新增：Web API 服务入口。 启动方式： - uvicorn api.main:app --host 0.0.0.0 --port 8000 + uvicorn api.main:app --host 0.0.0.0 --port 18080 或通过环境变量配置： - DY_API_HOST=0.0.0.0 DY_API_PORT=8000 uvicorn api.main:app + DY_API_HOST=0.0.0.0 DY_API_PORT=18080 uvicorn api.main:app + +Docker/compose 宿主访问端口固定为 18080，容器内部 API 端口保持 8000。 我实际执行时踩过的坑： - 直接在 HTTP handler 中跑采集 → 请求超时 @@ -89,9 +91,10 @@ async def log_broadcaster(): # ═══════════════════════════════════════════════════════════════ API_HOST = os.environ.get("DY_API_HOST", "0.0.0.0") -API_PORT = int(os.environ.get("DY_API_PORT", "8000")) +API_PUBLIC_PORT = int(os.environ.get("DY_API_PUBLIC_PORT", "18080")) +API_PORT = int(os.environ.get("DY_API_PORT", str(API_PUBLIC_PORT))) WORKSPACE_DIR = os.environ.get("DY_WORKSPACE_DIR", "./workspaces") -CHROME_PORT = int(os.environ.get("DY_CHROME_PORT", "9222")) +CHROME_PORT = int(os.environ.get("DY_CHROME_PORT", "19222")) LOG_LEVEL = os.environ.get("DY_LOG_LEVEL", "INFO") diff --git a/api/requirements.txt b/api/requirements.txt new file mode 100644 index 000000000..8c99f3c39 --- /dev/null +++ b/api/requirements.txt @@ -0,0 +1,8 @@ +fastapi>=0.104 +uvicorn[standard]>=0.24 +python-multipart>=0.0.6 +httpx>=0.24 +pydantic>=2.0 + +# 可选：YAML 配置支持 +# pyyaml>=6.0 diff --git a/config/base_config.py b/config/base_config.py index 381cadc9a..a84caf7b4 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -60,7 +60,7 @@ # CDP 调试端口，用于与浏览器通信 # 如果端口被占用，系统会自动尝试下一个可用端口 -CDP_DEBUG_PORT = 9222 +CDP_DEBUG_PORT = 19222 # 自定义浏览器路径（可选） # 如果为空，系统会自动检测 Chrome/Edge 的安装路径 @@ -78,7 +78,7 @@ # 是否连接用户已打开的浏览器，而不是启动新的浏览器 # 开启后，程序会连接一个已经启用了远程调试的浏览器 # 用户需要在 Chrome 中开启远程调试：chrome://inspect/#remote-debugging -# 或者使用命令行参数启动 Chrome：--remote-debugging-port=9222 +# 或者使用命令行参数启动 Chrome：--remote-debugging-port=19222 # 这种方式反检测效果最好，因为直接使用用户真实浏览器的所有 Cookie、扩展和浏览历史 CDP_CONNECT_EXISTING = True diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..c335d0363 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +services: + api: + build: + context: . + dockerfile: Dockerfile + ports: + - "18080:8000" + env_file: + - .env + volumes: + - ./workspaces:/app/workspaces + - ./data:/app/data + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import httpx; httpx.get('http://localhost:8000/health')"] + interval: 30s + timeout: 10s + retries: 3 + depends_on: + - redis + + redis: + image: redis:7-alpine + volumes: + - redis_data:/data + command: redis-server --appendonly yes + restart: unless-stopped + +volumes: + redis_data: diff --git a/douyin_scraper/core.py b/douyin_scraper/core.py index 18e23f11b..6ff83690b 100644 --- a/douyin_scraper/core.py +++ b/douyin_scraper/core.py @@ -838,7 +838,7 @@ def _do_search(self) -> Path: if self._config.enable_cdp_mode: raise RetryableError( f"Chrome CDP 端口 {self._config.chrome_debugging_port} 未就绪，" - "请先启动 Chrome: chrome.exe --remote-debugging-port=9222", + "请先启动 Chrome: chrome.exe --remote-debugging-port=19222", step=self.STEP_SEARCH, ) else: diff --git a/web/.env.example b/web/.env.example new file mode 100644 index 000000000..428d39c90 --- /dev/null +++ b/web/.env.example @@ -0,0 +1,3 @@ +WEB_DEV_PORT=15173 +VITE_API_BASE_URL=http://localhost:18080 +VITE_WS_BASE_URL=ws://localhost:18080 From 044371241666685ef4fef7b859305c19b75bf416 Mon Sep 17 00:00:00 2001 From: shouheng liu Date: Mon, 22 Jun 2026 19:20:42 +0800 Subject: [PATCH 15/32] chore(webui): refresh embedded frontend build artifacts --- api/webui/assets/index-CZxKLnjr.css | 1 + api/webui/assets/index-DWzGTcmP.js | 336 +++++++++++++++++++++++++ api/webui/assets/index-DvClRayq.js | 353 --------------------------- api/webui/assets/index-OiBmsgXF.css | 1 - api/webui/index.html | 10 +- api/webui/logos/bilibili_logo.png | Bin 43010 -> 0 bytes api/webui/logos/douyin.png | Bin 25846 -> 0 bytes api/webui/logos/github.png | Bin 7967 -> 0 bytes api/webui/logos/my_logo.png | Bin 319037 -> 0 bytes api/webui/logos/xiaohongshu_logo.png | Bin 6360 -> 0 bytes api/webui/vite.svg | 5 +- 11 files changed, 346 insertions(+), 360 deletions(-) create mode 100644 api/webui/assets/index-CZxKLnjr.css create mode 100644 api/webui/assets/index-DWzGTcmP.js delete mode 100644 api/webui/assets/index-DvClRayq.js delete mode 100644 api/webui/assets/index-OiBmsgXF.css delete mode 100644 api/webui/logos/bilibili_logo.png delete mode 100644 api/webui/logos/douyin.png delete mode 100644 api/webui/logos/github.png delete mode 100644 api/webui/logos/my_logo.png delete mode 100644 api/webui/logos/xiaohongshu_logo.png diff --git a/api/webui/assets/index-CZxKLnjr.css b/api/webui/assets/index-CZxKLnjr.css new file mode 100644 index 000000000..e4fe4bef8 --- /dev/null +++ b/api/webui/assets/index-CZxKLnjr.css @@ -0,0 +1 @@ +*,:before,:after{--tw-border-spacing-x: 0;--tw-border-spacing-y: 0;--tw-translate-x: 0;--tw-translate-y: 0;--tw-rotate: 0;--tw-skew-x: 0;--tw-skew-y: 0;--tw-scale-x: 1;--tw-scale-y: 1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness: proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width: 0px;--tw-ring-offset-color: #fff;--tw-ring-color: rgb(59 130 246 / .5);--tw-ring-offset-shadow: 0 0 #0000;--tw-ring-shadow: 0 0 #0000;--tw-shadow: 0 0 #0000;--tw-shadow-colored: 0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: ;--tw-contain-size: ;--tw-contain-layout: ;--tw-contain-paint: ;--tw-contain-style: }::backdrop{--tw-border-spacing-x: 0;--tw-border-spacing-y: 0;--tw-translate-x: 0;--tw-translate-y: 0;--tw-rotate: 0;--tw-skew-x: 0;--tw-skew-y: 0;--tw-scale-x: 1;--tw-scale-y: 1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness: proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width: 0px;--tw-ring-offset-color: #fff;--tw-ring-color: rgb(59 130 246 / .5);--tw-ring-offset-shadow: 0 0 #0000;--tw-ring-shadow: 0 0 #0000;--tw-shadow: 0 0 #0000;--tw-shadow-colored: 0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: ;--tw-contain-size: ;--tw-contain-layout: ;--tw-contain-paint: ;--tw-contain-style: }*,:before,:after{box-sizing:border-box;border-width:0;border-style:solid;border-color:#e5e7eb}:before,:after{--tw-content: ""}html,:host{line-height:1.5;-webkit-text-size-adjust:100%;-moz-tab-size:4;-o-tab-size:4;tab-size:4;font-family:Inter,system-ui,sans-serif;font-feature-settings:normal;font-variation-settings:normal;-webkit-tap-highlight-color:transparent}body{margin:0;line-height:inherit}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-family:JetBrains Mono,monospace;font-feature-settings:normal;font-variation-settings:normal;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}button,input,optgroup,select,textarea{font-family:inherit;font-feature-settings:inherit;font-variation-settings:inherit;font-size:100%;font-weight:inherit;line-height:inherit;letter-spacing:inherit;color:inherit;margin:0;padding:0}button,select{text-transform:none}button,input:where([type=button]),input:where([type=reset]),input:where([type=submit]){-webkit-appearance:button;background-color:transparent;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:baseline}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dl,dd,h1,h2,h3,h4,h5,h6,hr,figure,p,pre{margin:0}fieldset{margin:0;padding:0}legend{padding:0}ol,ul,menu{list-style:none;margin:0;padding:0}dialog{padding:0}textarea{resize:vertical}input::-moz-placeholder,textarea::-moz-placeholder{opacity:1;color:#9ca3af}input::placeholder,textarea::placeholder{opacity:1;color:#9ca3af}button,[role=button]{cursor:pointer}:disabled{cursor:default}img,svg,video,canvas,audio,iframe,embed,object{display:block;vertical-align:middle}img,video{max-width:100%;height:auto}[hidden]:where(:not([hidden=until-found])){display:none}.container{width:100%}@media(min-width:640px){.container{max-width:640px}}@media(min-width:768px){.container{max-width:768px}}@media(min-width:1024px){.container{max-width:1024px}}@media(min-width:1280px){.container{max-width:1280px}}@media(min-width:1536px){.container{max-width:1536px}}.visible{visibility:visible}.invisible{visibility:hidden}.absolute{position:absolute}.relative{position:relative}.sticky{position:sticky}.mb-4{margin-bottom:1rem}.block{display:block}.flex{display:flex}.grid{display:grid}.hidden{display:none}.cursor-pointer{cursor:pointer}.items-center{align-items:center}.justify-center{justify-content:center}.gap-1{gap:.25rem}.break-all{word-break:break-all}.rounded{border-radius:.25rem}.border{border-width:1px}.bg-gray-100{--tw-bg-opacity: 1;background-color:rgb(243 244 246 / var(--tw-bg-opacity, 1))}.bg-gray-50{--tw-bg-opacity: 1;background-color:rgb(249 250 251 / var(--tw-bg-opacity, 1))}.px-1{padding-left:.25rem;padding-right:.25rem}.py-12{padding-top:3rem;padding-bottom:3rem}.font-mono{font-family:JetBrains Mono,monospace}.text-xs{font-size:.75rem;line-height:1rem}.tabular-nums{--tw-numeric-spacing: tabular-nums;font-variant-numeric:var(--tw-ordinal) var(--tw-slashed-zero) var(--tw-numeric-figure) var(--tw-numeric-spacing) var(--tw-numeric-fraction)}.filter{filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow)}.transition{transition-property:color,background-color,border-color,text-decoration-color,fill,stroke,opacity,box-shadow,transform,filter,backdrop-filter;transition-timing-function:cubic-bezier(.4,0,.2,1);transition-duration:.15s}.ease-in-out{transition-timing-function:cubic-bezier(.4,0,.2,1)}body{font-family:Inter,system-ui,sans-serif;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}@keyframes pulse-blue{0%,to{opacity:1}50%{opacity:.5}}.animate-pulse-blue{animation:pulse-blue 2s ease-in-out infinite}.hover\:bg-gray-200:hover{--tw-bg-opacity: 1;background-color:rgb(229 231 235 / var(--tw-bg-opacity, 1))} diff --git a/api/webui/assets/index-DWzGTcmP.js b/api/webui/assets/index-DWzGTcmP.js new file mode 100644 index 000000000..75841ba89 --- /dev/null +++ b/api/webui/assets/index-DWzGTcmP.js @@ -0,0 +1,336 @@ +var hz=Object.defineProperty;var zk=e=>{throw TypeError(e)};var mz=(e,t,r)=>t in e?hz(e,t,{enumerable:!0,configurable:!0,writable:!0,value:r}):e[t]=r;var Lr=(e,t,r)=>mz(e,typeof t!="symbol"?t+"":t,r),Sy=(e,t,r)=>t.has(e)||zk("Cannot "+r);var ye=(e,t,r)=>(Sy(e,t,"read from private field"),r?r.call(e):t.get(e)),$o=(e,t,r)=>t.has(e)?zk("Cannot add the same private member more than once"):t instanceof WeakSet?t.add(e):t.set(e,r),vo=(e,t,r,n)=>(Sy(e,t,"write to private field"),n?n.call(e,r):t.set(e,r),r),kt=(e,t,r)=>(Sy(e,t,"access private method"),r);var Fk=(e,t,r,n)=>({set _(o){vo(e,t,o,r)},get _(){return ye(e,t,n)}});function vz(e,t){for(var r=0;rn[o]})}}}return Object.freeze(Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}))}(function(){const t=document.createElement("link").relList;if(t&&t.supports&&t.supports("modulepreload"))return;for(const o of document.querySelectorAll('link[rel="modulepreload"]'))n(o);new MutationObserver(o=>{for(const i of o)if(i.type==="childList")for(const a of i.addedNodes)a.tagName==="LINK"&&a.rel==="modulepreload"&&n(a)}).observe(document,{childList:!0,subtree:!0});function r(o){const i={};return o.integrity&&(i.integrity=o.integrity),o.referrerPolicy&&(i.referrerPolicy=o.referrerPolicy),o.crossOrigin==="use-credentials"?i.credentials="include":o.crossOrigin==="anonymous"?i.credentials="omit":i.credentials="same-origin",i}function n(o){if(o.ep)return;o.ep=!0;const i=r(o);fetch(o.href,i)}})();var sp=typeof globalThis<"u"?globalThis:typeof window<"u"?window:typeof global<"u"?global:typeof self<"u"?self:{};function bt(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}var Cy={exports:{}},ec={},Py={exports:{}},ht={};/** + * @license React + * react.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */var Wk;function gz(){if(Wk)return ht;Wk=1;var e=Symbol.for("react.element"),t=Symbol.for("react.portal"),r=Symbol.for("react.fragment"),n=Symbol.for("react.strict_mode"),o=Symbol.for("react.profiler"),i=Symbol.for("react.provider"),a=Symbol.for("react.context"),l=Symbol.for("react.forward_ref"),c=Symbol.for("react.suspense"),f=Symbol.for("react.memo"),d=Symbol.for("react.lazy"),h=Symbol.iterator;function m(q){return q===null||typeof q!="object"?null:(q=h&&q[h]||q["@@iterator"],typeof q=="function"?q:null)}var b={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},w=Object.assign,y={};function g(q,K,Z){this.props=q,this.context=K,this.refs=y,this.updater=Z||b}g.prototype.isReactComponent={},g.prototype.setState=function(q,K){if(typeof q!="object"&&typeof q!="function"&&q!=null)throw Error("setState(...): takes an object of state variables to update or a function which returns an object of state variables.");this.updater.enqueueSetState(this,q,K,"setState")},g.prototype.forceUpdate=function(q){this.updater.enqueueForceUpdate(this,q,"forceUpdate")};function S(){}S.prototype=g.prototype;function E(q,K,Z){this.props=q,this.context=K,this.refs=y,this.updater=Z||b}var P=E.prototype=new S;P.constructor=E,w(P,g.prototype),P.isPureReactComponent=!0;var T=Array.isArray,C=Object.prototype.hasOwnProperty,k={current:null},j={key:!0,ref:!0,__self:!0,__source:!0};function $(q,K,Z){var ie,oe={},ae=null,ge=null;if(K!=null)for(ie in K.ref!==void 0&&(ge=K.ref),K.key!==void 0&&(ae=""+K.key),K)C.call(K,ie)&&!j.hasOwnProperty(ie)&&(oe[ie]=K[ie]);var ve=arguments.length-2;if(ve===1)oe.children=Z;else if(1>>1,K=W[q];if(0>>1;qo(oe,V))aeo(ge,oe)?(W[q]=ge,W[ae]=V,q=ae):(W[q]=oe,W[ie]=V,q=ie);else if(aeo(ge,V))W[q]=ge,W[ae]=V,q=ae;else break e}}return U}function o(W,U){var V=W.sortIndex-U.sortIndex;return V!==0?V:W.id-U.id}if(typeof performance=="object"&&typeof performance.now=="function"){var i=performance;e.unstable_now=function(){return i.now()}}else{var a=Date,l=a.now();e.unstable_now=function(){return a.now()-l}}var c=[],f=[],d=1,h=null,m=3,b=!1,w=!1,y=!1,g=typeof setTimeout=="function"?setTimeout:null,S=typeof clearTimeout=="function"?clearTimeout:null,E=typeof setImmediate<"u"?setImmediate:null;typeof navigator<"u"&&navigator.scheduling!==void 0&&navigator.scheduling.isInputPending!==void 0&&navigator.scheduling.isInputPending.bind(navigator.scheduling);function P(W){for(var U=r(f);U!==null;){if(U.callback===null)n(f);else if(U.startTime<=W)n(f),U.sortIndex=U.expirationTime,t(c,U);else break;U=r(f)}}function T(W){if(y=!1,P(W),!w)if(r(c)!==null)w=!0,L(C);else{var U=r(f);U!==null&&H(T,U.startTime-W)}}function C(W,U){w=!1,y&&(y=!1,S($),$=-1),b=!0;var V=m;try{for(P(U),h=r(c);h!==null&&(!(h.expirationTime>U)||W&&!I());){var q=h.callback;if(typeof q=="function"){h.callback=null,m=h.priorityLevel;var K=q(h.expirationTime<=U);U=e.unstable_now(),typeof K=="function"?h.callback=K:h===r(c)&&n(c),P(U)}else n(c);h=r(c)}if(h!==null)var Z=!0;else{var ie=r(f);ie!==null&&H(T,ie.startTime-U),Z=!1}return Z}finally{h=null,m=V,b=!1}}var k=!1,j=null,$=-1,M=5,A=-1;function I(){return!(e.unstable_now()-AW||125q?(W.sortIndex=V,t(f,W),r(c)===null&&W===r(f)&&(y?(S($),$=-1):y=!0,H(T,V-q))):(W.sortIndex=K,t(c,W),w||b||(w=!0,L(C))),W},e.unstable_shouldYield=I,e.unstable_wrapCallback=function(W){var U=m;return function(){var V=m;m=U;try{return W.apply(this,arguments)}finally{m=V}}}})(ky)),ky}var Kk;function wz(){return Kk||(Kk=1,Ty.exports=xz()),Ty.exports}/** + * @license React + * react-dom.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */var Gk;function Sz(){if(Gk)return fn;Gk=1;var e=ZS(),t=wz();function r(s){for(var u="https://reactjs.org/docs/error-decoder.html?invariant="+s,p=1;p"u"||typeof window.document>"u"||typeof window.document.createElement>"u"),c=Object.prototype.hasOwnProperty,f=/^[:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD][:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\-.0-9\u00B7\u0300-\u036F\u203F-\u2040]*$/,d={},h={};function m(s){return c.call(h,s)?!0:c.call(d,s)?!1:f.test(s)?h[s]=!0:(d[s]=!0,!1)}function b(s,u,p,v){if(p!==null&&p.type===0)return!1;switch(typeof u){case"function":case"symbol":return!0;case"boolean":return v?!1:p!==null?!p.acceptsBooleans:(s=s.toLowerCase().slice(0,5),s!=="data-"&&s!=="aria-");default:return!1}}function w(s,u,p,v){if(u===null||typeof u>"u"||b(s,u,p,v))return!0;if(v)return!1;if(p!==null)switch(p.type){case 3:return!u;case 4:return u===!1;case 5:return isNaN(u);case 6:return isNaN(u)||1>u}return!1}function y(s,u,p,v,O,R,D){this.acceptsBooleans=u===2||u===3||u===4,this.attributeName=v,this.attributeNamespace=O,this.mustUseProperty=p,this.propertyName=s,this.type=u,this.sanitizeURL=R,this.removeEmptyString=D}var g={};"children dangerouslySetInnerHTML defaultValue defaultChecked innerHTML suppressContentEditableWarning suppressHydrationWarning style".split(" ").forEach(function(s){g[s]=new y(s,0,!1,s,null,!1,!1)}),[["acceptCharset","accept-charset"],["className","class"],["htmlFor","for"],["httpEquiv","http-equiv"]].forEach(function(s){var u=s[0];g[u]=new y(u,1,!1,s[1],null,!1,!1)}),["contentEditable","draggable","spellCheck","value"].forEach(function(s){g[s]=new y(s,2,!1,s.toLowerCase(),null,!1,!1)}),["autoReverse","externalResourcesRequired","focusable","preserveAlpha"].forEach(function(s){g[s]=new y(s,2,!1,s,null,!1,!1)}),"allowFullScreen async autoFocus autoPlay controls default defer disabled disablePictureInPicture disableRemotePlayback formNoValidate hidden loop noModule noValidate open playsInline readOnly required reversed scoped seamless itemScope".split(" ").forEach(function(s){g[s]=new y(s,3,!1,s.toLowerCase(),null,!1,!1)}),["checked","multiple","muted","selected"].forEach(function(s){g[s]=new y(s,3,!0,s,null,!1,!1)}),["capture","download"].forEach(function(s){g[s]=new y(s,4,!1,s,null,!1,!1)}),["cols","rows","size","span"].forEach(function(s){g[s]=new y(s,6,!1,s,null,!1,!1)}),["rowSpan","start"].forEach(function(s){g[s]=new y(s,5,!1,s.toLowerCase(),null,!1,!1)});var S=/[\-:]([a-z])/g;function E(s){return s[1].toUpperCase()}"accent-height alignment-baseline arabic-form baseline-shift cap-height clip-path clip-rule color-interpolation color-interpolation-filters color-profile color-rendering dominant-baseline enable-background fill-opacity fill-rule flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x image-rendering letter-spacing lighting-color marker-end marker-mid marker-start overline-position overline-thickness paint-order panose-1 pointer-events rendering-intent shape-rendering stop-color stop-opacity strikethrough-position strikethrough-thickness stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width text-anchor text-decoration text-rendering underline-position underline-thickness unicode-bidi unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical vector-effect vert-adv-y vert-origin-x vert-origin-y word-spacing writing-mode xmlns:xlink x-height".split(" ").forEach(function(s){var u=s.replace(S,E);g[u]=new y(u,1,!1,s,null,!1,!1)}),"xlink:actuate xlink:arcrole xlink:role xlink:show xlink:title xlink:type".split(" ").forEach(function(s){var u=s.replace(S,E);g[u]=new y(u,1,!1,s,"http://www.w3.org/1999/xlink",!1,!1)}),["xml:base","xml:lang","xml:space"].forEach(function(s){var u=s.replace(S,E);g[u]=new y(u,1,!1,s,"http://www.w3.org/XML/1998/namespace",!1,!1)}),["tabIndex","crossOrigin"].forEach(function(s){g[s]=new y(s,1,!1,s.toLowerCase(),null,!1,!1)}),g.xlinkHref=new y("xlinkHref",1,!1,"xlink:href","http://www.w3.org/1999/xlink",!0,!1),["src","href","action","formAction"].forEach(function(s){g[s]=new y(s,1,!1,s.toLowerCase(),null,!0,!0)});function P(s,u,p,v){var O=g.hasOwnProperty(u)?g[u]:null;(O!==null?O.type!==0:v||!(2G||O[D]!==R[G]){var X=` +`+O[D].replace(" at new "," at ");return s.displayName&&X.includes("")&&(X=X.replace("",s.displayName)),X}while(1<=D&&0<=G);break}}}finally{Z=!1,Error.prepareStackTrace=p}return(s=s?s.displayName||s.name:"")?K(s):""}function oe(s){switch(s.tag){case 5:return K(s.type);case 16:return K("Lazy");case 13:return K("Suspense");case 19:return K("SuspenseList");case 0:case 2:case 15:return s=ie(s.type,!1),s;case 11:return s=ie(s.type.render,!1),s;case 1:return s=ie(s.type,!0),s;default:return""}}function ae(s){if(s==null)return null;if(typeof s=="function")return s.displayName||s.name||null;if(typeof s=="string")return s;switch(s){case j:return"Fragment";case k:return"Portal";case M:return"Profiler";case $:return"StrictMode";case F:return"Suspense";case z:return"SuspenseList"}if(typeof s=="object")switch(s.$$typeof){case I:return(s.displayName||"Context")+".Consumer";case A:return(s._context.displayName||"Context")+".Provider";case N:var u=s.render;return s=s.displayName,s||(s=u.displayName||u.name||"",s=s!==""?"ForwardRef("+s+")":"ForwardRef"),s;case B:return u=s.displayName||null,u!==null?u:ae(s.type)||"Memo";case L:u=s._payload,s=s._init;try{return ae(s(u))}catch{}}return null}function ge(s){var u=s.type;switch(s.tag){case 24:return"Cache";case 9:return(u.displayName||"Context")+".Consumer";case 10:return(u._context.displayName||"Context")+".Provider";case 18:return"DehydratedFragment";case 11:return s=u.render,s=s.displayName||s.name||"",u.displayName||(s!==""?"ForwardRef("+s+")":"ForwardRef");case 7:return"Fragment";case 5:return u;case 4:return"Portal";case 3:return"Root";case 6:return"Text";case 16:return ae(u);case 8:return u===$?"StrictMode":"Mode";case 22:return"Offscreen";case 12:return"Profiler";case 21:return"Scope";case 13:return"Suspense";case 19:return"SuspenseList";case 25:return"TracingMarker";case 1:case 0:case 17:case 2:case 14:case 15:if(typeof u=="function")return u.displayName||u.name||null;if(typeof u=="string")return u}return null}function ve(s){switch(typeof s){case"boolean":case"number":case"string":case"undefined":return s;case"object":return s;default:return""}}function te(s){var u=s.type;return(s=s.nodeName)&&s.toLowerCase()==="input"&&(u==="checkbox"||u==="radio")}function se(s){var u=te(s)?"checked":"value",p=Object.getOwnPropertyDescriptor(s.constructor.prototype,u),v=""+s[u];if(!s.hasOwnProperty(u)&&typeof p<"u"&&typeof p.get=="function"&&typeof p.set=="function"){var O=p.get,R=p.set;return Object.defineProperty(s,u,{configurable:!0,get:function(){return O.call(this)},set:function(D){v=""+D,R.call(this,D)}}),Object.defineProperty(s,u,{enumerable:p.enumerable}),{getValue:function(){return v},setValue:function(D){v=""+D},stopTracking:function(){s._valueTracker=null,delete s[u]}}}}function pe(s){s._valueTracker||(s._valueTracker=se(s))}function J(s){if(!s)return!1;var u=s._valueTracker;if(!u)return!0;var p=u.getValue(),v="";return s&&(v=te(s)?s.checked?"true":"false":s.value),s=v,s!==p?(u.setValue(s),!0):!1}function Ce(s){if(s=s||(typeof document<"u"?document:void 0),typeof s>"u")return null;try{return s.activeElement||s.body}catch{return s.body}}function me(s,u){var p=u.checked;return V({},u,{defaultChecked:void 0,defaultValue:void 0,value:void 0,checked:p??s._wrapperState.initialChecked})}function je(s,u){var p=u.defaultValue==null?"":u.defaultValue,v=u.checked!=null?u.checked:u.defaultChecked;p=ve(u.value!=null?u.value:p),s._wrapperState={initialChecked:v,initialValue:p,controlled:u.type==="checkbox"||u.type==="radio"?u.checked!=null:u.value!=null}}function et(s,u){u=u.checked,u!=null&&P(s,"checked",u,!1)}function Be(s,u){et(s,u);var p=ve(u.value),v=u.type;if(p!=null)v==="number"?(p===0&&s.value===""||s.value!=p)&&(s.value=""+p):s.value!==""+p&&(s.value=""+p);else if(v==="submit"||v==="reset"){s.removeAttribute("value");return}u.hasOwnProperty("value")?pt(s,u.type,p):u.hasOwnProperty("defaultValue")&&pt(s,u.type,ve(u.defaultValue)),u.checked==null&&u.defaultChecked!=null&&(s.defaultChecked=!!u.defaultChecked)}function Ye(s,u,p){if(u.hasOwnProperty("value")||u.hasOwnProperty("defaultValue")){var v=u.type;if(!(v!=="submit"&&v!=="reset"||u.value!==void 0&&u.value!==null))return;u=""+s._wrapperState.initialValue,p||u===s.value||(s.value=u),s.defaultValue=u}p=s.name,p!==""&&(s.name=""),s.defaultChecked=!!s._wrapperState.initialChecked,p!==""&&(s.name=p)}function pt(s,u,p){(u!=="number"||Ce(s.ownerDocument)!==s)&&(p==null?s.defaultValue=""+s._wrapperState.initialValue:s.defaultValue!==""+p&&(s.defaultValue=""+p))}var nt=Array.isArray;function rt(s,u,p,v){if(s=s.options,u){u={};for(var O=0;O"+u.valueOf().toString()+"",u=Me.firstChild;s.firstChild;)s.removeChild(s.firstChild);for(;u.firstChild;)s.appendChild(u.firstChild)}});function ke(s,u){if(u){var p=s.firstChild;if(p&&p===s.lastChild&&p.nodeType===3){p.nodeValue=u;return}}s.textContent=u}var Ie={animationIterationCount:!0,aspectRatio:!0,borderImageOutset:!0,borderImageSlice:!0,borderImageWidth:!0,boxFlex:!0,boxFlexGroup:!0,boxOrdinalGroup:!0,columnCount:!0,columns:!0,flex:!0,flexGrow:!0,flexPositive:!0,flexShrink:!0,flexNegative:!0,flexOrder:!0,gridArea:!0,gridRow:!0,gridRowEnd:!0,gridRowSpan:!0,gridRowStart:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnSpan:!0,gridColumnStart:!0,fontWeight:!0,lineClamp:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,tabSize:!0,widows:!0,zIndex:!0,zoom:!0,fillOpacity:!0,floodOpacity:!0,stopOpacity:!0,strokeDasharray:!0,strokeDashoffset:!0,strokeMiterlimit:!0,strokeOpacity:!0,strokeWidth:!0},ct=["Webkit","ms","Moz","O"];Object.keys(Ie).forEach(function(s){ct.forEach(function(u){u=u+s.charAt(0).toUpperCase()+s.substring(1),Ie[u]=Ie[s]})});function st(s,u,p){return u==null||typeof u=="boolean"||u===""?"":p||typeof u!="number"||u===0||Ie.hasOwnProperty(s)&&Ie[s]?(""+u).trim():u+"px"}function Tt(s,u){s=s.style;for(var p in u)if(u.hasOwnProperty(p)){var v=p.indexOf("--")===0,O=st(p,u[p],v);p==="float"&&(p="cssFloat"),v?s.setProperty(p,O):s[p]=O}}var Ge=V({menuitem:!0},{area:!0,base:!0,br:!0,col:!0,embed:!0,hr:!0,img:!0,input:!0,keygen:!0,link:!0,meta:!0,param:!0,source:!0,track:!0,wbr:!0});function yt(s,u){if(u){if(Ge[s]&&(u.children!=null||u.dangerouslySetInnerHTML!=null))throw Error(r(137,s));if(u.dangerouslySetInnerHTML!=null){if(u.children!=null)throw Error(r(60));if(typeof u.dangerouslySetInnerHTML!="object"||!("__html"in u.dangerouslySetInnerHTML))throw Error(r(61))}if(u.style!=null&&typeof u.style!="object")throw Error(r(62))}}function Nt(s,u){if(s.indexOf("-")===-1)return typeof u.is=="string";switch(s){case"annotation-xml":case"color-profile":case"font-face":case"font-face-src":case"font-face-uri":case"font-face-format":case"font-face-name":case"missing-glyph":return!1;default:return!0}}var $r=null;function In(s){return s=s.target||s.srcElement||window,s.correspondingUseElement&&(s=s.correspondingUseElement),s.nodeType===3?s.parentNode:s}var ri=null,ko=null,jt=null;function or(s){if(s=Bu(s)){if(typeof ri!="function")throw Error(r(280));var u=s.stateNode;u&&(u=Sd(u),ri(s.stateNode,s.type,u))}}function Bi(s){ko?jt?jt.push(s):jt=[s]:ko=s}function ni(){if(ko){var s=ko,u=jt;if(jt=ko=null,or(s),u)for(s=0;s>>=0,s===0?32:31-(A4(s)/j4|0)|0}var od=64,id=4194304;function wu(s){switch(s&-s){case 1:return 1;case 2:return 2;case 4:return 4;case 8:return 8;case 16:return 16;case 32:return 32;case 64:case 128:case 256:case 512:case 1024:case 2048:case 4096:case 8192:case 16384:case 32768:case 65536:case 131072:case 262144:case 524288:case 1048576:case 2097152:return s&4194240;case 4194304:case 8388608:case 16777216:case 33554432:case 67108864:return s&130023424;case 134217728:return 134217728;case 268435456:return 268435456;case 536870912:return 536870912;case 1073741824:return 1073741824;default:return s}}function ad(s,u){var p=s.pendingLanes;if(p===0)return 0;var v=0,O=s.suspendedLanes,R=s.pingedLanes,D=p&268435455;if(D!==0){var G=D&~O;G!==0?v=wu(G):(R&=D,R!==0&&(v=wu(R)))}else D=p&~O,D!==0?v=wu(D):R!==0&&(v=wu(R));if(v===0)return 0;if(u!==0&&u!==v&&(u&O)===0&&(O=v&-v,R=u&-u,O>=R||O===16&&(R&4194240)!==0))return u;if((v&4)!==0&&(v|=p&16),u=s.entangledLanes,u!==0)for(s=s.entanglements,u&=v;0p;p++)u.push(s);return u}function Su(s,u,p){s.pendingLanes|=u,u!==536870912&&(s.suspendedLanes=0,s.pingedLanes=0),s=s.eventTimes,u=31-lo(u),s[u]=p}function I4(s,u){var p=s.pendingLanes&~u;s.pendingLanes=u,s.suspendedLanes=0,s.pingedLanes=0,s.expiredLanes&=u,s.mutableReadLanes&=u,s.entangledLanes&=u,u=s.entanglements;var v=s.eventTimes;for(s=s.expirationTimes;0=Au),$O=" ",MO=!1;function IO(s,u){switch(s){case"keyup":return u3.indexOf(u.keyCode)!==-1;case"keydown":return u.keyCode!==229;case"keypress":case"mousedown":case"focusout":return!0;default:return!1}}function LO(s){return s=s.detail,typeof s=="object"&&"data"in s?s.data:null}var $s=!1;function f3(s,u){switch(s){case"compositionend":return LO(u);case"keypress":return u.which!==32?null:(MO=!0,$O);case"textInput":return s=u.data,s===$O&&MO?null:s;default:return null}}function d3(s,u){if($s)return s==="compositionend"||!rg&&IO(s,u)?(s=kO(),fd=Xv=Ui=null,$s=!1,s):null;switch(s){case"paste":return null;case"keypress":if(!(u.ctrlKey||u.altKey||u.metaKey)||u.ctrlKey&&u.altKey){if(u.char&&1=u)return{node:p,offset:u-s};s=v}e:{for(;p;){if(p.nextSibling){p=p.nextSibling;break e}p=p.parentNode}p=void 0}p=qO(p)}}function HO(s,u){return s&&u?s===u?!0:s&&s.nodeType===3?!1:u&&u.nodeType===3?HO(s,u.parentNode):"contains"in s?s.contains(u):s.compareDocumentPosition?!!(s.compareDocumentPosition(u)&16):!1:!1}function VO(){for(var s=window,u=Ce();u instanceof s.HTMLIFrameElement;){try{var p=typeof u.contentWindow.location.href=="string"}catch{p=!1}if(p)s=u.contentWindow;else break;u=Ce(s.document)}return u}function ig(s){var u=s&&s.nodeName&&s.nodeName.toLowerCase();return u&&(u==="input"&&(s.type==="text"||s.type==="search"||s.type==="tel"||s.type==="url"||s.type==="password")||u==="textarea"||s.contentEditable==="true")}function w3(s){var u=VO(),p=s.focusedElem,v=s.selectionRange;if(u!==p&&p&&p.ownerDocument&&HO(p.ownerDocument.documentElement,p)){if(v!==null&&ig(p)){if(u=v.start,s=v.end,s===void 0&&(s=u),"selectionStart"in p)p.selectionStart=u,p.selectionEnd=Math.min(s,p.value.length);else if(s=(u=p.ownerDocument||document)&&u.defaultView||window,s.getSelection){s=s.getSelection();var O=p.textContent.length,R=Math.min(v.start,O);v=v.end===void 0?R:Math.min(v.end,O),!s.extend&&R>v&&(O=v,v=R,R=O),O=UO(p,R);var D=UO(p,v);O&&D&&(s.rangeCount!==1||s.anchorNode!==O.node||s.anchorOffset!==O.offset||s.focusNode!==D.node||s.focusOffset!==D.offset)&&(u=u.createRange(),u.setStart(O.node,O.offset),s.removeAllRanges(),R>v?(s.addRange(u),s.extend(D.node,D.offset)):(u.setEnd(D.node,D.offset),s.addRange(u)))}}for(u=[],s=p;s=s.parentNode;)s.nodeType===1&&u.push({element:s,left:s.scrollLeft,top:s.scrollTop});for(typeof p.focus=="function"&&p.focus(),p=0;p=document.documentMode,Ms=null,ag=null,Mu=null,sg=!1;function KO(s,u,p){var v=p.window===p?p.document:p.nodeType===9?p:p.ownerDocument;sg||Ms==null||Ms!==Ce(v)||(v=Ms,"selectionStart"in v&&ig(v)?v={start:v.selectionStart,end:v.selectionEnd}:(v=(v.ownerDocument&&v.ownerDocument.defaultView||window).getSelection(),v={anchorNode:v.anchorNode,anchorOffset:v.anchorOffset,focusNode:v.focusNode,focusOffset:v.focusOffset}),Mu&&$u(Mu,v)||(Mu=v,v=bd(ag,"onSelect"),0Bs||(s.current=bg[Bs],bg[Bs]=null,Bs--)}function Ft(s,u){Bs++,bg[Bs]=s.current,s.current=u}var Gi={},zr=Ki(Gi),an=Ki(!1),Aa=Gi;function zs(s,u){var p=s.type.contextTypes;if(!p)return Gi;var v=s.stateNode;if(v&&v.__reactInternalMemoizedUnmaskedChildContext===u)return v.__reactInternalMemoizedMaskedChildContext;var O={},R;for(R in p)O[R]=u[R];return v&&(s=s.stateNode,s.__reactInternalMemoizedUnmaskedChildContext=u,s.__reactInternalMemoizedMaskedChildContext=O),O}function sn(s){return s=s.childContextTypes,s!=null}function Cd(){Vt(an),Vt(zr)}function lT(s,u,p){if(zr.current!==Gi)throw Error(r(168));Ft(zr,u),Ft(an,p)}function uT(s,u,p){var v=s.stateNode;if(u=u.childContextTypes,typeof v.getChildContext!="function")return p;v=v.getChildContext();for(var O in v)if(!(O in u))throw Error(r(108,ge(s)||"Unknown",O));return V({},p,v)}function Pd(s){return s=(s=s.stateNode)&&s.__reactInternalMemoizedMergedChildContext||Gi,Aa=zr.current,Ft(zr,s),Ft(an,an.current),!0}function cT(s,u,p){var v=s.stateNode;if(!v)throw Error(r(169));p?(s=uT(s,u,Aa),v.__reactInternalMemoizedMergedChildContext=s,Vt(an),Vt(zr),Ft(zr,s)):Vt(an),Ft(an,p)}var ii=null,Od=!1,xg=!1;function fT(s){ii===null?ii=[s]:ii.push(s)}function $3(s){Od=!0,fT(s)}function Yi(){if(!xg&&ii!==null){xg=!0;var s=0,u=Rt;try{var p=ii;for(Rt=1;s>=D,O-=D,ai=1<<32-lo(u)+O|p<it?(Er=Je,Je=null):Er=Je.sibling;var wt=be(ee,Je,re[it],Pe);if(wt===null){Je===null&&(Je=Er);break}s&&Je&&wt.alternate===null&&u(ee,Je),Q=R(wt,Q,it),Qe===null?Ve=wt:Qe.sibling=wt,Qe=wt,Je=Er}if(it===re.length)return p(ee,Je),Xt&&Ra(ee,it),Ve;if(Je===null){for(;itit?(Er=Je,Je=null):Er=Je.sibling;var oa=be(ee,Je,wt.value,Pe);if(oa===null){Je===null&&(Je=Er);break}s&&Je&&oa.alternate===null&&u(ee,Je),Q=R(oa,Q,it),Qe===null?Ve=oa:Qe.sibling=oa,Qe=oa,Je=Er}if(wt.done)return p(ee,Je),Xt&&Ra(ee,it),Ve;if(Je===null){for(;!wt.done;it++,wt=re.next())wt=Se(ee,wt.value,Pe),wt!==null&&(Q=R(wt,Q,it),Qe===null?Ve=wt:Qe.sibling=wt,Qe=wt);return Xt&&Ra(ee,it),Ve}for(Je=v(ee,Je);!wt.done;it++,wt=re.next())wt=Ae(Je,ee,it,wt.value,Pe),wt!==null&&(s&&wt.alternate!==null&&Je.delete(wt.key===null?it:wt.key),Q=R(wt,Q,it),Qe===null?Ve=wt:Qe.sibling=wt,Qe=wt);return s&&Je.forEach(function(pz){return u(ee,pz)}),Xt&&Ra(ee,it),Ve}function dr(ee,Q,re,Pe){if(typeof re=="object"&&re!==null&&re.type===j&&re.key===null&&(re=re.props.children),typeof re=="object"&&re!==null){switch(re.$$typeof){case C:e:{for(var Ve=re.key,Qe=Q;Qe!==null;){if(Qe.key===Ve){if(Ve=re.type,Ve===j){if(Qe.tag===7){p(ee,Qe.sibling),Q=O(Qe,re.props.children),Q.return=ee,ee=Q;break e}}else if(Qe.elementType===Ve||typeof Ve=="object"&&Ve!==null&&Ve.$$typeof===L&&gT(Ve)===Qe.type){p(ee,Qe.sibling),Q=O(Qe,re.props),Q.ref=zu(ee,Qe,re),Q.return=ee,ee=Q;break e}p(ee,Qe);break}else u(ee,Qe);Qe=Qe.sibling}re.type===j?(Q=za(re.props.children,ee.mode,Pe,re.key),Q.return=ee,ee=Q):(Pe=Zd(re.type,re.key,re.props,null,ee.mode,Pe),Pe.ref=zu(ee,Q,re),Pe.return=ee,ee=Pe)}return D(ee);case k:e:{for(Qe=re.key;Q!==null;){if(Q.key===Qe)if(Q.tag===4&&Q.stateNode.containerInfo===re.containerInfo&&Q.stateNode.implementation===re.implementation){p(ee,Q.sibling),Q=O(Q,re.children||[]),Q.return=ee,ee=Q;break e}else{p(ee,Q);break}else u(ee,Q);Q=Q.sibling}Q=gy(re,ee.mode,Pe),Q.return=ee,ee=Q}return D(ee);case L:return Qe=re._init,dr(ee,Q,Qe(re._payload),Pe)}if(nt(re))return De(ee,Q,re,Pe);if(U(re))return ze(ee,Q,re,Pe);_d(ee,re)}return typeof re=="string"&&re!==""||typeof re=="number"?(re=""+re,Q!==null&&Q.tag===6?(p(ee,Q.sibling),Q=O(Q,re),Q.return=ee,ee=Q):(p(ee,Q),Q=vy(re,ee.mode,Pe),Q.return=ee,ee=Q),D(ee)):p(ee,Q)}return dr}var Us=yT(!0),bT=yT(!1),Ad=Ki(null),jd=null,Hs=null,Tg=null;function kg(){Tg=Hs=jd=null}function Eg(s){var u=Ad.current;Vt(Ad),s._currentValue=u}function _g(s,u,p){for(;s!==null;){var v=s.alternate;if((s.childLanes&u)!==u?(s.childLanes|=u,v!==null&&(v.childLanes|=u)):v!==null&&(v.childLanes&u)!==u&&(v.childLanes|=u),s===p)break;s=s.return}}function Vs(s,u){jd=s,Tg=Hs=null,s=s.dependencies,s!==null&&s.firstContext!==null&&((s.lanes&u)!==0&&(ln=!0),s.firstContext=null)}function Dn(s){var u=s._currentValue;if(Tg!==s)if(s={context:s,memoizedValue:u,next:null},Hs===null){if(jd===null)throw Error(r(308));Hs=s,jd.dependencies={lanes:0,firstContext:s}}else Hs=Hs.next=s;return u}var $a=null;function Ag(s){$a===null?$a=[s]:$a.push(s)}function xT(s,u,p,v){var O=u.interleaved;return O===null?(p.next=p,Ag(u)):(p.next=O.next,O.next=p),u.interleaved=p,li(s,v)}function li(s,u){s.lanes|=u;var p=s.alternate;for(p!==null&&(p.lanes|=u),p=s,s=s.return;s!==null;)s.childLanes|=u,p=s.alternate,p!==null&&(p.childLanes|=u),p=s,s=s.return;return p.tag===3?p.stateNode:null}var Xi=!1;function jg(s){s.updateQueue={baseState:s.memoizedState,firstBaseUpdate:null,lastBaseUpdate:null,shared:{pending:null,interleaved:null,lanes:0},effects:null}}function wT(s,u){s=s.updateQueue,u.updateQueue===s&&(u.updateQueue={baseState:s.baseState,firstBaseUpdate:s.firstBaseUpdate,lastBaseUpdate:s.lastBaseUpdate,shared:s.shared,effects:s.effects})}function ui(s,u){return{eventTime:s,lane:u,tag:0,payload:null,callback:null,next:null}}function Qi(s,u,p){var v=s.updateQueue;if(v===null)return null;if(v=v.shared,(xt&2)!==0){var O=v.pending;return O===null?u.next=u:(u.next=O.next,O.next=u),v.pending=u,li(s,p)}return O=v.interleaved,O===null?(u.next=u,Ag(v)):(u.next=O.next,O.next=u),v.interleaved=u,li(s,p)}function Rd(s,u,p){if(u=u.updateQueue,u!==null&&(u=u.shared,(p&4194240)!==0)){var v=u.lanes;v&=s.pendingLanes,p|=v,u.lanes=p,Hv(s,p)}}function ST(s,u){var p=s.updateQueue,v=s.alternate;if(v!==null&&(v=v.updateQueue,p===v)){var O=null,R=null;if(p=p.firstBaseUpdate,p!==null){do{var D={eventTime:p.eventTime,lane:p.lane,tag:p.tag,payload:p.payload,callback:p.callback,next:null};R===null?O=R=D:R=R.next=D,p=p.next}while(p!==null);R===null?O=R=u:R=R.next=u}else O=R=u;p={baseState:v.baseState,firstBaseUpdate:O,lastBaseUpdate:R,shared:v.shared,effects:v.effects},s.updateQueue=p;return}s=p.lastBaseUpdate,s===null?p.firstBaseUpdate=u:s.next=u,p.lastBaseUpdate=u}function $d(s,u,p,v){var O=s.updateQueue;Xi=!1;var R=O.firstBaseUpdate,D=O.lastBaseUpdate,G=O.shared.pending;if(G!==null){O.shared.pending=null;var X=G,ne=X.next;X.next=null,D===null?R=ne:D.next=ne,D=X;var xe=s.alternate;xe!==null&&(xe=xe.updateQueue,G=xe.lastBaseUpdate,G!==D&&(G===null?xe.firstBaseUpdate=ne:G.next=ne,xe.lastBaseUpdate=X))}if(R!==null){var Se=O.baseState;D=0,xe=ne=X=null,G=R;do{var be=G.lane,Ae=G.eventTime;if((v&be)===be){xe!==null&&(xe=xe.next={eventTime:Ae,lane:0,tag:G.tag,payload:G.payload,callback:G.callback,next:null});e:{var De=s,ze=G;switch(be=u,Ae=p,ze.tag){case 1:if(De=ze.payload,typeof De=="function"){Se=De.call(Ae,Se,be);break e}Se=De;break e;case 3:De.flags=De.flags&-65537|128;case 0:if(De=ze.payload,be=typeof De=="function"?De.call(Ae,Se,be):De,be==null)break e;Se=V({},Se,be);break e;case 2:Xi=!0}}G.callback!==null&&G.lane!==0&&(s.flags|=64,be=O.effects,be===null?O.effects=[G]:be.push(G))}else Ae={eventTime:Ae,lane:be,tag:G.tag,payload:G.payload,callback:G.callback,next:null},xe===null?(ne=xe=Ae,X=Se):xe=xe.next=Ae,D|=be;if(G=G.next,G===null){if(G=O.shared.pending,G===null)break;be=G,G=be.next,be.next=null,O.lastBaseUpdate=be,O.shared.pending=null}}while(!0);if(xe===null&&(X=Se),O.baseState=X,O.firstBaseUpdate=ne,O.lastBaseUpdate=xe,u=O.shared.interleaved,u!==null){O=u;do D|=O.lane,O=O.next;while(O!==u)}else R===null&&(O.shared.lanes=0);La|=D,s.lanes=D,s.memoizedState=Se}}function CT(s,u,p){if(s=u.effects,u.effects=null,s!==null)for(u=0;up?p:4,s(!0);var v=Lg.transition;Lg.transition={};try{s(!1),u()}finally{Rt=p,Lg.transition=v}}function WT(){return Bn().memoizedState}function N3(s,u,p){var v=ta(s);if(p={lane:v,action:p,hasEagerState:!1,eagerState:null,next:null},qT(s))UT(u,p);else if(p=xT(s,u,p,v),p!==null){var O=Zr();mo(p,s,v,O),HT(p,u,v)}}function D3(s,u,p){var v=ta(s),O={lane:v,action:p,hasEagerState:!1,eagerState:null,next:null};if(qT(s))UT(u,O);else{var R=s.alternate;if(s.lanes===0&&(R===null||R.lanes===0)&&(R=u.lastRenderedReducer,R!==null))try{var D=u.lastRenderedState,G=R(D,p);if(O.hasEagerState=!0,O.eagerState=G,uo(G,D)){var X=u.interleaved;X===null?(O.next=O,Ag(u)):(O.next=X.next,X.next=O),u.interleaved=O;return}}catch{}finally{}p=xT(s,u,O,v),p!==null&&(O=Zr(),mo(p,s,v,O),HT(p,u,v))}}function qT(s){var u=s.alternate;return s===nr||u!==null&&u===nr}function UT(s,u){Uu=Ld=!0;var p=s.pending;p===null?u.next=u:(u.next=p.next,p.next=u),s.pending=u}function HT(s,u,p){if((p&4194240)!==0){var v=u.lanes;v&=s.pendingLanes,p|=v,u.lanes=p,Hv(s,p)}}var Bd={readContext:Dn,useCallback:Fr,useContext:Fr,useEffect:Fr,useImperativeHandle:Fr,useInsertionEffect:Fr,useLayoutEffect:Fr,useMemo:Fr,useReducer:Fr,useRef:Fr,useState:Fr,useDebugValue:Fr,useDeferredValue:Fr,useTransition:Fr,useMutableSource:Fr,useSyncExternalStore:Fr,useId:Fr,unstable_isNewReconciler:!1},B3={readContext:Dn,useCallback:function(s,u){return jo().memoizedState=[s,u===void 0?null:u],s},useContext:Dn,useEffect:MT,useImperativeHandle:function(s,u,p){return p=p!=null?p.concat([s]):null,Nd(4194308,4,NT.bind(null,u,s),p)},useLayoutEffect:function(s,u){return Nd(4194308,4,s,u)},useInsertionEffect:function(s,u){return Nd(4,2,s,u)},useMemo:function(s,u){var p=jo();return u=u===void 0?null:u,s=s(),p.memoizedState=[s,u],s},useReducer:function(s,u,p){var v=jo();return u=p!==void 0?p(u):u,v.memoizedState=v.baseState=u,s={pending:null,interleaved:null,lanes:0,dispatch:null,lastRenderedReducer:s,lastRenderedState:u},v.queue=s,s=s.dispatch=N3.bind(null,nr,s),[v.memoizedState,s]},useRef:function(s){var u=jo();return s={current:s},u.memoizedState=s},useState:RT,useDebugValue:qg,useDeferredValue:function(s){return jo().memoizedState=s},useTransition:function(){var s=RT(!1),u=s[0];return s=L3.bind(null,s[1]),jo().memoizedState=s,[u,s]},useMutableSource:function(){},useSyncExternalStore:function(s,u,p){var v=nr,O=jo();if(Xt){if(p===void 0)throw Error(r(407));p=p()}else{if(p=u(),kr===null)throw Error(r(349));(Ia&30)!==0||kT(v,u,p)}O.memoizedState=p;var R={value:p,getSnapshot:u};return O.queue=R,MT(_T.bind(null,v,R,s),[s]),v.flags|=2048,Ku(9,ET.bind(null,v,R,p,u),void 0,null),p},useId:function(){var s=jo(),u=kr.identifierPrefix;if(Xt){var p=si,v=ai;p=(v&~(1<<32-lo(v)-1)).toString(32)+p,u=":"+u+"R"+p,p=Hu++,0<\/script>",s=s.removeChild(s.firstChild)):typeof v.is=="string"?s=D.createElement(p,{is:v.is}):(s=D.createElement(p),p==="select"&&(D=s,v.multiple?D.multiple=!0:v.size&&(D.size=v.size))):s=D.createElementNS(s,p),s[_o]=u,s[Du]=v,fk(s,u,!1,!1),u.stateNode=s;e:{switch(D=Nt(p,v),p){case"dialog":Ht("cancel",s),Ht("close",s),O=v;break;case"iframe":case"object":case"embed":Ht("load",s),O=v;break;case"video":case"audio":for(O=0;OQs&&(u.flags|=128,v=!0,Gu(R,!1),u.lanes=4194304)}else{if(!v)if(s=Md(D),s!==null){if(u.flags|=128,v=!0,p=s.updateQueue,p!==null&&(u.updateQueue=p,u.flags|=4),Gu(R,!0),R.tail===null&&R.tailMode==="hidden"&&!D.alternate&&!Xt)return Wr(u),null}else 2*fr()-R.renderingStartTime>Qs&&p!==1073741824&&(u.flags|=128,v=!0,Gu(R,!1),u.lanes=4194304);R.isBackwards?(D.sibling=u.child,u.child=D):(p=R.last,p!==null?p.sibling=D:u.child=D,R.last=D)}return R.tail!==null?(u=R.tail,R.rendering=u,R.tail=u.sibling,R.renderingStartTime=fr(),u.sibling=null,p=rr.current,Ft(rr,v?p&1|2:p&1),u):(Wr(u),null);case 22:case 23:return py(),v=u.memoizedState!==null,s!==null&&s.memoizedState!==null!==v&&(u.flags|=8192),v&&(u.mode&1)!==0?(Tn&1073741824)!==0&&(Wr(u),u.subtreeFlags&6&&(u.flags|=8192)):Wr(u),null;case 24:return null;case 25:return null}throw Error(r(156,u.tag))}function K3(s,u){switch(Sg(u),u.tag){case 1:return sn(u.type)&&Cd(),s=u.flags,s&65536?(u.flags=s&-65537|128,u):null;case 3:return Ks(),Vt(an),Vt(zr),Ig(),s=u.flags,(s&65536)!==0&&(s&128)===0?(u.flags=s&-65537|128,u):null;case 5:return $g(u),null;case 13:if(Vt(rr),s=u.memoizedState,s!==null&&s.dehydrated!==null){if(u.alternate===null)throw Error(r(340));qs()}return s=u.flags,s&65536?(u.flags=s&-65537|128,u):null;case 19:return Vt(rr),null;case 4:return Ks(),null;case 10:return Eg(u.type._context),null;case 22:case 23:return py(),null;case 24:return null;default:return null}}var qd=!1,qr=!1,G3=typeof WeakSet=="function"?WeakSet:Set,Le=null;function Ys(s,u){var p=s.ref;if(p!==null)if(typeof p=="function")try{p(null)}catch(v){ir(s,u,v)}else p.current=null}function ty(s,u,p){try{p()}catch(v){ir(s,u,v)}}var hk=!1;function Y3(s,u){if(pg=ud,s=VO(),ig(s)){if("selectionStart"in s)var p={start:s.selectionStart,end:s.selectionEnd};else e:{p=(p=s.ownerDocument)&&p.defaultView||window;var v=p.getSelection&&p.getSelection();if(v&&v.rangeCount!==0){p=v.anchorNode;var O=v.anchorOffset,R=v.focusNode;v=v.focusOffset;try{p.nodeType,R.nodeType}catch{p=null;break e}var D=0,G=-1,X=-1,ne=0,xe=0,Se=s,be=null;t:for(;;){for(var Ae;Se!==p||O!==0&&Se.nodeType!==3||(G=D+O),Se!==R||v!==0&&Se.nodeType!==3||(X=D+v),Se.nodeType===3&&(D+=Se.nodeValue.length),(Ae=Se.firstChild)!==null;)be=Se,Se=Ae;for(;;){if(Se===s)break t;if(be===p&&++ne===O&&(G=D),be===R&&++xe===v&&(X=D),(Ae=Se.nextSibling)!==null)break;Se=be,be=Se.parentNode}Se=Ae}p=G===-1||X===-1?null:{start:G,end:X}}else p=null}p=p||{start:0,end:0}}else p=null;for(hg={focusedElem:s,selectionRange:p},ud=!1,Le=u;Le!==null;)if(u=Le,s=u.child,(u.subtreeFlags&1028)!==0&&s!==null)s.return=u,Le=s;else for(;Le!==null;){u=Le;try{var De=u.alternate;if((u.flags&1024)!==0)switch(u.tag){case 0:case 11:case 15:break;case 1:if(De!==null){var ze=De.memoizedProps,dr=De.memoizedState,ee=u.stateNode,Q=ee.getSnapshotBeforeUpdate(u.elementType===u.type?ze:fo(u.type,ze),dr);ee.__reactInternalSnapshotBeforeUpdate=Q}break;case 3:var re=u.stateNode.containerInfo;re.nodeType===1?re.textContent="":re.nodeType===9&&re.documentElement&&re.removeChild(re.documentElement);break;case 5:case 6:case 4:case 17:break;default:throw Error(r(163))}}catch(Pe){ir(u,u.return,Pe)}if(s=u.sibling,s!==null){s.return=u.return,Le=s;break}Le=u.return}return De=hk,hk=!1,De}function Yu(s,u,p){var v=u.updateQueue;if(v=v!==null?v.lastEffect:null,v!==null){var O=v=v.next;do{if((O.tag&s)===s){var R=O.destroy;O.destroy=void 0,R!==void 0&&ty(u,p,R)}O=O.next}while(O!==v)}}function Ud(s,u){if(u=u.updateQueue,u=u!==null?u.lastEffect:null,u!==null){var p=u=u.next;do{if((p.tag&s)===s){var v=p.create;p.destroy=v()}p=p.next}while(p!==u)}}function ry(s){var u=s.ref;if(u!==null){var p=s.stateNode;switch(s.tag){case 5:s=p;break;default:s=p}typeof u=="function"?u(s):u.current=s}}function mk(s){var u=s.alternate;u!==null&&(s.alternate=null,mk(u)),s.child=null,s.deletions=null,s.sibling=null,s.tag===5&&(u=s.stateNode,u!==null&&(delete u[_o],delete u[Du],delete u[yg],delete u[j3],delete u[R3])),s.stateNode=null,s.return=null,s.dependencies=null,s.memoizedProps=null,s.memoizedState=null,s.pendingProps=null,s.stateNode=null,s.updateQueue=null}function vk(s){return s.tag===5||s.tag===3||s.tag===4}function gk(s){e:for(;;){for(;s.sibling===null;){if(s.return===null||vk(s.return))return null;s=s.return}for(s.sibling.return=s.return,s=s.sibling;s.tag!==5&&s.tag!==6&&s.tag!==18;){if(s.flags&2||s.child===null||s.tag===4)continue e;s.child.return=s,s=s.child}if(!(s.flags&2))return s.stateNode}}function ny(s,u,p){var v=s.tag;if(v===5||v===6)s=s.stateNode,u?p.nodeType===8?p.parentNode.insertBefore(s,u):p.insertBefore(s,u):(p.nodeType===8?(u=p.parentNode,u.insertBefore(s,p)):(u=p,u.appendChild(s)),p=p._reactRootContainer,p!=null||u.onclick!==null||(u.onclick=wd));else if(v!==4&&(s=s.child,s!==null))for(ny(s,u,p),s=s.sibling;s!==null;)ny(s,u,p),s=s.sibling}function oy(s,u,p){var v=s.tag;if(v===5||v===6)s=s.stateNode,u?p.insertBefore(s,u):p.appendChild(s);else if(v!==4&&(s=s.child,s!==null))for(oy(s,u,p),s=s.sibling;s!==null;)oy(s,u,p),s=s.sibling}var Mr=null,po=!1;function Ji(s,u,p){for(p=p.child;p!==null;)yk(s,u,p),p=p.sibling}function yk(s,u,p){if(Eo&&typeof Eo.onCommitFiberUnmount=="function")try{Eo.onCommitFiberUnmount(nd,p)}catch{}switch(p.tag){case 5:qr||Ys(p,u);case 6:var v=Mr,O=po;Mr=null,Ji(s,u,p),Mr=v,po=O,Mr!==null&&(po?(s=Mr,p=p.stateNode,s.nodeType===8?s.parentNode.removeChild(p):s.removeChild(p)):Mr.removeChild(p.stateNode));break;case 18:Mr!==null&&(po?(s=Mr,p=p.stateNode,s.nodeType===8?gg(s.parentNode,p):s.nodeType===1&&gg(s,p),ku(s)):gg(Mr,p.stateNode));break;case 4:v=Mr,O=po,Mr=p.stateNode.containerInfo,po=!0,Ji(s,u,p),Mr=v,po=O;break;case 0:case 11:case 14:case 15:if(!qr&&(v=p.updateQueue,v!==null&&(v=v.lastEffect,v!==null))){O=v=v.next;do{var R=O,D=R.destroy;R=R.tag,D!==void 0&&((R&2)!==0||(R&4)!==0)&&ty(p,u,D),O=O.next}while(O!==v)}Ji(s,u,p);break;case 1:if(!qr&&(Ys(p,u),v=p.stateNode,typeof v.componentWillUnmount=="function"))try{v.props=p.memoizedProps,v.state=p.memoizedState,v.componentWillUnmount()}catch(G){ir(p,u,G)}Ji(s,u,p);break;case 21:Ji(s,u,p);break;case 22:p.mode&1?(qr=(v=qr)||p.memoizedState!==null,Ji(s,u,p),qr=v):Ji(s,u,p);break;default:Ji(s,u,p)}}function bk(s){var u=s.updateQueue;if(u!==null){s.updateQueue=null;var p=s.stateNode;p===null&&(p=s.stateNode=new G3),u.forEach(function(v){var O=oz.bind(null,s,v);p.has(v)||(p.add(v),v.then(O,O))})}}function ho(s,u){var p=u.deletions;if(p!==null)for(var v=0;vO&&(O=D),v&=~R}if(v=O,v=fr()-v,v=(120>v?120:480>v?480:1080>v?1080:1920>v?1920:3e3>v?3e3:4320>v?4320:1960*Q3(v/1960))-v,10s?16:s,ea===null)var v=!1;else{if(s=ea,ea=null,Yd=0,(xt&6)!==0)throw Error(r(331));var O=xt;for(xt|=4,Le=s.current;Le!==null;){var R=Le,D=R.child;if((Le.flags&16)!==0){var G=R.deletions;if(G!==null){for(var X=0;Xfr()-sy?Da(s,0):ay|=p),cn(s,u)}function Rk(s,u){u===0&&((s.mode&1)===0?u=1:(u=id,id<<=1,(id&130023424)===0&&(id=4194304)));var p=Zr();s=li(s,u),s!==null&&(Su(s,u,p),cn(s,p))}function nz(s){var u=s.memoizedState,p=0;u!==null&&(p=u.retryLane),Rk(s,p)}function oz(s,u){var p=0;switch(s.tag){case 13:var v=s.stateNode,O=s.memoizedState;O!==null&&(p=O.retryLane);break;case 19:v=s.stateNode;break;default:throw Error(r(314))}v!==null&&v.delete(u),Rk(s,p)}var $k;$k=function(s,u,p){if(s!==null)if(s.memoizedProps!==u.pendingProps||an.current)ln=!0;else{if((s.lanes&p)===0&&(u.flags&128)===0)return ln=!1,H3(s,u,p);ln=(s.flags&131072)!==0}else ln=!1,Xt&&(u.flags&1048576)!==0&&dT(u,kd,u.index);switch(u.lanes=0,u.tag){case 2:var v=u.type;Wd(s,u),s=u.pendingProps;var O=zs(u,zr.current);Vs(u,p),O=Dg(null,u,v,s,O,p);var R=Bg();return u.flags|=1,typeof O=="object"&&O!==null&&typeof O.render=="function"&&O.$$typeof===void 0?(u.tag=1,u.memoizedState=null,u.updateQueue=null,sn(v)?(R=!0,Pd(u)):R=!1,u.memoizedState=O.state!==null&&O.state!==void 0?O.state:null,jg(u),O.updater=zd,u.stateNode=O,O._reactInternals=u,Hg(u,v,s,p),u=Yg(null,u,v,!0,R,p)):(u.tag=0,Xt&&R&&wg(u),Jr(null,u,O,p),u=u.child),u;case 16:v=u.elementType;e:{switch(Wd(s,u),s=u.pendingProps,O=v._init,v=O(v._payload),u.type=v,O=u.tag=az(v),s=fo(v,s),O){case 0:u=Gg(null,u,v,s,p);break e;case 1:u=ik(null,u,v,s,p);break e;case 11:u=ek(null,u,v,s,p);break e;case 14:u=tk(null,u,v,fo(v.type,s),p);break e}throw Error(r(306,v,""))}return u;case 0:return v=u.type,O=u.pendingProps,O=u.elementType===v?O:fo(v,O),Gg(s,u,v,O,p);case 1:return v=u.type,O=u.pendingProps,O=u.elementType===v?O:fo(v,O),ik(s,u,v,O,p);case 3:e:{if(ak(u),s===null)throw Error(r(387));v=u.pendingProps,R=u.memoizedState,O=R.element,wT(s,u),$d(u,v,null,p);var D=u.memoizedState;if(v=D.element,R.isDehydrated)if(R={element:v,isDehydrated:!1,cache:D.cache,pendingSuspenseBoundaries:D.pendingSuspenseBoundaries,transitions:D.transitions},u.updateQueue.baseState=R,u.memoizedState=R,u.flags&256){O=Gs(Error(r(423)),u),u=sk(s,u,v,p,O);break e}else if(v!==O){O=Gs(Error(r(424)),u),u=sk(s,u,v,p,O);break e}else for(On=Vi(u.stateNode.containerInfo.firstChild),Pn=u,Xt=!0,co=null,p=bT(u,null,v,p),u.child=p;p;)p.flags=p.flags&-3|4096,p=p.sibling;else{if(qs(),v===O){u=ci(s,u,p);break e}Jr(s,u,v,p)}u=u.child}return u;case 5:return PT(u),s===null&&Pg(u),v=u.type,O=u.pendingProps,R=s!==null?s.memoizedProps:null,D=O.children,mg(v,O)?D=null:R!==null&&mg(v,R)&&(u.flags|=32),ok(s,u),Jr(s,u,D,p),u.child;case 6:return s===null&&Pg(u),null;case 13:return lk(s,u,p);case 4:return Rg(u,u.stateNode.containerInfo),v=u.pendingProps,s===null?u.child=Us(u,null,v,p):Jr(s,u,v,p),u.child;case 11:return v=u.type,O=u.pendingProps,O=u.elementType===v?O:fo(v,O),ek(s,u,v,O,p);case 7:return Jr(s,u,u.pendingProps,p),u.child;case 8:return Jr(s,u,u.pendingProps.children,p),u.child;case 12:return Jr(s,u,u.pendingProps.children,p),u.child;case 10:e:{if(v=u.type._context,O=u.pendingProps,R=u.memoizedProps,D=O.value,Ft(Ad,v._currentValue),v._currentValue=D,R!==null)if(uo(R.value,D)){if(R.children===O.children&&!an.current){u=ci(s,u,p);break e}}else for(R=u.child,R!==null&&(R.return=u);R!==null;){var G=R.dependencies;if(G!==null){D=R.child;for(var X=G.firstContext;X!==null;){if(X.context===v){if(R.tag===1){X=ui(-1,p&-p),X.tag=2;var ne=R.updateQueue;if(ne!==null){ne=ne.shared;var xe=ne.pending;xe===null?X.next=X:(X.next=xe.next,xe.next=X),ne.pending=X}}R.lanes|=p,X=R.alternate,X!==null&&(X.lanes|=p),_g(R.return,p,u),G.lanes|=p;break}X=X.next}}else if(R.tag===10)D=R.type===u.type?null:R.child;else if(R.tag===18){if(D=R.return,D===null)throw Error(r(341));D.lanes|=p,G=D.alternate,G!==null&&(G.lanes|=p),_g(D,p,u),D=R.sibling}else D=R.child;if(D!==null)D.return=R;else for(D=R;D!==null;){if(D===u){D=null;break}if(R=D.sibling,R!==null){R.return=D.return,D=R;break}D=D.return}R=D}Jr(s,u,O.children,p),u=u.child}return u;case 9:return O=u.type,v=u.pendingProps.children,Vs(u,p),O=Dn(O),v=v(O),u.flags|=1,Jr(s,u,v,p),u.child;case 14:return v=u.type,O=fo(v,u.pendingProps),O=fo(v.type,O),tk(s,u,v,O,p);case 15:return rk(s,u,u.type,u.pendingProps,p);case 17:return v=u.type,O=u.pendingProps,O=u.elementType===v?O:fo(v,O),Wd(s,u),u.tag=1,sn(v)?(s=!0,Pd(u)):s=!1,Vs(u,p),KT(u,v,O),Hg(u,v,O,p),Yg(null,u,v,!0,s,p);case 19:return ck(s,u,p);case 22:return nk(s,u,p)}throw Error(r(156,u.tag))};function Mk(s,u){return pO(s,u)}function iz(s,u,p,v){this.tag=s,this.key=p,this.sibling=this.child=this.return=this.stateNode=this.type=this.elementType=null,this.index=0,this.ref=null,this.pendingProps=u,this.dependencies=this.memoizedState=this.updateQueue=this.memoizedProps=null,this.mode=v,this.subtreeFlags=this.flags=0,this.deletions=null,this.childLanes=this.lanes=0,this.alternate=null}function Fn(s,u,p,v){return new iz(s,u,p,v)}function my(s){return s=s.prototype,!(!s||!s.isReactComponent)}function az(s){if(typeof s=="function")return my(s)?1:0;if(s!=null){if(s=s.$$typeof,s===N)return 11;if(s===B)return 14}return 2}function na(s,u){var p=s.alternate;return p===null?(p=Fn(s.tag,u,s.key,s.mode),p.elementType=s.elementType,p.type=s.type,p.stateNode=s.stateNode,p.alternate=s,s.alternate=p):(p.pendingProps=u,p.type=s.type,p.flags=0,p.subtreeFlags=0,p.deletions=null),p.flags=s.flags&14680064,p.childLanes=s.childLanes,p.lanes=s.lanes,p.child=s.child,p.memoizedProps=s.memoizedProps,p.memoizedState=s.memoizedState,p.updateQueue=s.updateQueue,u=s.dependencies,p.dependencies=u===null?null:{lanes:u.lanes,firstContext:u.firstContext},p.sibling=s.sibling,p.index=s.index,p.ref=s.ref,p}function Zd(s,u,p,v,O,R){var D=2;if(v=s,typeof s=="function")my(s)&&(D=1);else if(typeof s=="string")D=5;else e:switch(s){case j:return za(p.children,O,R,u);case $:D=8,O|=8;break;case M:return s=Fn(12,p,u,O|2),s.elementType=M,s.lanes=R,s;case F:return s=Fn(13,p,u,O),s.elementType=F,s.lanes=R,s;case z:return s=Fn(19,p,u,O),s.elementType=z,s.lanes=R,s;case H:return ep(p,O,R,u);default:if(typeof s=="object"&&s!==null)switch(s.$$typeof){case A:D=10;break e;case I:D=9;break e;case N:D=11;break e;case B:D=14;break e;case L:D=16,v=null;break e}throw Error(r(130,s==null?s:typeof s,""))}return u=Fn(D,p,u,O),u.elementType=s,u.type=v,u.lanes=R,u}function za(s,u,p,v){return s=Fn(7,s,v,u),s.lanes=p,s}function ep(s,u,p,v){return s=Fn(22,s,v,u),s.elementType=H,s.lanes=p,s.stateNode={isHidden:!1},s}function vy(s,u,p){return s=Fn(6,s,null,u),s.lanes=p,s}function gy(s,u,p){return u=Fn(4,s.children!==null?s.children:[],s.key,u),u.lanes=p,u.stateNode={containerInfo:s.containerInfo,pendingChildren:null,implementation:s.implementation},u}function sz(s,u,p,v,O){this.tag=u,this.containerInfo=s,this.finishedWork=this.pingCache=this.current=this.pendingChildren=null,this.timeoutHandle=-1,this.callbackNode=this.pendingContext=this.context=null,this.callbackPriority=0,this.eventTimes=Uv(0),this.expirationTimes=Uv(-1),this.entangledLanes=this.finishedLanes=this.mutableReadLanes=this.expiredLanes=this.pingedLanes=this.suspendedLanes=this.pendingLanes=0,this.entanglements=Uv(0),this.identifierPrefix=v,this.onRecoverableError=O,this.mutableSourceEagerHydrationData=null}function yy(s,u,p,v,O,R,D,G,X){return s=new sz(s,u,p,G,X),u===1?(u=1,R===!0&&(u|=8)):u=0,R=Fn(3,null,null,u),s.current=R,R.stateNode=s,R.memoizedState={element:v,isDehydrated:p,cache:null,transitions:null,pendingSuspenseBoundaries:null},jg(R),s}function lz(s,u,p){var v=3"u"||typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE!="function"))try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(e)}catch(t){console.error(t)}}return e(),Oy.exports=Sz(),Oy.exports}var Xk;function Cz(){if(Xk)return lp;Xk=1;var e=HI();return lp.createRoot=e.createRoot,lp.hydrateRoot=e.hydrateRoot,lp}var Pz=Cz();const Oz=bt(Pz);/** + * react-router v7.17.0 + * + * Copyright (c) Remix Software Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE.md file in the root directory of this source tree. + * + * @license MIT + */var Qk="popstate";function Jk(e){return typeof e=="object"&&e!=null&&"pathname"in e&&"search"in e&&"hash"in e&&"state"in e&&"key"in e}function Tz(e={}){function t(n,o){var f;let i=(f=o.state)==null?void 0:f.masked,{pathname:a,search:l,hash:c}=i||n.location;return J1("",{pathname:a,search:l,hash:c},o.state&&o.state.usr||null,o.state&&o.state.key||"default",i?{pathname:n.location.pathname,search:n.location.search,hash:n.location.hash}:void 0)}function r(n,o){return typeof o=="string"?o:zc(o)}return Ez(t,r,null,e)}function tr(e,t){if(e===!1||e===null||typeof e>"u")throw new Error(t)}function Zn(e,t){if(!e){typeof console<"u"&&console.warn(t);try{throw new Error(t)}catch{}}}function kz(){return Math.random().toString(36).substring(2,10)}function Zk(e,t){return{usr:e.state,key:e.key,idx:t,masked:e.mask?{pathname:e.pathname,search:e.search,hash:e.hash}:void 0}}function J1(e,t,r=null,n,o){return{pathname:typeof e=="string"?e:e.pathname,search:"",hash:"",...typeof t=="string"?nu(t):t,state:r,key:t&&t.key||n||kz(),mask:o}}function zc({pathname:e="/",search:t="",hash:r=""}){return t&&t!=="?"&&(e+=t.charAt(0)==="?"?t:"?"+t),r&&r!=="#"&&(e+=r.charAt(0)==="#"?r:"#"+r),e}function nu(e){let t={};if(e){let r=e.indexOf("#");r>=0&&(t.hash=e.substring(r),e=e.substring(0,r));let n=e.indexOf("?");n>=0&&(t.search=e.substring(n),e=e.substring(0,n)),e&&(t.pathname=e)}return t}function Ez(e,t,r,n={}){let{window:o=document.defaultView,v5Compat:i=!1}=n,a=o.history,l="POP",c=null,f=d();f==null&&(f=0,a.replaceState({...a.state,idx:f},""));function d(){return(a.state||{idx:null}).idx}function h(){l="POP";let g=d(),S=g==null?null:g-f;f=g,c&&c({action:l,location:y.location,delta:S})}function m(g,S){l="PUSH";let E=Jk(g)?g:J1(y.location,g,S);f=d()+1;let P=Zk(E,f),T=y.createHref(E.mask||E);try{a.pushState(P,"",T)}catch(C){if(C instanceof DOMException&&C.name==="DataCloneError")throw C;o.location.assign(T)}i&&c&&c({action:l,location:y.location,delta:1})}function b(g,S){l="REPLACE";let E=Jk(g)?g:J1(y.location,g,S);f=d();let P=Zk(E,f),T=y.createHref(E.mask||E);a.replaceState(P,"",T),i&&c&&c({action:l,location:y.location,delta:0})}function w(g){return _z(o,g)}let y={get action(){return l},get location(){return e(o,a)},listen(g){if(c)throw new Error("A history only accepts one active listener");return o.addEventListener(Qk,h),c=g,()=>{o.removeEventListener(Qk,h),c=null}},createHref(g){return t(o,g)},createURL:w,encodeLocation(g){let S=w(g);return{pathname:S.pathname,search:S.search,hash:S.hash}},push:m,replace:b,go(g){return a.go(g)}};return y}function _z(e,t,r=!1){let n="http://localhost";e&&(n=e.location.origin!=="null"?e.location.origin:e.location.href),tr(n,"No window.location.(origin|href) available to create URL");let o=typeof t=="string"?t:zc(t);return o=o.replace(/ $/,"%20"),!r&&o.startsWith("//")&&(o=n+o),new URL(o,n)}function VI(e,t,r="/"){return Az(e,t,r,!1)}function Az(e,t,r,n,o){let i=typeof t=="string"?nu(t):t,a=Pi(i.pathname||"/",r);if(a==null)return null;let l=jz(e),c=null,f=qz(a);for(let d=0;c==null&&d{let d={relativePath:f===void 0?a.path||"":f,caseSensitive:a.caseSensitive===!0,childrenIndex:l,route:a};if(d.relativePath.startsWith("/")){if(!d.relativePath.startsWith(n)&&c)return;tr(d.relativePath.startsWith(n),`Absolute route path "${d.relativePath}" nested under path "${n}" is not valid. An absolute child route path must start with the combined path of all its parent routes.`),d.relativePath=d.relativePath.slice(n.length)}let h=So([n,d.relativePath]),m=r.concat(d);a.children&&a.children.length>0&&(tr(a.index!==!0,`Index routes must not have child routes. Please remove all child routes from route path "${h}".`),KI(a.children,t,m,h,c)),!(a.path==null&&!a.index)&&t.push({path:h,score:Bz(h,a.index),routesMeta:m})};return e.forEach((a,l)=>{var c;if(a.path===""||!((c=a.path)!=null&&c.includes("?")))i(a,l);else for(let f of GI(a.path))i(a,l,!0,f)}),t}function GI(e){let t=e.split("/");if(t.length===0)return[];let[r,...n]=t,o=r.endsWith("?"),i=r.replace(/\?$/,"");if(n.length===0)return o?[i,""]:[i];let a=GI(n.join("/")),l=[];return l.push(...a.map(c=>c===""?i:[i,c].join("/"))),o&&l.push(...a),l.map(c=>e.startsWith("/")&&c===""?"/":c)}function Rz(e){e.sort((t,r)=>t.score!==r.score?r.score-t.score:zz(t.routesMeta.map(n=>n.childrenIndex),r.routesMeta.map(n=>n.childrenIndex)))}var $z=/^:[\w-]+$/,Mz=3,Iz=2,Lz=1,Nz=10,Dz=-2,eE=e=>e==="*";function Bz(e,t){let r=e.split("/"),n=r.length;return r.some(eE)&&(n+=Dz),t&&(n+=Iz),r.filter(o=>!eE(o)).reduce((o,i)=>o+($z.test(i)?Mz:i===""?Lz:Nz),n)}function zz(e,t){return e.length===t.length&&e.slice(0,-1).every((n,o)=>n===t[o])?e[e.length-1]-t[t.length-1]:0}function Fz(e,t,r=!1){let{routesMeta:n}=e,o={},i="/",a=[];for(let l=0;l{if(d==="*"){let w=l[m]||"";a=i.slice(0,i.length-w.length).replace(/(.)\/+$/,"$1")}const b=l[m];return h&&!b?f[d]=void 0:f[d]=(b||"").replace(/%2F/g,"/"),f},{}),pathname:i,pathnameBase:a,pattern:e}}function Wz(e,t=!1,r=!0){Zn(e==="*"||!e.endsWith("*")||e.endsWith("/*"),`Route path "${e}" will be treated as if it were "${e.replace(/\*$/,"/*")}" because the \`*\` character must always follow a \`/\` in the pattern. To get rid of this warning, please change the route path to "${e.replace(/\*$/,"/*")}".`);let n=[],o="^"+e.replace(/\/*\*?$/,"").replace(/^\/*/,"/").replace(/[\\.*+^${}|()[\]]/g,"\\$&").replace(/\/:([\w-]+)(\?)?/g,(a,l,c,f,d)=>{if(n.push({paramName:l,isOptional:c!=null}),c){let h=d.charAt(f+a.length);return h&&h!=="/"?"/([^\\/]*)":"(?:/([^\\/]*))?"}return"/([^\\/]+)"}).replace(/\/([\w-]+)\?(\/|$)/g,"(/$1)?$2");return e.endsWith("*")?(n.push({paramName:"*"}),o+=e==="*"||e==="/*"?"(.*)$":"(?:\\/(.+)|\\/*)$"):r?o+="\\/*$":e!==""&&e!=="/"&&(o+="(?:(?=\\/|$))"),[new RegExp(o,t?void 0:"i"),n]}function qz(e){try{return e.split("/").map(t=>decodeURIComponent(t).replace(/\//g,"%2F")).join("/")}catch(t){return Zn(!1,`The URL path "${e}" could not be decoded because it is a malformed URL segment. This is probably due to a bad percent encoding (${t}).`),e}}function Pi(e,t){if(t==="/")return e;if(!e.toLowerCase().startsWith(t.toLowerCase()))return null;let r=t.endsWith("/")?t.length-1:t.length,n=e.charAt(r);return n&&n!=="/"?null:e.slice(r)||"/"}var Uz=/^(?:[a-z][a-z0-9+.-]*:|\/\/)/i;function Hz(e,t="/"){let{pathname:r,search:n="",hash:o=""}=typeof e=="string"?nu(e):e,i;return r?(r=YI(r),r.startsWith("/")?i=tE(r.substring(1),"/"):i=tE(r,t)):i=t,{pathname:i,search:Gz(n),hash:Yz(o)}}function tE(e,t){let r=uh(t).split("/");return e.split("/").forEach(o=>{o===".."?r.length>1&&r.pop():o!=="."&&r.push(o)}),r.length>1?r.join("/"):"/"}function Ey(e,t,r,n){return`Cannot include a '${e}' character in a manually specified \`to.${t}\` field [${JSON.stringify(n)}]. Please separate it out to the \`to.${r}\` field. Alternatively you may provide the full path as a string in and the router will parse it for you.`}function Vz(e){return e.filter((t,r)=>r===0||t.route.path&&t.route.path.length>0)}function eC(e){let t=Vz(e);return t.map((r,n)=>n===t.length-1?r.pathname:r.pathnameBase)}function wm(e,t,r,n=!1){let o;typeof e=="string"?o=nu(e):(o={...e},tr(!o.pathname||!o.pathname.includes("?"),Ey("?","pathname","search",o)),tr(!o.pathname||!o.pathname.includes("#"),Ey("#","pathname","hash",o)),tr(!o.search||!o.search.includes("#"),Ey("#","search","hash",o)));let i=e===""||o.pathname==="",a=i?"/":o.pathname,l;if(a==null)l=r;else{let h=t.length-1;if(!n&&a.startsWith("..")){let m=a.split("/");for(;m[0]==="..";)m.shift(),h-=1;o.pathname=m.join("/")}l=h>=0?t[h]:"/"}let c=Hz(o,l),f=a&&a!=="/"&&a.endsWith("/"),d=(i||a===".")&&r.endsWith("/");return!c.pathname.endsWith("/")&&(f||d)&&(c.pathname+="/"),c}var YI=e=>e.replace(/\/\/+/g,"/"),So=e=>YI(e.join("/")),uh=e=>e.replace(/\/+$/,""),Kz=e=>uh(e).replace(/^\/*/,"/"),Gz=e=>!e||e==="?"?"":e.startsWith("?")?e:"?"+e,Yz=e=>!e||e==="#"?"":e.startsWith("#")?e:"#"+e,Xz=class{constructor(e,t,r,n=!1){this.status=e,this.statusText=t||"",this.internal=n,r instanceof Error?(this.data=r.toString(),this.error=r):this.data=r}};function Qz(e){return e!=null&&typeof e.status=="number"&&typeof e.statusText=="string"&&typeof e.internal=="boolean"&&"data"in e}function Jz(e){let t=e.map(r=>r.route.path).filter(Boolean);return So(t)||"/"}var XI=typeof window<"u"&&typeof window.document<"u"&&typeof window.document.createElement<"u";function QI(e,t){let r=e;if(typeof r!="string"||!Uz.test(r))return{absoluteURL:void 0,isExternal:!1,to:r};let n=r,o=!1;if(XI)try{let i=new URL(window.location.href),a=r.startsWith("//")?new URL(i.protocol+r):new URL(r),l=Pi(a.pathname,t);a.origin===i.origin&&l!=null?r=l+a.search+a.hash:o=!0}catch{Zn(!1,` contains an invalid URL which will probably break when clicked - please update to a valid URL path.`)}return{absoluteURL:n,isExternal:o,to:r}}Object.getOwnPropertyNames(Object.prototype).sort().join("\0");var JI=["POST","PUT","PATCH","DELETE"];new Set(JI);var Zz=["GET",...JI];new Set(Zz);var ou=_.createContext(null);ou.displayName="DataRouter";var Sm=_.createContext(null);Sm.displayName="DataRouterState";var ZI=_.createContext(!1);function eF(){return _.useContext(ZI)}var eL=_.createContext({isTransitioning:!1});eL.displayName="ViewTransition";var tF=_.createContext(new Map);tF.displayName="Fetchers";var rF=_.createContext(null);rF.displayName="Await";var Mn=_.createContext(null);Mn.displayName="Navigation";var Lf=_.createContext(null);Lf.displayName="Location";var no=_.createContext({outlet:null,matches:[],isDataRoute:!1});no.displayName="Route";var tC=_.createContext(null);tC.displayName="RouteError";var tL="REACT_ROUTER_ERROR",nF="REDIRECT",oF="ROUTE_ERROR_RESPONSE";function iF(e){if(e.startsWith(`${tL}:${nF}:{`))try{let t=JSON.parse(e.slice(28));if(typeof t=="object"&&t&&typeof t.status=="number"&&typeof t.statusText=="string"&&typeof t.location=="string"&&typeof t.reloadDocument=="boolean"&&typeof t.replace=="boolean")return t}catch{}}function aF(e){if(e.startsWith(`${tL}:${oF}:{`))try{let t=JSON.parse(e.slice(40));if(typeof t=="object"&&t&&typeof t.status=="number"&&typeof t.statusText=="string")return new Xz(t.status,t.statusText,t.data)}catch{}}function sF(e,{relative:t}={}){tr(iu(),"useHref() may be used only in the context of a component.");let{basename:r,navigator:n}=_.useContext(Mn),{hash:o,pathname:i,search:a}=Nf(e,{relative:t}),l=i;return r!=="/"&&(l=i==="/"?r:So([r,i])),n.createHref({pathname:l,search:a,hash:o})}function iu(){return _.useContext(Lf)!=null}function oo(){return tr(iu(),"useLocation() may be used only in the context of a component."),_.useContext(Lf).location}var rL="You should call navigate() in a React.useEffect(), not when your component is first rendered.";function nL(e){_.useContext(Mn).static||_.useLayoutEffect(e)}function Jo(){let{isDataRoute:e}=_.useContext(no);return e?CF():lF()}function lF(){tr(iu(),"useNavigate() may be used only in the context of a component.");let e=_.useContext(ou),{basename:t,navigator:r}=_.useContext(Mn),{matches:n}=_.useContext(no),{pathname:o}=oo(),i=JSON.stringify(eC(n)),a=_.useRef(!1);return nL(()=>{a.current=!0}),_.useCallback((c,f={})=>{if(Zn(a.current,rL),!a.current)return;if(typeof c=="number"){r.go(c);return}let d=wm(c,JSON.parse(i),o,f.relative==="path");e==null&&t!=="/"&&(d.pathname=d.pathname==="/"?t:So([t,d.pathname])),(f.replace?r.replace:r.push)(d,f.state,f)},[t,r,i,o,e])}var uF=_.createContext(null);function cF(e){let t=_.useContext(no).outlet;return _.useMemo(()=>t&&_.createElement(uF.Provider,{value:e},t),[t,e])}function fF(){let{matches:e}=_.useContext(no),t=e[e.length-1];return(t==null?void 0:t.params)??{}}function Nf(e,{relative:t}={}){let{matches:r}=_.useContext(no),{pathname:n}=oo(),o=JSON.stringify(eC(r));return _.useMemo(()=>wm(e,JSON.parse(o),n,t==="path"),[e,o,n,t])}function dF(e,t){return oL(e,t)}function oL(e,t,r){var g;tr(iu(),"useRoutes() may be used only in the context of a component.");let{navigator:n}=_.useContext(Mn),{matches:o}=_.useContext(no),i=o[o.length-1],a=i?i.params:{},l=i?i.pathname:"/",c=i?i.pathnameBase:"/",f=i&&i.route;{let S=f&&f.path||"";aL(l,!f||S.endsWith("*")||S.endsWith("*?"),`You rendered descendant (or called \`useRoutes()\`) at "${l}" (under ) but the parent route path has no trailing "*". This means if you navigate deeper, the parent won't match anymore and therefore the child routes will never render. + +Please change the parent to .`)}let d=oo(),h;if(t){let S=typeof t=="string"?nu(t):t;tr(c==="/"||((g=S.pathname)==null?void 0:g.startsWith(c)),`When overriding the location using \`\` or \`useRoutes(routes, location)\`, the location pathname must begin with the portion of the URL pathname that was matched by all parent routes. The current pathname base is "${c}" but pathname "${S.pathname}" was given in the \`location\` prop.`),h=S}else h=d;let m=h.pathname||"/",b=m;if(c!=="/"){let S=c.replace(/^\//,"").split("/");b="/"+m.replace(/^\//,"").split("/").slice(S.length).join("/")}let w=r&&r.state.matches.length?r.state.matches.map(S=>Object.assign(S,{route:r.manifest[S.route.id]||S.route})):VI(e,{pathname:b});Zn(f||w!=null,`No routes matched location "${h.pathname}${h.search}${h.hash}" `),Zn(w==null||w[w.length-1].route.element!==void 0||w[w.length-1].route.Component!==void 0||w[w.length-1].route.lazy!==void 0,`Matched leaf route at location "${h.pathname}${h.search}${h.hash}" does not have an element or Component. This means it will render an with a null value by default resulting in an "empty" page.`);let y=gF(w&&w.map(S=>Object.assign({},S,{params:Object.assign({},a,S.params),pathname:So([c,n.encodeLocation?n.encodeLocation(S.pathname.replace(/%/g,"%25").replace(/\?/g,"%3F").replace(/#/g,"%23")).pathname:S.pathname]),pathnameBase:S.pathnameBase==="/"?c:So([c,n.encodeLocation?n.encodeLocation(S.pathnameBase.replace(/%/g,"%25").replace(/\?/g,"%3F").replace(/#/g,"%23")).pathname:S.pathnameBase])})),o,r);return t&&y?_.createElement(Lf.Provider,{value:{location:{pathname:"/",search:"",hash:"",state:null,key:"default",mask:void 0,...h},navigationType:"POP"}},y):y}function pF(){let e=SF(),t=Qz(e)?`${e.status} ${e.statusText}`:e instanceof Error?e.message:JSON.stringify(e),r=e instanceof Error?e.stack:null,n="rgba(200,200,200, 0.5)",o={padding:"0.5rem",backgroundColor:n},i={padding:"2px 4px",backgroundColor:n},a=null;return console.error("Error handled by React Router default ErrorBoundary:",e),a=_.createElement(_.Fragment,null,_.createElement("p",null,"💿 Hey developer 👋"),_.createElement("p",null,"You can provide a way better UX than this when your app throws errors by providing your own ",_.createElement("code",{style:i},"ErrorBoundary")," or"," ",_.createElement("code",{style:i},"errorElement")," prop on your route.")),_.createElement(_.Fragment,null,_.createElement("h2",null,"Unexpected Application Error!"),_.createElement("h3",{style:{fontStyle:"italic"}},t),r?_.createElement("pre",{style:o},r):null,a)}var hF=_.createElement(pF,null),iL=class extends _.Component{constructor(e){super(e),this.state={location:e.location,revalidation:e.revalidation,error:e.error}}static getDerivedStateFromError(e){return{error:e}}static getDerivedStateFromProps(e,t){return t.location!==e.location||t.revalidation!=="idle"&&e.revalidation==="idle"?{error:e.error,location:e.location,revalidation:e.revalidation}:{error:e.error!==void 0?e.error:t.error,location:t.location,revalidation:e.revalidation||t.revalidation}}componentDidCatch(e,t){this.props.onError?this.props.onError(e,t):console.error("React Router caught the following error during render",e)}render(){let e=this.state.error;if(this.context&&typeof e=="object"&&e&&"digest"in e&&typeof e.digest=="string"){const r=aF(e.digest);r&&(e=r)}let t=e!==void 0?_.createElement(no.Provider,{value:this.props.routeContext},_.createElement(tC.Provider,{value:e,children:this.props.component})):this.props.children;return this.context?_.createElement(mF,{error:e},t):t}};iL.contextType=ZI;var _y=new WeakMap;function mF({children:e,error:t}){let{basename:r}=_.useContext(Mn);if(typeof t=="object"&&t&&"digest"in t&&typeof t.digest=="string"){let n=iF(t.digest);if(n){let o=_y.get(t);if(o)throw o;let i=QI(n.location,r);if(XI&&!_y.get(t))if(i.isExternal||n.reloadDocument)window.location.href=i.absoluteURL||i.to;else{const a=Promise.resolve().then(()=>window.__reactRouterDataRouter.navigate(i.to,{replace:n.replace}));throw _y.set(t,a),a}return _.createElement("meta",{httpEquiv:"refresh",content:`0;url=${i.absoluteURL||i.to}`})}}return e}function vF({routeContext:e,match:t,children:r}){let n=_.useContext(ou);return n&&n.static&&n.staticContext&&(t.route.errorElement||t.route.ErrorBoundary)&&(n.staticContext._deepestRenderedBoundaryId=t.route.id),_.createElement(no.Provider,{value:e},r)}function gF(e,t=[],r){let n=r==null?void 0:r.state;if(e==null){if(!n)return null;if(n.errors)e=n.matches;else if(t.length===0&&!n.initialized&&n.matches.length>0)e=n.matches;else return null}let o=e,i=n==null?void 0:n.errors;if(i!=null){let d=o.findIndex(h=>h.route.id&&(i==null?void 0:i[h.route.id])!==void 0);tr(d>=0,`Could not find a matching route for errors on route IDs: ${Object.keys(i).join(",")}`),o=o.slice(0,Math.min(o.length,d+1))}let a=!1,l=-1;if(r&&n){a=n.renderFallback;for(let d=0;d=0?o=o.slice(0,l+1):o=[o[0]];break}}}}let c=r==null?void 0:r.onError,f=n&&c?(d,h)=>{var m,b;c(d,{location:n.location,params:((b=(m=n.matches)==null?void 0:m[0])==null?void 0:b.params)??{},pattern:Jz(n.matches),errorInfo:h})}:void 0;return o.reduceRight((d,h,m)=>{let b,w=!1,y=null,g=null;n&&(b=i&&h.route.id?i[h.route.id]:void 0,y=h.route.errorElement||hF,a&&(l<0&&m===0?(aL("route-fallback",!1,"No `HydrateFallback` element provided to render during initial hydration"),w=!0,g=null):l===m&&(w=!0,g=h.route.hydrateFallbackElement||null)));let S=t.concat(o.slice(0,m+1)),E=()=>{let P;return b?P=y:w?P=g:h.route.Component?P=_.createElement(h.route.Component,null):h.route.element?P=h.route.element:P=d,_.createElement(vF,{match:h,routeContext:{outlet:d,matches:S,isDataRoute:n!=null},children:P})};return n&&(h.route.ErrorBoundary||h.route.errorElement||m===0)?_.createElement(iL,{location:n.location,revalidation:n.revalidation,component:y,error:b,children:E(),routeContext:{outlet:null,matches:S,isDataRoute:!0},onError:f}):E()},null)}function rC(e){return`${e} must be used within a data router. See https://reactrouter.com/en/main/routers/picking-a-router.`}function yF(e){let t=_.useContext(ou);return tr(t,rC(e)),t}function bF(e){let t=_.useContext(Sm);return tr(t,rC(e)),t}function xF(e){let t=_.useContext(no);return tr(t,rC(e)),t}function nC(e){let t=xF(e),r=t.matches[t.matches.length-1];return tr(r.route.id,`${e} can only be used on routes that contain a unique "id"`),r.route.id}function wF(){return nC("useRouteId")}function SF(){var n;let e=_.useContext(tC),t=bF("useRouteError"),r=nC("useRouteError");return e!==void 0?e:(n=t.errors)==null?void 0:n[r]}function CF(){let{router:e}=yF("useNavigate"),t=nC("useNavigate"),r=_.useRef(!1);return nL(()=>{r.current=!0}),_.useCallback(async(o,i={})=>{Zn(r.current,rL),r.current&&(typeof o=="number"?await e.navigate(o):await e.navigate(o,{fromRouteId:t,...i}))},[e,t])}var rE={};function aL(e,t,r){!t&&!rE[e]&&(rE[e]=!0,Zn(!1,r))}_.memo(PF);function PF({routes:e,manifest:t,future:r,state:n,isStatic:o,onError:i}){return oL(e,void 0,{manifest:t,state:n,isStatic:o,onError:i})}function OF({to:e,replace:t,state:r,relative:n}){tr(iu()," may be used only in the context of a component.");let{static:o}=_.useContext(Mn);Zn(!o," must not be used on the initial render in a . This is a no-op, but you should modify your code so the is only ever rendered in response to some user interaction or state change.");let{matches:i}=_.useContext(no),{pathname:a}=oo(),l=Jo(),c=wm(e,eC(i),a,n==="path"),f=JSON.stringify(c);return _.useEffect(()=>{l(JSON.parse(f),{replace:t,state:r,relative:n})},[l,f,n,t,r]),null}function TF(e){return cF(e.context)}function Lo(e){tr(!1,"A is only ever to be used as the child of element, never rendered directly. Please wrap your in a .")}function kF({basename:e="/",children:t=null,location:r,navigationType:n="POP",navigator:o,static:i=!1,useTransitions:a}){tr(!iu(),"You cannot render a inside another . You should never have more than one in your app.");let l=e.replace(/^\/*/,"/"),c=_.useMemo(()=>({basename:l,navigator:o,static:i,useTransitions:a,future:{}}),[l,o,i,a]);typeof r=="string"&&(r=nu(r));let{pathname:f="/",search:d="",hash:h="",state:m=null,key:b="default",mask:w}=r,y=_.useMemo(()=>{let g=Pi(f,l);return g==null?null:{location:{pathname:g,search:d,hash:h,state:m,key:b,mask:w},navigationType:n}},[l,f,d,h,m,b,n,w]);return Zn(y!=null,` is not able to match the URL "${f}${d}${h}" because it does not start with the basename, so the won't render anything.`),y==null?null:_.createElement(Mn.Provider,{value:c},_.createElement(Lf.Provider,{children:t,value:y}))}function EF({children:e,location:t}){return dF(Z1(e),t)}function Z1(e,t=[]){let r=[];return _.Children.forEach(e,(n,o)=>{if(!_.isValidElement(n))return;let i=[...t,o];if(n.type===_.Fragment){r.push.apply(r,Z1(n.props.children,i));return}tr(n.type===Lo,`[${typeof n.type=="string"?n.type:n.type.name}] is not a component. All component children of must be a or `),tr(!n.props.index||!n.props.children,"An index route cannot have child routes.");let a={id:n.props.id||i.join("-"),caseSensitive:n.props.caseSensitive,element:n.props.element,Component:n.props.Component,index:n.props.index,path:n.props.path,middleware:n.props.middleware,loader:n.props.loader,action:n.props.action,hydrateFallbackElement:n.props.hydrateFallbackElement,HydrateFallback:n.props.HydrateFallback,errorElement:n.props.errorElement,ErrorBoundary:n.props.ErrorBoundary,hasErrorBoundary:n.props.hasErrorBoundary===!0||n.props.ErrorBoundary!=null||n.props.errorElement!=null,shouldRevalidate:n.props.shouldRevalidate,handle:n.props.handle,lazy:n.props.lazy};n.props.children&&(a.children=Z1(n.props.children,i)),r.push(a)}),r}var Up="get",Hp="application/x-www-form-urlencoded";function Cm(e){return typeof HTMLElement<"u"&&e instanceof HTMLElement}function _F(e){return Cm(e)&&e.tagName.toLowerCase()==="button"}function AF(e){return Cm(e)&&e.tagName.toLowerCase()==="form"}function jF(e){return Cm(e)&&e.tagName.toLowerCase()==="input"}function RF(e){return!!(e.metaKey||e.altKey||e.ctrlKey||e.shiftKey)}function $F(e,t){return e.button===0&&(!t||t==="_self")&&!RF(e)}function ew(e=""){return new URLSearchParams(typeof e=="string"||Array.isArray(e)||e instanceof URLSearchParams?e:Object.keys(e).reduce((t,r)=>{let n=e[r];return t.concat(Array.isArray(n)?n.map(o=>[r,o]):[[r,n]])},[]))}function MF(e,t){let r=ew(e);return t&&t.forEach((n,o)=>{r.has(o)||t.getAll(o).forEach(i=>{r.append(o,i)})}),r}var up=null;function IF(){if(up===null)try{new FormData(document.createElement("form"),0),up=!1}catch{up=!0}return up}var LF=new Set(["application/x-www-form-urlencoded","multipart/form-data","text/plain"]);function Ay(e){return e!=null&&!LF.has(e)?(Zn(!1,`"${e}" is not a valid \`encType\` for \`
\`/\`\` and will default to "${Hp}"`),null):e}function NF(e,t){let r,n,o,i,a;if(AF(e)){let l=e.getAttribute("action");n=l?Pi(l,t):null,r=e.getAttribute("method")||Up,o=Ay(e.getAttribute("enctype"))||Hp,i=new FormData(e)}else if(_F(e)||jF(e)&&(e.type==="submit"||e.type==="image")){let l=e.form;if(l==null)throw new Error('Cannot submit a