diff --git a/.gitignore b/.gitignore index 87bf8fc9d..34e4855ae 100644 --- a/.gitignore +++ b/.gitignore @@ -165,6 +165,7 @@ cython_debug/ /temp_image/ /browser_data/ /data/ +/data/xhs/private/ */.DS_Store .vscode @@ -183,4 +184,5 @@ agent_zone debug_tools database/*.db -.omx/ \ No newline at end of file +.omx/ +.worktrees/ diff --git a/api/routers/crawler.py b/api/routers/crawler.py index eead9e1ae..6342f24c7 100644 --- a/api/routers/crawler.py +++ b/api/routers/crawler.py @@ -27,14 +27,18 @@ @router.post("/start") async def start_crawler(request: CrawlerStartRequest): """Start crawler task""" - success = await crawler_manager.start(request) - if not success: + task_id = await crawler_manager.start(request) + if not task_id: # Handle concurrent/duplicate requests: if process is already running, return 400 instead of 500 if crawler_manager.process and crawler_manager.process.poll() is None: raise HTTPException(status_code=400, detail="Crawler is already running") raise HTTPException(status_code=500, detail="Failed to start crawler") - return {"status": "ok", "message": "Crawler started successfully"} + return { + "status": "ok", + "message": "Crawler started successfully", + "task_id": task_id, + } @router.post("/stop") diff --git a/api/schemas/crawler.py b/api/schemas/crawler.py index 6eb3b1b7e..cd25be3ef 100644 --- a/api/schemas/crawler.py +++ b/api/schemas/crawler.py @@ -71,6 +71,7 @@ class CrawlerStartRequest(BaseModel): start_page: int = 1 enable_comments: bool = True enable_sub_comments: bool = False + capture_creator_ids: bool = False save_option: SaveDataOptionEnum = SaveDataOptionEnum.JSONL cookies: str = "" headless: bool = False @@ -85,6 +86,9 @@ class CrawlerStatusResponse(BaseModel): crawler_type: Optional[str] = None started_at: Optional[str] = None error_message: Optional[str] = None + task_id: Optional[str] = None + last_exit_code: Optional[int] = None + finished_at: Optional[str] = None class LogEntry(BaseModel): diff --git a/api/services/crawler_manager.py b/api/services/crawler_manager.py index 9af954b3d..9a3f1f3d8 100644 --- a/api/services/crawler_manager.py +++ b/api/services/crawler_manager.py @@ -23,6 +23,7 @@ from typing import Optional, List from datetime import datetime from pathlib import Path +from uuid import uuid4 from ..schemas import CrawlerStartRequest, LogEntry @@ -36,6 +37,10 @@ def __init__(self): self.status = "idle" self.started_at: Optional[datetime] = None self.current_config: Optional[CrawlerStartRequest] = None + self.task_id: Optional[str] = None + self.last_exit_code: Optional[int] = None + self.finished_at: Optional[datetime] = None + self.error_message: Optional[str] = None self._log_id = 0 self._logs: List[LogEntry] = [] self._read_task: Optional[asyncio.Task] = None @@ -90,11 +95,15 @@ def _parse_log_level(self, line: str) -> str: return "debug" return "info" - async def start(self, config: CrawlerStartRequest) -> bool: - """Start crawler process""" + async def start(self, config: CrawlerStartRequest) -> Optional[str]: + """Start crawler process and return its task id.""" async with self._lock: - if self.process and self.process.poll() is None: - return False + if ( + self._read_task and not self._read_task.done() + ) or ( + self.process and self.process.poll() is None + ): + return None # Clear old logs self._logs = [] @@ -130,6 +139,10 @@ async def start(self, config: CrawlerStartRequest) -> bool: env={**os.environ, "PYTHONUNBUFFERED": "1"} ) + self.task_id = str(uuid4()) + self.last_exit_code = None + self.finished_at = None + self.error_message = None self.status = "running" self.started_at = datetime.now() self.current_config = config @@ -143,52 +156,76 @@ async def start(self, config: CrawlerStartRequest) -> bool: # Start log reading task self._read_task = asyncio.create_task(self._read_output()) - return True + return self.task_id except Exception as e: self.status = "error" entry = self._create_log_entry(f"Failed to start crawler: {str(e)}", "error") await self._push_log(entry) - return False + return None async def stop(self) -> bool: """Stop crawler process""" async with self._lock: - if not self.process or self.process.poll() is not None: + process = self.process + if not process or process.poll() is not None: return False self.status = "stopping" entry = self._create_log_entry("Sending SIGTERM to crawler process...", "warning") await self._push_log(entry) + errors = [] try: - self.process.send_signal(signal.SIGTERM) - + process.send_signal(signal.SIGTERM) + except Exception as e: + errors.append(str(e)) + else: # Wait for graceful exit (up to 15 seconds) for _ in range(30): - if self.process.poll() is not None: + if process.poll() is not None: break await asyncio.sleep(0.5) - # If still not exited, force kill - if self.process.poll() is None: - entry = self._create_log_entry("Process not responding, sending SIGKILL...", "warning") - await self._push_log(entry) - self.process.kill() - - entry = self._create_log_entry("Crawler process terminated", "info") + # If still not exited, force kill and wait for confirmed exit. + if process.poll() is None: + entry = self._create_log_entry("Process not responding, sending SIGKILL...", "warning") await self._push_log(entry) - - except Exception as e: - entry = self._create_log_entry(f"Error stopping crawler: {str(e)}", "error") + try: + process.kill() + await asyncio.to_thread(process.wait, timeout=5) + except Exception as e: + errors.append(str(e)) + + exit_code = process.poll() + if exit_code is None: + message = "Error stopping crawler: " + "; ".join( + errors or ["process did not exit"] + ) + self.status = "error" + self.error_message = message + entry = self._create_log_entry(message, "error") await self._push_log(entry) + return False + entry = self._create_log_entry("Crawler process terminated", "info") + await self._push_log(entry) self.status = "idle" self.current_config = None - - # Cancel log reading task - if self._read_task: - self._read_task.cancel() - self._read_task = None + self.last_exit_code = exit_code + self.finished_at = datetime.now() + self.error_message = None + + # Cancel and settle log reader before allowing a new start. + read_task = self._read_task + if read_task: + read_task.cancel() + try: + await read_task + except asyncio.CancelledError: + pass + finally: + if self._read_task is read_task: + self._read_task = None return True @@ -199,7 +236,10 @@ def get_status(self) -> dict: "platform": self.current_config.platform.value if self.current_config else None, "crawler_type": self.current_config.crawler_type.value if self.current_config else None, "started_at": self.started_at.isoformat() if self.started_at else None, - "error_message": None + "error_message": self.error_message, + "task_id": self.task_id, + "last_exit_code": self.last_exit_code, + "finished_at": self.finished_at.isoformat() if self.finished_at else None, } def _build_command(self, config: CrawlerStartRequest) -> list: @@ -225,6 +265,9 @@ def _build_command(self, config: CrawlerStartRequest) -> list: cmd.extend(["--get_comment", "true" if config.enable_comments else "false"]) cmd.extend(["--get_sub_comment", "true" if config.enable_sub_comments else "false"]) + if config.capture_creator_ids: + cmd.extend(["--capture_creator_ids", "true"]) + if config.max_notes_count is not None: cmd.extend(["--crawler_max_notes_count", str(config.max_notes_count)]) @@ -241,12 +284,14 @@ def _build_command(self, config: CrawlerStartRequest) -> list: async def _read_output(self): """Asynchronously read process output""" loop = asyncio.get_event_loop() + process = self.process + task_id = self.task_id try: - while self.process and self.process.poll() is None: + while process and process.poll() is None: # Read a line in thread pool line = await loop.run_in_executor( - None, self.process.stdout.readline + None, process.stdout.readline ) if line: line = line.strip() @@ -256,9 +301,9 @@ async def _read_output(self): await self._push_log(entry) # Read remaining output - if self.process and self.process.stdout: + if process and process.stdout: remaining = await loop.run_in_executor( - None, self.process.stdout.read + None, process.stdout.read ) if remaining: for line in remaining.strip().split('\n'): @@ -268,20 +313,35 @@ async def _read_output(self): await self._push_log(entry) # Process ended - if self.status == "running": - exit_code = self.process.returncode if self.process else -1 + if ( + self.process is process + and self.task_id == task_id + and self.status == "running" + ): + exit_code = process.returncode if process else -1 + self.last_exit_code = exit_code + self.finished_at = datetime.now() + self.error_message = None if exit_code == 0: entry = self._create_log_entry("Crawler completed successfully", "success") else: entry = self._create_log_entry(f"Crawler exited with code: {exit_code}", "warning") await self._push_log(entry) self.status = "idle" + self.current_config = None except asyncio.CancelledError: pass except Exception as e: - entry = self._create_log_entry(f"Error reading output: {str(e)}", "error") - await self._push_log(entry) + if self.process is process and self.task_id == task_id: + message = f"Error reading output: {str(e)}" + returncode = process.returncode if process else None + self.last_exit_code = returncode if isinstance(returncode, int) else -1 + self.finished_at = datetime.now() + self.error_message = message + self.status = "error" + entry = self._create_log_entry(message, "error") + await self._push_log(entry) # Global singleton diff --git a/cmd_arg/arg.py b/cmd_arg/arg.py index 20d1d3976..4ae1bc0c5 100644 --- a/cmd_arg/arg.py +++ b/cmd_arg/arg.py @@ -216,6 +216,15 @@ def main( show_default=True, ), ] = str(config.ENABLE_GET_SUB_COMMENTS), + capture_creator_ids: Annotated[ + str, + typer.Option( + "--capture_creator_ids", + help="Whether to capture creator IDs from XHS search results, supports yes/true/t/y/1 or no/false/f/n/0", + rich_help_panel="Basic Configuration", + show_default=True, + ), + ] = str(config.ENABLE_XHS_CREATOR_ID_CAPTURE), headless: Annotated[ str, typer.Option( @@ -337,6 +346,7 @@ def main( enable_comment = _to_bool(get_comment) enable_sub_comment = _to_bool(get_sub_comment) + enable_creator_id_capture = _to_bool(capture_creator_ids) enable_headless = _to_bool(headless) enable_ip_proxy_value = _to_bool(enable_ip_proxy) init_db_value = init_db.value if init_db else None @@ -353,6 +363,7 @@ def main( config.KEYWORDS = keywords config.ENABLE_GET_COMMENTS = enable_comment config.ENABLE_GET_SUB_COMMENTS = enable_sub_comment + config.ENABLE_XHS_CREATOR_ID_CAPTURE = enable_creator_id_capture config.HEADLESS = enable_headless config.CDP_HEADLESS = enable_headless config.SAVE_DATA_OPTION = save_data_option.value diff --git a/config/base_config.py b/config/base_config.py index 28a852e08..beeee9817 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -92,6 +92,10 @@ # Data saving path, if not specified by default, it will be saved to the data folder. SAVE_DATA_PATH = "" +# Opt-in local sidecar for XHS creator IDs captured during keyword search. +ENABLE_XHS_CREATOR_ID_CAPTURE = False +XHS_CREATOR_ID_CAPTURE_DIR = "" + # Browser file configuration cached by the user's browser USER_DATA_DIR = "%s_user_data_dir" # %s will be replaced by platform name diff --git a/docs/superpowers/plans/2026-07-27-keyword-crawl-runner.md b/docs/superpowers/plans/2026-07-27-keyword-crawl-runner.md new file mode 100644 index 000000000..b26df5a86 --- /dev/null +++ b/docs/superpowers/plans/2026-07-27-keyword-crawl-runner.md @@ -0,0 +1,1076 @@ +# 关键词自动采集队列实施计划 + +> **供自动化执行代理使用：** 必须使用 `subagent-driven-development`（推荐）或 `executing-plans`，按任务逐项实施本计划。步骤使用复选框（`- [ ]`）跟踪进度。 + +**目标：** 构建一个可断点续跑的本地队列，读取已确认的20个关键词，逐词调用 MediaCrawler API，并避免重复执行已成功任务。 + +**架构：** 为 MediaCrawler 增加最小任务标识与状态字段（`task_id`、`last_exit_code`、`finished_at`）。同级目录中的独立 Python 程序只使用标准库，负责解析 Markdown、原子保存 JSON 进度、逐个提交 API 任务，并依据任务 ID 恢复中断任务。 + +**技术栈：** Python 3.11、FastAPI/Pydantic、pytest/pytest-asyncio，以及 Python 标准库（`argparse`、`hashlib`、`json`、`urllib.request`、`unittest`）。 + +## 全局约束 + +- 只解析 `## 建议首先执行的20个词`，并要求恰好得到20个唯一关键词。 +- 每次只提交一个关键词，设置 `max_notes_count=20`，并关闭两级评论采集。 +- 使用 MediaCrawler 现有 API `http://127.0.0.1:8080/api`，不自动操作 WebUI 输入框。 +- 每次状态变化都原子保存；绝不自动重复执行 `succeeded` 或 `needs_review` 项。 +- 明确失败后继续下一个关键词；只有指定 `--retry-failed` 时才重试失败项。 +- 队列程序不增加任何第三方依赖。 +- 保持 MediaCrawler 当前匿名存储行为，不改动用户现有的未跟踪讨论文档。 +- 仅用于个人、非商业研究和公开笔记。 + +--- + +## 文件结构 + +### MediaCrawler 仓库（`D:\red_note_rich_search\MediaCrawler`） + +- 修改 `api/schemas/crawler.py`：公开任务标识和完成字段。 +- 修改 `api/services/crawler_manager.py`：创建任务 ID 并保留退出元数据。 +- 修改 `api/routers/crawler.py`：返回创建的任务 ID。 +- 修改 `tests/test_api_limits.py`：使现有启动接口模拟响应适配新接口。 +- 创建 `tests/test_crawler_task_status.py`：验证任务生命周期和 API 序列化。 + +### 独立队列程序仓库（`D:\red_note_rich_search\keyword_crawl_runner`） + +- 创建 `.gitignore`：排除运行进度和 Python 缓存。 +- 创建 `run_keywords.py`：实现 Markdown 解析、进度持久化、HTTP 客户端、恢复和命令行入口。 +- 创建 `test_run_keywords.py`：使用模拟 API 客户端编写标准库单元测试。 +- 创建 `README.md`：记录准确的启动、续跑和重试命令。 + +--- + +### 任务1：为 MediaCrawler API 增加可追踪的任务标识 + +**文件：** +- 修改：`api/schemas/crawler.py:81-87` +- 修改：`api/services/crawler_manager.py:19-285` +- 修改：`api/routers/crawler.py:27-37` +- 修改：`tests/test_api_limits.py:64-107` +- 创建：`tests/test_crawler_task_status.py` + +**接口：** +- 产出：`CrawlerManager.start(config) -> Optional[str]`，成功时返回 UUID 字符串，未启动进程时返回 `None`。 +- 产出：`CrawlerManager.get_status() -> dict`，包含 `task_id`、`last_exit_code` 和 `finished_at`。 +- 产出：`POST /api/crawler/start` 响应包含 `task_id`。 + +- [ ] **步骤1：编写失败的响应结构与生命周期测试** + +创建 `tests/test_crawler_task_status.py`： + +```python +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from api.main import app +from api.schemas import CrawlerStartRequest, PlatformEnum +from api.services.crawler_manager import CrawlerManager + + +def request() -> CrawlerStartRequest: + return CrawlerStartRequest(platform=PlatformEnum.XHS, keywords="重庆企业主资产配置") + + +@pytest.mark.asyncio +async def test_start_returns_task_id_and_clears_completion_state(): + manager = CrawlerManager() + manager.last_exit_code = 1 + manager.finished_at = datetime.now() + process = MagicMock() + process.poll.return_value = None + + with patch("api.services.crawler_manager.subprocess.Popen", return_value=process): + with patch("api.services.crawler_manager.asyncio.create_task"): + task_id = await manager.start(request()) + + assert isinstance(task_id, str) + assert task_id == manager.task_id + assert manager.last_exit_code is None + assert manager.finished_at is None + assert manager.status == "running" + + +@pytest.mark.asyncio +async def test_read_output_records_exit_metadata(): + manager = CrawlerManager() + manager.status = "running" + manager.current_config = request() + manager.task_id = "task-123" + process = MagicMock() + process.poll.side_effect = [0, 0] + process.returncode = 0 + process.stdout.readline.return_value = "" + process.stdout.read.return_value = "" + manager.process = process + + await manager._read_output() + + status = manager.get_status() + assert status["status"] == "idle" + assert status["task_id"] == "task-123" + assert status["last_exit_code"] == 0 + assert status["finished_at"] is not None + assert status["platform"] is None + + +def test_start_endpoint_returns_task_id(): + client = TestClient(app) + with patch( + "api.routers.crawler.crawler_manager.start", + new_callable=AsyncMock, + return_value="task-123", + ): + response = client.post( + "/api/crawler/start", + json={"platform": "xhs", "keywords": "重庆企业主资产配置"}, + ) + + assert response.status_code == 200 + assert response.json() == { + "status": "ok", + "message": "Crawler started successfully", + "task_id": "task-123", + } + + +def test_status_endpoint_serializes_completion_fields(): + client = TestClient(app) + status = { + "status": "idle", + "platform": None, + "crawler_type": None, + "started_at": "2026-07-27T10:00:00+08:00", + "error_message": None, + "task_id": "task-123", + "last_exit_code": 0, + "finished_at": "2026-07-27T10:02:00+08:00", + } + with patch("api.routers.crawler.crawler_manager.get_status", return_value=status): + response = client.get("/api/crawler/status") + + assert response.status_code == 200 + assert response.json() == status +``` + +- [ ] **步骤2：更新现有 API 模拟值，并运行测试确认失败** + +在 `tests/test_api_limits.py` 的两个启动接口测试中，将： + +```python +mock_start.return_value = True +``` + +替换为： + +```python +mock_start.return_value = "task-123" +``` + +将精确响应断言更新为： + +```python +assert response.json() == { + "status": "ok", + "message": "Crawler started successfully", + "task_id": "task-123", +} +``` + +运行： + +```powershell +uv run pytest tests/test_crawler_task_status.py tests/test_api_limits.py -q +``` + +预期：失败，因为 `CrawlerStatusResponse` 尚无完成字段，且 `start()` 仍返回 `bool`。 + +- [ ] **步骤3：扩展状态响应结构** + +将 `api/schemas/crawler.py` 中的 `CrawlerStatusResponse` 替换为： + +```python +class CrawlerStatusResponse(BaseModel): + """Crawler status response""" + status: Literal["idle", "running", "stopping", "error"] + platform: Optional[str] = None + crawler_type: Optional[str] = None + started_at: Optional[str] = None + error_message: Optional[str] = None + task_id: Optional[str] = None + last_exit_code: Optional[int] = None + finished_at: Optional[str] = None +``` + +- [ ] **步骤4：在任务管理器中实现生命周期元数据** + +增加导入： + +```python +from uuid import uuid4 +``` + +在 `CrawlerManager.__init__` 中增加字段： + +```python +self.task_id: Optional[str] = None +self.last_exit_code: Optional[int] = None +self.finished_at: Optional[datetime] = None +``` + +修改方法签名及失败返回值： + +```python +async def start(self, config: CrawlerStartRequest) -> Optional[str]: + """Start crawler process and return its task id.""" +``` + +`subprocess.Popen` 调用成功后立即设置： + +```python +self.task_id = str(uuid4()) +self.last_exit_code = None +self.finished_at = None +self.status = "running" +self.started_at = datetime.now() +self.current_config = config +``` + +成功时返回 `self.task_id` 而不是 `True`；两个失败分支返回 `None` 而不是 `False`。 + +在 `_read_output` 中，将进程结束处理代码替换为： + +```python +if self.status == "running": + exit_code = self.process.returncode if self.process else -1 + self.last_exit_code = exit_code + self.finished_at = datetime.now() + if exit_code == 0: + entry = self._create_log_entry("Crawler completed successfully", "success") + else: + entry = self._create_log_entry( + f"Crawler exited with code: {exit_code}", "warning" + ) + await self._push_log(entry) + self.status = "idle" + self.current_config = None +``` + +扩展 `get_status()`： + +```python +return { + "status": self.status, + "platform": self.current_config.platform.value if self.current_config else None, + "crawler_type": self.current_config.crawler_type.value if self.current_config else None, + "started_at": self.started_at.isoformat() if self.started_at else None, + "error_message": None, + "task_id": self.task_id, + "last_exit_code": self.last_exit_code, + "finished_at": self.finished_at.isoformat() if self.finished_at else None, +} +``` + +`stop()` 完成进程终止后设置： + +```python +self.last_exit_code = self.process.poll() +self.finished_at = datetime.now() +``` + +- [ ] **步骤5：由路由返回任务 ID** + +将 `api/routers/crawler.py` 中启动处理函数的函数体替换为： + +```python +task_id = await crawler_manager.start(request) +if not task_id: + if crawler_manager.process and crawler_manager.process.poll() is None: + raise HTTPException(status_code=400, detail="Crawler is already running") + raise HTTPException(status_code=500, detail="Failed to start crawler") + +return { + "status": "ok", + "message": "Crawler started successfully", + "task_id": task_id, +} +``` + +- [ ] **步骤6：运行聚焦测试和回归测试** + +运行： + +```powershell +uv run pytest tests/test_crawler_task_status.py tests/test_api_limits.py -q +uv run pytest tests -q +``` + +预期：两条命令均通过；第一条覆盖新增生命周期测试，第二条无回归失败。 + +- [ ] **步骤7：提交 API 增强** + +```powershell +git add api/schemas/crawler.py api/services/crawler_manager.py api/routers/crawler.py tests/test_api_limits.py tests/test_crawler_task_status.py +git commit -m "feat: expose crawler task completion status" +``` + +--- + +### 任务2：实现 Markdown 解析和进度原子保存 + +**文件：** +- 创建：`D:\red_note_rich_search\keyword_crawl_runner\.gitignore` +- 创建：`D:\red_note_rich_search\keyword_crawl_runner\run_keywords.py` +- 创建：`D:\red_note_rich_search\keyword_crawl_runner\test_run_keywords.py` + +**接口：** +- 产出：`extract_keywords(markdown: str) -> list[str]`。 +- 产出：`keyword_digest(keywords: list[str]) -> str`。 +- 产出：`new_progress(source: Path, keywords: list[str]) -> dict`。 +- 产出：`atomic_write_json(path: Path, value: dict) -> None`。 +- 产出：`load_progress(path: Path, source: Path, keywords: list[str]) -> dict`。 + +- [ ] **步骤1：创建并初始化独立仓库** + +```powershell +New-Item -ItemType Directory -Force -Path D:\red_note_rich_search\keyword_crawl_runner +git -C D:\red_note_rich_search\keyword_crawl_runner init +``` + +创建 `.gitignore`： + +```gitignore +__pycache__/ +*.pyc +crawl_progress.json +crawl_progress.json.tmp +``` + +- [ ] **步骤2：编写失败的解析和持久化测试** + +创建 `test_run_keywords.py`： + +```python +import json +import tempfile +import unittest +from pathlib import Path + +from run_keywords import ( + atomic_write_json, + extract_keywords, + keyword_digest, + load_progress, + new_progress, +) + + +KEYWORDS = [f"关键词{i:02d}" for i in range(1, 21)] + + +def markdown(words=KEYWORDS): + body = "\n".join(words) + return f"""# 词表 + +## 第一批 +```text +不应读取 +``` + +## 建议首先执行的20个词 +```text +{body} +``` + +## 建议配置的排除方向 +```text +广告 +``` +""" + + +class ParsingTests(unittest.TestCase): + def test_extracts_only_approved_twenty(self): + self.assertEqual(extract_keywords(markdown()), KEYWORDS) + + def test_rejects_wrong_count_after_deduplication(self): + with self.assertRaisesRegex(ValueError, "恰好包含20个"): + extract_keywords(markdown(KEYWORDS[:-1] + [KEYWORDS[0]])) + + def test_digest_changes_when_order_changes(self): + self.assertNotEqual(keyword_digest(KEYWORDS), keyword_digest(KEYWORDS[::-1])) + + +class ProgressTests(unittest.TestCase): + def test_new_progress_starts_pending(self): + state = new_progress(Path("words.md"), KEYWORDS) + self.assertTrue(all(item["status"] == "pending" for item in state["keywords"])) + self.assertEqual(state["keywords_sha256"], keyword_digest(KEYWORDS)) + + def test_atomic_write_produces_valid_json(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "progress.json" + atomic_write_json(path, {"ok": True}) + self.assertEqual(json.loads(path.read_text(encoding="utf-8")), {"ok": True}) + self.assertFalse(path.with_suffix(path.suffix + ".tmp").exists()) + + def test_load_rejects_changed_keyword_plan(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "progress.json" + atomic_write_json(path, new_progress(Path("words.md"), KEYWORDS)) + with self.assertRaisesRegex(ValueError, "词表已变化"): + load_progress(path, Path("words.md"), KEYWORDS[::-1]) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **步骤3：运行测试确认失败** + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +python -m unittest -v test_run_keywords.py +``` + +预期：报错 `ModuleNotFoundError: No module named 'run_keywords'`。 + +- [ ] **步骤4：实现解析和进度函数** + +创建 `run_keywords.py`，包含以下导入和函数： + +```python +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import time +from datetime import datetime +from pathlib import Path +from typing import Callable +from urllib import error, request + + +SECTION = "## 建议首先执行的20个词" +VALID_STATUSES = {"pending", "running", "succeeded", "failed", "needs_review"} + + +def now_iso() -> str: + return datetime.now().astimezone().isoformat(timespec="seconds") + + +def extract_keywords(markdown: str) -> list[str]: + lines = markdown.splitlines() + try: + section_index = next(i for i, line in enumerate(lines) if line.strip() == SECTION) + fence_start = next( + i for i in range(section_index + 1, len(lines)) + if lines[i].strip() == "```text" + ) + fence_end = next( + i for i in range(fence_start + 1, len(lines)) + if lines[i].strip() == "```" + ) + except StopIteration as exc: + raise ValueError("找不到建议首先执行的20个词代码块") from exc + + keywords = list(dict.fromkeys( + line.strip() for line in lines[fence_start + 1:fence_end] if line.strip() + )) + if len(keywords) != 20: + raise ValueError(f"建议词代码块去重后必须恰好包含20个关键词，实际为{len(keywords)}个") + return keywords + + +def keyword_digest(keywords: list[str]) -> str: + return hashlib.sha256("\n".join(keywords).encode("utf-8")).hexdigest() + + +def new_progress(source: Path, keywords: list[str]) -> dict: + timestamp = now_iso() + return { + "source_file": str(source.resolve()), + "keywords_sha256": keyword_digest(keywords), + "created_at": timestamp, + "updated_at": timestamp, + "keywords": [ + { + "keyword": keyword, + "status": "pending", + "attempts": 0, + "task_id": None, + "started_at": None, + "finished_at": None, + "exit_code": None, + "error": None, + } + for keyword in keywords + ], + } + + +def atomic_write_json(path: Path, value: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(value, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + os.replace(temporary, path) + + +def load_progress(path: Path, source: Path, keywords: list[str]) -> dict: + if not path.exists(): + state = new_progress(source, keywords) + atomic_write_json(path, state) + return state + + try: + state = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"进度文件无法读取: {exc}") from exc + if state.get("keywords_sha256") != keyword_digest(keywords): + raise ValueError("词表已变化，不能复用现有进度文件") + items = state.get("keywords") + if not isinstance(items, list) or any( + not isinstance(item, dict) or item.get("status") not in VALID_STATUSES + for item in items + ): + raise ValueError("进度文件结构无效") + return state +``` + +- [ ] **步骤5：运行解析和持久化测试** + +```powershell +python -m unittest -v test_run_keywords.py +``` + +预期：6项测试全部通过。 + +- [ ] **步骤6：提交独立程序基础功能** + +```powershell +git add .gitignore run_keywords.py test_run_keywords.py +git commit -m "feat: parse keyword plan and persist progress" +``` + +--- + +### 任务3：实现 API 编排和中断恢复 + +**文件：** +- 修改：`D:\red_note_rich_search\keyword_crawl_runner\run_keywords.py` +- 修改：`D:\red_note_rich_search\keyword_crawl_runner\test_run_keywords.py` + +**接口：** +- 产出：`CrawlerApi.status() -> dict` 和 `CrawlerApi.start(keyword: str) -> str`。 +- 产出：`reconcile_running(state, api, save, sleep) -> bool`。 +- 产出：`run_queue(state, api, save, retry_failed, sleep) -> int`。 +- 依赖：任务1的 API 字段和任务2的进度函数。 + +- [ ] **步骤1：编写失败的队列行为测试** + +追加到 `test_run_keywords.py`： + +```python +from run_keywords import run_queue + + +class FakeApi: + def __init__(self, outcomes): + self.outcomes = list(outcomes) + self.started = [] + self.current = None + + def status(self): + if self.current is None: + return { + "status": "idle", + "task_id": None, + "last_exit_code": None, + "finished_at": None, + } + task_id, exit_code = self.current + return { + "status": "idle", + "task_id": task_id, + "last_exit_code": exit_code, + "finished_at": "2026-07-27T10:02:00+08:00", + } + + def start(self, keyword): + self.started.append(keyword) + exit_code = self.outcomes.pop(0) + task_id = f"task-{len(self.started)}" + self.current = (task_id, exit_code) + return task_id + + +class QueueTests(unittest.TestCase): + def state(self): + return new_progress(Path("words.md"), KEYWORDS) + + def test_skips_success_and_continues_after_failure(self): + state = self.state() + state["keywords"][0]["status"] = "succeeded" + api = FakeApi([1] + [0] * 18) + saves = [] + result = run_queue(state, api, saves.append, False, lambda _: None) + self.assertEqual(result, 1) + self.assertNotIn(KEYWORDS[0], api.started) + self.assertEqual(state["keywords"][1]["status"], "failed") + self.assertEqual(state["keywords"][2]["status"], "succeeded") + self.assertGreater(len(saves), 20) + + def test_retry_failed_only_resets_failed(self): + state = self.state() + for item in state["keywords"]: + item["status"] = "succeeded" + state["keywords"][3]["status"] = "failed" + state["keywords"][4]["status"] = "needs_review" + api = FakeApi([0]) + run_queue(state, api, lambda _: None, True, lambda _: None) + self.assertEqual(api.started, [KEYWORDS[3]]) + self.assertEqual(state["keywords"][4]["status"], "needs_review") + + def test_mismatched_running_task_needs_review(self): + state = self.state() + item = state["keywords"][0] + item.update({"status": "running", "task_id": "old-task"}) + api = FakeApi([]) + api.current = ("other-task", 0) + result = run_queue(state, api, lambda _: None, False, lambda _: None) + self.assertEqual(result, 2) + self.assertEqual(item["status"], "needs_review") + self.assertEqual(api.started, []) +``` + +- [ ] **步骤2：运行队列测试确认失败** + +```powershell +python -m unittest -v test_run_keywords.py +``` + +预期：报错，因为 `run_queue` 尚未定义。 + +- [ ] **步骤3：实现 HTTP 客户端** + +追加到 `run_keywords.py`： + +```python +class ApiError(RuntimeError): + def __init__(self, message: str, status_code: int | None = None): + super().__init__(message) + self.status_code = status_code + + +class CrawlerApi: + def __init__(self, base_url: str, timeout: float = 10.0): + self.base_url = base_url.rstrip("/") + self.timeout = timeout + + def _request(self, method: str, path: str, payload: dict | None = None) -> dict: + body = None if payload is None else json.dumps(payload).encode("utf-8") + req = request.Request( + f"{self.base_url}{path}", + data=body, + method=method, + headers={"Content-Type": "application/json"}, + ) + try: + with request.urlopen(req, timeout=self.timeout) as response: + return json.loads(response.read().decode("utf-8")) + except error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise ApiError(detail or str(exc), exc.code) from exc + except (error.URLError, TimeoutError, json.JSONDecodeError) as exc: + raise ApiError(str(exc)) from exc + + def status(self) -> dict: + return self._request("GET", "/crawler/status") + + def start(self, keyword: str) -> str: + response = self._request( + "POST", + "/crawler/start", + { + "platform": "xhs", + "login_type": "qrcode", + "crawler_type": "search", + "keywords": keyword, + "start_page": 1, + "enable_comments": False, + "enable_sub_comments": False, + "save_option": "json", + "cookies": "", + "headless": False, + "max_notes_count": 20, + }, + ) + task_id = response.get("task_id") + if not isinstance(task_id, str) or not task_id: + raise ApiError("启动响应缺少 task_id") + return task_id +``` + +- [ ] **步骤4：实现任务核对和串行队列** + +追加到 `run_keywords.py`： + +```python +def save_state(state: dict, save: Callable[[dict], None]) -> None: + state["updated_at"] = now_iso() + save(state) + + +def finish_item(item: dict, status: dict) -> None: + exit_code = status.get("last_exit_code") + item["finished_at"] = status.get("finished_at") or now_iso() + item["exit_code"] = exit_code + if exit_code == 0: + item["status"] = "succeeded" + item["error"] = None + else: + item["status"] = "failed" + item["error"] = f"crawler exited with code {exit_code}" + + +def wait_for_task(item, api, save_state_callback, sleep): + while True: + try: + status = api.status() + except ApiError as exc: + item["status"] = "needs_review" + item["error"] = f"无法确认 API 任务结果: {exc}" + save_state_callback() + return False + if status.get("task_id") != item.get("task_id"): + item["status"] = "needs_review" + item["error"] = "API task_id 与进度文件不匹配" + save_state_callback() + return False + if status.get("status") in {"running", "stopping"}: + sleep(2) + continue + if status.get("status") == "idle" and status.get("last_exit_code") is not None: + finish_item(item, status) + save_state_callback() + return True + item["status"] = "needs_review" + item["error"] = "API 未返回可确认的任务结果" + save_state_callback() + return False + + +def run_queue( + state: dict, + api, + save: Callable[[dict], None], + retry_failed: bool, + sleep: Callable[[float], None] = time.sleep, +) -> int: + def persist(): + save_state(state, save) + + if retry_failed: + for item in state["keywords"]: + if item["status"] == "failed": + item.update({ + "status": "pending", + "task_id": None, + "started_at": None, + "finished_at": None, + "exit_code": None, + "error": None, + }) + persist() + + running = next((item for item in state["keywords"] if item["status"] == "running"), None) + if running and not wait_for_task(running, api, persist, sleep): + return 2 + + status = api.status() + if status.get("status") in {"running", "stopping"}: + raise RuntimeError("API 已有其他爬虫任务运行") + + pending = [item for item in state["keywords"] if item["status"] == "pending"] + total = len(state["keywords"]) + for item in pending: + position = state["keywords"].index(item) + 1 + print(f"[{position}/{total}] 开始：{item['keyword']}") + item["status"] = "running" + item["attempts"] += 1 + item["started_at"] = now_iso() + item["task_id"] = None + persist() + try: + item["task_id"] = api.start(item["keyword"]) + persist() + if not wait_for_task(item, api, persist, sleep): + return 2 + except ApiError as exc: + if exc.status_code == 400: + item["status"] = "needs_review" + item["error"] = str(exc) + persist() + return 2 + item["status"] = "failed" + item["finished_at"] = now_iso() + item["error"] = str(exc) + persist() + print(f"[{position}/{total}] {item['status']}：{item['keyword']}") + sleep(5) + + failed = sum(item["status"] == "failed" for item in state["keywords"]) + review = sum(item["status"] == "needs_review" for item in state["keywords"]) + return 2 if review else (1 if failed else 0) +``` + +- [ ] **步骤5：运行队列测试，仅修复测试证实的问题** + +```powershell +python -m unittest -v test_run_keywords.py +``` + +预期：解析、持久化和队列测试全部通过。 + +- [ ] **步骤6：提交队列编排功能** + +```powershell +git add run_keywords.py test_run_keywords.py +git commit -m "feat: run resumable keyword crawl queue" +``` + +--- + +### 任务4：增加命令行入口、操作文档和端到端验证 + +**文件：** +- 修改：`D:\red_note_rich_search\keyword_crawl_runner\run_keywords.py` +- 修改：`D:\red_note_rich_search\keyword_crawl_runner\test_run_keywords.py` +- 创建：`D:\red_note_rich_search\keyword_crawl_runner\README.md` + +**接口：** +- 产出：`main(argv: list[str] | None = None) -> int`。 +- 依赖：任务2和任务3的全部函数。 + +- [ ] **步骤1：编写命令行请求参数测试** + +追加到 `test_run_keywords.py`： + +```python +from unittest.mock import patch +from run_keywords import CrawlerApi + + +class HttpPayloadTests(unittest.TestCase): + def test_start_payload_disables_comments_and_limits_notes(self): + api = CrawlerApi("http://127.0.0.1:8080/api") + with patch.object(api, "_request", return_value={"task_id": "task-123"}) as call: + self.assertEqual(api.start("重庆企业主资产配置"), "task-123") + + method, path, payload = call.call_args.args + self.assertEqual((method, path), ("POST", "/crawler/start")) + self.assertEqual(payload["keywords"], "重庆企业主资产配置") + self.assertEqual(payload["max_notes_count"], 20) + self.assertFalse(payload["enable_comments"]) + self.assertFalse(payload["enable_sub_comments"]) + self.assertEqual(payload["save_option"], "json") +``` + +运行： + +```powershell +python -m unittest -v test_run_keywords.py +``` + +预期：请求参数测试及此前所有测试均通过。该测试在接入命令行前锁定已确认的采集限制。 + +- [ ] **步骤2：实现命令行解析和 `main`** + +追加到 `run_keywords.py`： + +```python +DEFAULT_SOURCE = Path( + r"C:\Users\15377\Documents\民生银行课题\首批可立即使用的搜索词.md" +) +DEFAULT_PROGRESS = Path(__file__).with_name("crawl_progress.json") + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description="逐词调用 MediaCrawler API") + parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE) + parser.add_argument("--api-base", default="http://127.0.0.1:8080/api") + parser.add_argument("--progress", type=Path, default=DEFAULT_PROGRESS) + parser.add_argument("--retry-failed", action="store_true") + return parser.parse_args(argv) + + +def main(argv=None) -> int: + args = parse_args(argv) + try: + source_text = args.source.read_text(encoding="utf-8") + keywords = extract_keywords(source_text) + state = load_progress(args.progress, args.source, keywords) + api = CrawlerApi(args.api_base) + initial = api.status() + running = next( + (item for item in state["keywords"] if item["status"] == "running"), + None, + ) + if initial.get("status") in {"running", "stopping"} and not running: + raise RuntimeError("API 已有其他爬虫任务运行") + result = run_queue( + state, + api, + lambda value: atomic_write_json(args.progress, value), + args.retry_failed, + ) + except (OSError, ValueError, ApiError, RuntimeError) as exc: + print(f"错误：{exc}") + return 2 + + counts = { + status: sum(item["status"] == status for item in state["keywords"]) + for status in VALID_STATUSES + } + print( + f"成功 {counts['succeeded']}，失败 {counts['failed']}，" + f"待处理 {counts['pending']}，需要人工确认 {counts['needs_review']}" + ) + return result + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **步骤3：编写操作文档** + +创建 `README.md`： + +```markdown +# 关键词自动采集队列 + +逐个读取课题词表中“建议首先执行的20个词”，调用本机 MediaCrawler API，关闭评论并保存断点进度。 + +## 前置条件 + +1. Chrome 已开启远程调试，`127.0.0.1:9222` 可用。 +2. MediaCrawler API 已启动： + + ```powershell + cd D:\red_note_rich_search\MediaCrawler + uv run uvicorn api.main:app --port 8080 + ``` + +3. 确认当前没有其他爬虫任务。 + +## 首次运行或继续未完成任务 + +```powershell +cd D:\red_note_rich_search\MediaCrawler +uv run python ..\keyword_crawl_runner\run_keywords.py +``` + +程序自动读取： + +```text +C:\Users\15377\Documents\民生银行课题\首批可立即使用的搜索词.md +``` + +进度保存在 `crawl_progress.json`。再次执行会跳过已经成功的关键词。 + +## 重试明确失败项 + +```powershell +uv run python ..\keyword_crawl_runner\run_keywords.py --retry-failed +``` + +`needs_review` 不会自动重跑。先检查 MediaCrawler 输出和进度文件，再手工决定改为 `succeeded` 或 `failed`。 + +## 输出 + +笔记仍由 MediaCrawler 写入： + +```text +D:\red_note_rich_search\MediaCrawler\data\xhs\json +``` + +后续分析按 `(note_id, source_keyword)` 去重。 +``` + +- [ ] **步骤4：运行全部自动化验证** + +独立队列程序仓库： + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +python -m unittest -v +``` + +预期：不连接 MediaCrawler 或小红书，队列程序测试全部通过。 + +MediaCrawler 仓库： + +```powershell +cd D:\red_note_rich_search\MediaCrawler +uv run pytest tests/test_crawler_task_status.py tests/test_api_limits.py -q +uv run pytest tests -q +``` + +预期：聚焦测试及完整测试套件全部通过。 + +- [ ] **步骤5：执行受控的两关键词冒烟测试** + +使用真实的20关键词词表和专用冒烟测试进度文件。运行： + +```powershell +cd D:\red_note_rich_search\MediaCrawler +uv run uvicorn api.main:app --port 8080 +``` + +在第二个终端中： + +```powershell +uv run python ..\keyword_crawl_runner\run_keywords.py --progress D:\red_note_rich_search\keyword_crawl_runner\smoke_progress.json +``` + +第一个关键词成功后的5秒间隔内按 `Ctrl+C`。再次运行相同命令，程序必须跳过第一个关键词并开始第二个；第二个关键词成功后的间隔内再次按 `Ctrl+C`。 + +验证： + +- Two keywords reach terminal states. +- 两次 API 请求均关闭评论，并设置 `max_notes_count=20`。 +- 重启队列程序后跳过第一个已完成关键词。 +- MediaCrawler 输出记录包含匹配的 `source_keyword`。 + +验证后，可以保留被忽略的 `smoke_progress.json`，或仅删除该冒烟测试进度文件；不要改动真实的 `crawl_progress.json`。 + +- [ ] **步骤6：提交队列程序命令行和文档** + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +git add run_keywords.py test_run_keywords.py README.md +git commit -m "docs: add keyword runner operating guide" +``` + +- [ ] **步骤7：记录两个仓库的最终状态** + +```powershell +git -C D:\red_note_rich_search\MediaCrawler status --short +git -C D:\red_note_rich_search\MediaCrawler log -2 --oneline +git -C D:\red_note_rich_search\keyword_crawl_runner status --short +git -C D:\red_note_rich_search\keyword_crawl_runner log -2 --oneline +``` + +预期： + +- MediaCrawler 仅显示用户原有的未跟踪讨论文档。 +- 队列程序仓库状态干净，`crawl_progress.json` 仍被忽略。 +- 两个仓库都显示本计划对应的任务提交。 diff --git a/docs/superpowers/plans/2026-07-27-keyword-json-cdp-resilience.md b/docs/superpowers/plans/2026-07-27-keyword-json-cdp-resilience.md new file mode 100644 index 000000000..a7e92ea76 --- /dev/null +++ b/docs/superpowers/plans/2026-07-27-keyword-json-cdp-resilience.md @@ -0,0 +1,726 @@ +# 关键词 JSON 队列与 Chrome 稳定连接实施计划 + +> **供智能体执行：** 必须使用 `subagent-driven-development`（推荐）或 `executing-plans`，逐任务实施并在每个任务后评审。所有步骤使用复选框跟踪。 + +**目标：** 让外部关键词工具读取任意数量的 JSON 关键词并独立断点续跑，同时让 MediaCrawler 复用专用 9222 Chrome，并为小红书验证码保留5分钟人工处理窗口。 + +**架构：** `D:\red_note_rich_search\keyword_crawl_runner` 只负责输入校验、API 编排、进度和 Chrome 启动脚本；`D:\red_note_rich_search\MediaCrawler` 保留爬虫实现，修改 CDP 连接顺序和小红书请求错误处理。两者继续通过现有本地 HTTP API 通信，不复制爬虫逻辑。 + +**技术栈：** Python 3.12 标准库、`unittest`、PowerShell 5+、MediaCrawler 现有 `pytest`/`pytest-asyncio`、Playwright、httpx、tenacity。 + +## 全局约束 + +- 关键词严格串行；每词最多20条；一级、二级评论关闭；保存格式为 JSON。 +- JSON 关键词数量不限；去首尾空白、忽略空字符串、按首次出现顺序去重。 +- 不新增第三方依赖，不实现三词并行、AI 粗筛、用户 ID、数据库或验证码自动化。 +- 仅 `461/471` 进入人工等待；固定每10秒重试，最长5分钟。 +- 保留 `pending/running/succeeded/failed/needs_review` 语义；`--retry-failed` 只重置明确失败项，不自动重跑不确定任务。 +- 不修改或提交 `D:\red_note_rich_search\MediaCrawler\docs\公开账号索引研究模式改造方向.md`。 +- 两个仓库分别提交；每个任务先看到失败测试，再写最小实现。 + +--- + +## 文件结构 + +### 外部工具仓库 + +- 修改 `D:\red_note_rich_search\keyword_crawl_runner\run_keywords.py`：JSON 输入、必填 `--source`、默认进度路径。 +- 修改 `D:\red_note_rich_search\keyword_crawl_runner\test_run_keywords.py`：JSON 与路径回归测试。 +- 创建 `D:\red_note_rich_search\keyword_crawl_runner\start_chrome.ps1`：专用 Chrome 启动器。 +- 修改 `D:\red_note_rich_search\keyword_crawl_runner\.gitignore`：忽略输入、进度和 Chrome 数据。 +- 修改 `D:\red_note_rich_search\keyword_crawl_runner\README.md`：新命令与启动顺序。 + +### MediaCrawler 仓库 + +- 修改 `D:\red_note_rich_search\MediaCrawler\tools\cdp_browser.py`：HTTP 发现优先、direct CDP 回退。 +- 修改 `D:\red_note_rich_search\MediaCrawler\tests\test_cdp_browser.py`：连接顺序测试。 +- 修改 `D:\red_note_rich_search\MediaCrawler\media_platform\xhs\exception.py`：验证码专用异常。 +- 修改 `D:\red_note_rich_search\MediaCrawler\media_platform\xhs\client.py`：验证码分类和有界人工等待。 +- 创建 `D:\red_note_rich_search\MediaCrawler\tests\test_xhs_captcha_wait.py`：可控时钟测试。 + +--- + +### Task 1：JSON 输入与独立进度文件 + +**文件：** + +- 修改：`D:\red_note_rich_search\keyword_crawl_runner\run_keywords.py` +- 修改：`D:\red_note_rich_search\keyword_crawl_runner\test_run_keywords.py` +- 修改：`D:\red_note_rich_search\keyword_crawl_runner\.gitignore` +- 修改：`D:\red_note_rich_search\keyword_crawl_runner\README.md` + +**接口：** + +- 产出：`load_keywords(source: Path) -> list[str]` +- 产出：`default_progress_path(source: Path) -> Path` +- 产出：`parse_args(argv)` 中必填的 `--source` 和可空的 `--progress` +- 保持：`load_progress(path, source, keywords)`、`run_queue(...)` 及 API 请求体不变 + +- [ ] **Step 1：把解析测试改为 JSON 测试** + +在 `test_run_keywords.py` 中删除 Markdown 构造器和固定20词断言，导入 `load_keywords`、`default_progress_path`，加入： + +```python +class ParsingTests(unittest.TestCase): + def write_json(self, directory, value): + path = Path(directory) / "batch.json" + path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8") + return path + + def test_loads_any_number_and_deduplicates_in_order(self): + with tempfile.TemporaryDirectory() as directory: + source = self.write_json( + directory, + {"keywords": [" 词一 ", "", "词二", "词一", "词三"]}, + ) + self.assertEqual(load_keywords(source), ["词一", "词二", "词三"]) + + def test_rejects_invalid_keyword_documents(self): + invalid_values = [ + [], {}, {"keywords": "词一"}, + {"keywords": ["词一", 2]}, {"keywords": [" ", ""]}, + ] + for value in invalid_values: + with self.subTest(value=value), tempfile.TemporaryDirectory() as directory: + source = self.write_json(directory, value) + with self.assertRaises(ValueError): + load_keywords(source) + + def test_default_progress_uses_source_stem(self): + path = default_progress_path(Path("inputs/重庆客户.json")) + self.assertEqual(path.name, "重庆客户.progress.json") + self.assertEqual(path.parent.name, "progress") +``` + +补充 CLI 测试，确认缺少 `--source` 返回 argparse 的退出码2；将现有缺失源文件名改为 `.json`。 + +- [ ] **Step 2：运行失败测试** + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +python -m unittest -v test_run_keywords.ParsingTests test_run_keywords.CliTests +``` + +预期：导入 `load_keywords` 或 `default_progress_path` 失败。 + +- [ ] **Step 3：实现最小 JSON 解析和路径选择** + +在 `run_keywords.py` 中删除 `SECTION`、`DEFAULT_SOURCE`、`DEFAULT_PROGRESS` 和 `extract_keywords`，加入： + +```python +TOOL_ROOT = Path(__file__).resolve().parent +DEFAULT_PROGRESS_DIR = TOOL_ROOT / "progress" + + +def parse_args(argv: list[str] | None = None): + parser = argparse.ArgumentParser(description="逐词调用 MediaCrawler API") + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--api-base", default="http://127.0.0.1:8080/api") + parser.add_argument("--progress", type=Path) + parser.add_argument("--retry-failed", action="store_true") + return parser.parse_args(argv) + + +def load_keywords(source: Path) -> list[str]: + try: + value = json.loads(source.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"关键词 JSON 无法解析: {exc}") from exc + if not isinstance(value, dict) or not isinstance(value.get("keywords"), list): + raise ValueError("关键词 JSON 必须包含 keywords 数组") + raw_keywords = value["keywords"] + if any(not isinstance(keyword, str) for keyword in raw_keywords): + raise ValueError("keywords 数组元素必须全部为字符串") + keywords = list(dict.fromkeys( + keyword.strip() for keyword in raw_keywords if keyword.strip() + )) + if not keywords: + raise ValueError("keywords 数组没有有效关键词") + return keywords + + +def default_progress_path(source: Path) -> Path: + return DEFAULT_PROGRESS_DIR / f"{source.stem}.progress.json" +``` + +在 `main` 中统一使用： + +```python +keywords = load_keywords(args.source) +progress_path = args.progress or default_progress_path(args.source) +state = load_progress(progress_path, args.source, keywords) +result = run_queue( + state, + api, + lambda value: atomic_write_json(progress_path, value), + args.retry_failed, +) +``` + +`.gitignore` 增加： + +```gitignore +/inputs/ +/progress/ +/chrome_crawler_profile/ +``` + +README 加入以下输入说明，并删除旧 Markdown 示例： + +````markdown +## 关键词 JSON + +创建 `inputs\keywords.json`： + +```json +{ + "keywords": ["重庆企业主资产配置", "重庆公私资产隔离"] +} +``` + +运行： + +```powershell +uv run python run_keywords.py --source .\inputs\keywords.json +``` + +程序按输入文件名保存 `progress\keywords.progress.json`。同名 JSON 的关键词内容发生变化时，程序拒绝混用旧进度；请更换输入文件名，或用 `--progress` 指定新文件。 +```` + +- [ ] **Step 4：运行 runner 全套测试** + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +python -m unittest -v +``` + +预期：全部通过且无失败；现有 `running` 恢复、`failed` 重试和 `needs_review` 防重复测试继续通过。 + +- [ ] **Step 5：提交 Task 1** + +```powershell +git add -- run_keywords.py test_run_keywords.py .gitignore README.md +git commit -m "feat: accept JSON keyword batches" +``` + +--- + +### Task 2：专用 Chrome 一键启动脚本 + +**文件：** + +- 创建：`D:\red_note_rich_search\keyword_crawl_runner\start_chrome.ps1` +- 修改：`D:\red_note_rich_search\keyword_crawl_runner\README.md` + +**接口：** + +- 产出：`start_chrome.ps1 [-ProfileDir D:\red_note_rich_search\chrome_crawler_profile] [-Port 9222]` +- 依赖：Chrome `/json/version` 返回非空 `webSocketDebuggerUrl` + +- [ ] **Step 1：创建无第三方依赖的启动脚本** + +```powershell +param( + [string]$ProfileDir = 'D:\red_note_rich_search\chrome_crawler_profile', + [int]$Port = 9222 +) + +$ErrorActionPreference = 'Stop' +$versionUri = "http://127.0.0.1:$Port/json/version" + +function Test-CrawlerChrome { + try { + $version = Invoke-RestMethod -Uri $versionUri -TimeoutSec 2 + return [bool]$version.webSocketDebuggerUrl + } catch { + return $false + } +} + +if (Test-CrawlerChrome) { + Write-Host "专用 Chrome 已在 $Port 端口运行。" + exit 0 +} + +$listener = Get-NetTCPConnection -LocalPort $Port -State Listen -ErrorAction SilentlyContinue +if ($listener) { + throw "端口 $Port 已被占用，但不是可用的 Chrome 调试接口。" +} + +$candidates = @( + "$env:ProgramFiles\Google\Chrome\Application\chrome.exe", + "${env:ProgramFiles(x86)}\Google\Chrome\Application\chrome.exe", + "$env:LOCALAPPDATA\Google\Chrome\Application\chrome.exe" +) +$chrome = $candidates | Where-Object { Test-Path -LiteralPath $_ } | Select-Object -First 1 +if (-not $chrome) { throw '找不到 chrome.exe，请先安装 Google Chrome。' } + +New-Item -ItemType Directory -Force -Path $ProfileDir | Out-Null +Start-Process -FilePath $chrome -ArgumentList @( + "--remote-debugging-port=$Port", + "--user-data-dir=$ProfileDir", + 'https://www.xiaohongshu.com/explore' +) + +for ($attempt = 0; $attempt -lt 30; $attempt++) { + if (Test-CrawlerChrome) { + Write-Host "专用 Chrome 已启动。首次使用请在浏览器中登录小红书。" + exit 0 + } + Start-Sleep -Milliseconds 500 +} +throw "Chrome 已启动，但 $versionUri 在15秒内不可用。" +``` + +- [ ] **Step 2：执行 PowerShell 语法检查** + +```powershell +$tokens = $null +$errors = $null +[void][System.Management.Automation.Language.Parser]::ParseFile( + 'D:\red_note_rich_search\keyword_crawl_runner\start_chrome.ps1', + [ref]$tokens, + [ref]$errors +) +if ($errors.Count) { $errors | Format-List; exit 1 } +``` + +预期：退出码0且无语法错误。 + +- [ ] **Step 3：更新 README 的运行顺序** + +README 加入以下运行顺序： + +````markdown +## 完整运行顺序 + +```powershell +.\start_chrome.ps1 +``` + +首次使用时，在打开的专用 Chrome 中登录小红书。脚本不会关闭普通 Chrome；如果 9222 已经是可用的 Chrome 调试接口，脚本只提示已运行。 + +另开终端启动 API： + +```powershell +cd D:\red_note_rich_search\MediaCrawler +uv run uvicorn api.main:app --port 8080 +``` + +再开一个终端运行队列： + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +uv run python run_keywords.py --source .\inputs\keywords.json +``` +```` + +- [ ] **Step 4：人工验证幂等启动** + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +.\start_chrome.ps1 +.\start_chrome.ps1 +Invoke-RestMethod http://127.0.0.1:9222/json/version | Select-Object webSocketDebuggerUrl +``` + +预期：第一次打开专用 Chrome；第二次仅提示已运行；最后输出非空 WebSocket URL。 + +- [ ] **Step 5：提交 Task 2** + +```powershell +git add -- start_chrome.ps1 README.md +git commit -m "feat: add dedicated Chrome launcher" +``` + +--- + +### Task 3：MediaCrawler 优先连接 `/json/version` + +**文件：** + +- 修改：`D:\red_note_rich_search\MediaCrawler\tests\test_cdp_browser.py` +- 修改：`D:\red_note_rich_search\MediaCrawler\tools\cdp_browser.py` + +**接口：** + +- 消费：`CDPBrowserManager._get_browser_websocket_url(debug_port: int) -> str` +- 保持：`CDPBrowserManager._connect_via_cdp(playwright)` 的调用方不变 +- 产出：已有浏览器模式先使用发现 URL，发现失败才使用默认的 `ws://localhost:9222/devtools/browser`（端口取运行时配置） + +- [ ] **Step 1：把现有连接顺序测试改为新规则** + +```python +@pytest.mark.asyncio +async def test_existing_browser_prefers_discovered_websocket_url(monkeypatch): + monkeypatch.setattr(config, "CDP_CONNECT_EXISTING", True) + monkeypatch.setattr(config, "BROWSER_LAUNCH_TIMEOUT", 60) + manager = CDPBrowserManager() + manager.debug_port = 9222 + manager._get_browser_websocket_url = AsyncMock( + return_value="ws://localhost:9222/devtools/browser/generated-id" + ) + browser = MagicMock() + browser.is_connected.return_value = True + browser.contexts = [] + playwright = MagicMock() + playwright.chromium.connect_over_cdp = AsyncMock(return_value=browser) + + await manager._connect_via_cdp(playwright) + + manager._get_browser_websocket_url.assert_awaited_once_with(9222) + playwright.chromium.connect_over_cdp.assert_awaited_once_with( + "ws://localhost:9222/devtools/browser/generated-id", timeout=60000 + ) + + +@pytest.mark.asyncio +async def test_existing_browser_falls_back_to_direct_when_discovery_fails(monkeypatch): + monkeypatch.setattr(config, "CDP_CONNECT_EXISTING", True) + monkeypatch.setattr(config, "BROWSER_LAUNCH_TIMEOUT", 60) + manager = CDPBrowserManager() + manager.debug_port = 9222 + manager._get_browser_websocket_url = AsyncMock(side_effect=RuntimeError("no endpoint")) + browser = MagicMock() + browser.is_connected.return_value = True + browser.contexts = [] + playwright = MagicMock() + playwright.chromium.connect_over_cdp = AsyncMock(return_value=browser) + + await manager._connect_via_cdp(playwright) + + playwright.chromium.connect_over_cdp.assert_awaited_once_with( + "ws://localhost:9222/devtools/browser", timeout=60000 + ) +``` + +- [ ] **Step 2：运行失败测试** + +```powershell +cd D:\red_note_rich_search\MediaCrawler +uv run pytest tests/test_cdp_browser.py -q +``` + +预期：第一个测试失败，因为当前实现先 direct；第二个测试失败，因为当前 discovery 调用发生在 direct 失败之后。 + +- [ ] **Step 3：最小调整 `_connect_via_cdp`** + +```python +if config.CDP_CONNECT_EXISTING: + try: + ws_url = await self._get_browser_websocket_url(self.debug_port) + utils.logger.info( + f"[CDPBrowserManager] Connecting via discovered CDP URL: {ws_url}" + ) + except Exception as discovery_error: + ws_url = f"ws://localhost:{self.debug_port}/devtools/browser" + utils.logger.warning( + "[CDPBrowserManager] /json/version unavailable; falling back to " + "Chrome 136+ direct CDP mode, which may require browser approval: " + f"{discovery_error}" + ) + self.browser = await playwright.chromium.connect_over_cdp( + ws_url, timeout=config.BROWSER_LAUNCH_TIMEOUT * 1000 + ) +``` + +不要改变 `CDP_CONNECT_EXISTING = False` 的自动启动浏览器分支。 + +- [ ] **Step 4：运行 CDP 与 API 聚焦测试** + +```powershell +uv run pytest tests/test_cdp_browser.py tests/test_crawler_task_status.py tests/test_api_limits.py -q +``` + +预期：全部通过。 + +- [ ] **Step 5：提交 Task 3** + +```powershell +git add -- tests/test_cdp_browser.py tools/cdp_browser.py +git commit -m "fix: prefer discovered CDP websocket" +``` + +--- + +### Task 4：小红书验证码5分钟人工等待 + +**文件：** + +- 修改：`D:\red_note_rich_search\MediaCrawler\media_platform\xhs\exception.py` +- 修改：`D:\red_note_rich_search\MediaCrawler\media_platform\xhs\client.py` +- 创建：`D:\red_note_rich_search\MediaCrawler\tests\test_xhs_captcha_wait.py` + +**接口：** + +- 产出：`CaptchaRequiredError` +- 产出：`raise_for_captcha(response) -> None` +- 产出：`wait_for_captcha(operation, sleep=asyncio.sleep, monotonic=time.monotonic)` +- 保持：`XiaoHongShuClient.request(method, url, **kwargs)` 的公开签名和返回值不变 + +- [ ] **Step 1：写验证码分类和等待的失败测试** + +```python +from unittest.mock import AsyncMock + +import httpx +import pytest + +from media_platform.xhs.client import raise_for_captcha, wait_for_captcha +from media_platform.xhs.exception import CaptchaRequiredError + + +async def record_sleep(values, seconds): + values.append(seconds) + + +@pytest.mark.parametrize("status", [461, 471]) +def test_captcha_status_raises_specific_error(status): + response = httpx.Response( + status, + headers={"Verifytype": "slider", "Verifyuuid": "uuid-1"}, + request=httpx.Request("GET", "https://example.test"), + ) + with pytest.raises(CaptchaRequiredError, match="slider"): + raise_for_captcha(response) + + +@pytest.mark.asyncio +async def test_wait_for_captcha_continues_after_manual_verification(): + operation = AsyncMock( + side_effect=[CaptchaRequiredError("verify"), {"notes": []}] + ) + sleeps = [] + times = iter([0.0, 1.0]) + result = await wait_for_captcha( + operation, + sleep=lambda seconds: record_sleep(sleeps, seconds), + monotonic=lambda: next(times), + ) + assert result == {"notes": []} + assert sleeps == [10] + + +@pytest.mark.asyncio +async def test_wait_for_captcha_stops_after_five_minutes(): + operation = AsyncMock(side_effect=CaptchaRequiredError("verify")) + times = iter([0.0, 300.0]) + with pytest.raises(CaptchaRequiredError): + await wait_for_captcha( + operation, sleep=AsyncMock(), monotonic=lambda: next(times) + ) + + +@pytest.mark.asyncio +async def test_wait_for_captcha_does_not_retry_other_errors(): + operation = AsyncMock(side_effect=RuntimeError("network")) + sleep = AsyncMock() + with pytest.raises(RuntimeError, match="network"): + await wait_for_captcha(operation, sleep=sleep) + sleep.assert_not_awaited() +``` + +- [ ] **Step 2：运行失败测试** + +```powershell +cd D:\red_note_rich_search\MediaCrawler +uv run pytest tests/test_xhs_captcha_wait.py -q +``` + +预期：导入新增异常或函数失败。 + +- [ ] **Step 3：实现专用异常和纯辅助函数** + +在 `exception.py` 增加： + +```python +class CaptchaRequiredError(RuntimeError): + """Xiaohongshu requires the user to complete CAPTCHA manually.""" +``` + +在 `client.py` 导入 `time`、`Awaitable`、`CaptchaRequiredError`，加入： + +```python +CAPTCHA_RETRY_SECONDS = 10 +CAPTCHA_MAX_WAIT_SECONDS = 300 + + +def raise_for_captcha(response: httpx.Response) -> None: + if response.status_code not in {461, 471}: + return + verify_type = response.headers.get("Verifytype", "unknown") + verify_uuid = response.headers.get("Verifyuuid", "unknown") + raise CaptchaRequiredError( + "CAPTCHA appeared; please verify manually in Chrome. " + f"Verifytype: {verify_type}, Verifyuuid: {verify_uuid}" + ) + + +async def wait_for_captcha( + operation: Callable[[], Awaitable[Any]], + *, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + monotonic: Callable[[], float] = time.monotonic, +) -> Any: + deadline = monotonic() + CAPTCHA_MAX_WAIT_SECONDS + while True: + try: + return await operation() + except CaptchaRequiredError: + remaining = deadline - monotonic() + if remaining <= 0: + raise + utils.logger.warning( + "[XiaoHongShuClient] CAPTCHA appeared; complete verification " + "in Chrome within 5 minutes." + ) + await sleep(min(CAPTCHA_RETRY_SECONDS, remaining)) +``` + +- [ ] **Step 4：把现有短重试包在验证码等待内部** + +将当前 `request` 的实际请求和响应解析移动到私有 `_request_with_short_retry`，保留原三次短重试，但排除验证码： + +```python +async def request(self, method, url, **kwargs) -> Union[str, Any]: + return_response = kwargs.pop("return_response", False) + + async def operation(): + return await self._request_with_short_retry( + method, url, return_response=return_response, **kwargs + ) + + return await wait_for_captcha(operation) + + +@retry( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + retry=retry_if_not_exception_type((NoteNotFoundError, CaptchaRequiredError)), + reraise=True, +) +async def _request_with_short_retry( + self, method, url, *, return_response=False, **kwargs +) -> Union[str, Any]: + await self._refresh_proxy_if_expired() + async with make_async_client(proxy=self.proxy) as client: + response = await client.request(method, url, timeout=self.timeout, **kwargs) + raise_for_captcha(response) + if return_response: + return response.text + data: Dict = response.json() + if data["success"]: + return data.get("data", data.get("success", {})) + if data["code"] == self.IP_ERROR_CODE: + raise IPBlockError(self.IP_ERROR_STR) + if data["code"] in (self.NOTE_NOT_FOUND_CODE, self.NOTE_ABNORMAL_CODE): + raise NoteNotFoundError( + f"Note not found or abnormal, code: {data['code']}" + ) + err_msg = data.get("msg", None) or response.text + raise DataFetchError(err_msg) +``` + +删除旧的裸 `raise Exception(msg)`；除移动到私有方法外，不改变上述数据分支。 + +- [ ] **Step 5：运行验证码与小红书聚焦测试** + +```powershell +uv run pytest tests/test_xhs_captcha_wait.py tests/test_no_user_info.py -q +``` + +预期：全部通过，测试过程不真实等待。 + +- [ ] **Step 6：提交 Task 4** + +```powershell +git add -- media_platform/xhs/exception.py media_platform/xhs/client.py tests/test_xhs_captcha_wait.py +git commit -m "fix: wait for manual XHS CAPTCHA" +``` + +--- + +### Task 5：双仓库回归、人工冒烟与发布准备 + +**文件：** 验证 Task 1–4 的全部改动，不新增生产接口。 + +- [ ] **Step 1：运行外部工具完整测试和差异检查** + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +python -m unittest -v +git diff --check HEAD~2..HEAD +git status -sb +``` + +预期：全部测试通过，差异检查退出码0。 + +- [ ] **Step 2：运行 MediaCrawler 聚焦测试** + +```powershell +cd D:\red_note_rich_search\MediaCrawler +uv run pytest tests/test_cdp_browser.py tests/test_xhs_captcha_wait.py tests/test_crawler_task_status.py tests/test_api_limits.py tests/test_no_user_info.py -q +git diff --check HEAD~2..HEAD +``` + +预期：全部通过；不得把既有 Windows `grep` 兼容失败混入该聚焦结果。 + +- [ ] **Step 3：创建被忽略的最小人工验收输入** + +文件 `D:\red_note_rich_search\keyword_crawl_runner\inputs\smoke.json`： + +```json +{ + "keywords": [ + "重庆企业主资产配置", + "重庆公私资产隔离" + ] +} +``` + +- [ ] **Step 4：执行真实本地冒烟** + +终端一： + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +.\start_chrome.ps1 +``` + +终端二： + +```powershell +cd D:\red_note_rich_search\MediaCrawler +uv run uvicorn api.main:app --port 8080 +``` + +终端三： + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +uv run python run_keywords.py --source .\inputs\smoke.json +``` + +预期：两个关键词串行完成；第二个子进程不再要求 Chrome CDP 授权；生成 `progress\smoke.progress.json`；输出仍位于 `MediaCrawler\data\xhs\json`。 + +- [ ] **Step 5：验证断点与进度隔离** + +再次运行同一命令，确认成功词不重新提交。复制输入为 `inputs\smoke-copy.json` 后运行，确认使用 `progress\smoke-copy.progress.json`。修改原 `smoke.json` 后运行，确认程序拒绝复用 `smoke.progress.json`。 + +- [ ] **Step 6：确认两个仓库提交范围** + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +git status -sb +git log -3 --oneline + +cd D:\red_note_rich_search\MediaCrawler +git status -sb +git log -4 --oneline +``` + +预期：实现均已提交；MediaCrawler 中用户未跟踪的研究文档仍未加入；不提交 Chrome profile、关键词输入、进度或采集结果。 + +- [ ] **Step 7：进入完成分支流程** + +使用 `finishing-a-development-branch` 检查测试证据和提交范围。外部工具远程为 `yukikojo/mediaClawerTooler`，只有在用户确认发布后才推送。MediaCrawler 本地 `main` 与上游存在分歧，不执行强制推送或历史重写。 diff --git a/docs/superpowers/plans/2026-07-31-xhs-creator-id-pipeline.md b/docs/superpowers/plans/2026-07-31-xhs-creator-id-pipeline.md new file mode 100644 index 000000000..895244976 --- /dev/null +++ b/docs/superpowers/plans/2026-07-31-xhs-creator-id-pipeline.md @@ -0,0 +1,653 @@ +# XHS Creator ID Pipeline Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Capture XHS author IDs during the initial keyword crawl, attach IDs only to strong/medium AI candidates, and optionally crawl those candidates with the existing creator mode. + +**Architecture:** MediaCrawler gains an opt-in, local sidecar capture before anonymous storage. The keyword runner enables that option. Screening remains anonymous until all model responses are validated, then joins qualified author hashes to the local sidecar; an explicit flag submits each deduplicated ID to the existing creator API. + +**Tech Stack:** Python 3.11, asyncio, Typer, FastAPI/Pydantic, standard-library JSON/urllib, pytest and unittest. + +## Global Constraints + +- `store/xhs/__init__.py` remains unchanged and ordinary content storage remains anonymous. +- Only `potential_customer` notes with `strong` or `medium` evidence and non-empty evidence quotes qualify. +- Raw IDs never enter AI prompts, screening cache, `cleaned_notes.jsonl`, or `screened_notes.jsonl`. +- ID capture is off by default and is enabled only by the keyword research runner. +- No new dependency, database, WebUI, contact extraction, cross-platform matching, or marketing automation. +- Creator comments and media downloads stay disabled; `--creator-max-notes` must be positive. +- Preserve unrelated changes in both repositories, especially `keyword_crawl_runner/docs/superpowers/plans/2026-07-29-xhs-ai-screening.md` and `keyword_crawl_runner/results/`. + +--- + +### Task 1: Opt-in ID sidecar capture in MediaCrawler + +**Files:** +- Create: `tools/xhs_creator_id_capture.py` +- Create: `tests/test_xhs_creator_id_capture.py` +- Modify: `config/base_config.py` +- Modify: `media_platform/xhs/core.py:170-174` +- Modify: `.gitignore` + +**Interfaces:** +- Consumes: raw XHS `note_detail` dictionaries returned by `get_note_detail_async_task()`. +- Produces: `build_capture_record(note_detail, captured_at=None) -> dict | None` and `capture_creator_id(note_detail) -> None`. +- Produces local files at `data/xhs/private/search_creator_ids_YYYY-MM-DD.json`; files contain only `note_id`, `creator_hash`, `public_user_id`, `profile_url`, and `captured_at`. + +- [ ] **Step 1: Write failing unit tests for safe records and idempotent persistence** + +```python +# tests/test_xhs_creator_id_capture.py +import json + +import pytest + +import config +from tools.xhs_creator_id_capture import build_capture_record, capture_creator_id + + +VALID_ID = "5eb8e1d400000000010075ae" + + +def test_build_capture_record_contains_only_allowed_fields(): + record = build_capture_record({ + "note_id": "note-1", + "user": { + "user_id": VALID_ID, + "nickname": "不得保存", + "avatar": "https://private.invalid/avatar", + }, + "xsec_token": "secret-token", + }, captured_at="2026-07-31T10:00:00+08:00") + + assert record == { + "note_id": "note-1", + "creator_hash": "fb9185b3a79e6291", + "public_user_id": VALID_ID, + "profile_url": f"https://www.xiaohongshu.com/user/profile/{VALID_ID}", + "captured_at": "2026-07-31T10:00:00+08:00", + } + + +@pytest.mark.asyncio +async def test_capture_is_disabled_by_default_and_idempotent_when_enabled(tmp_path, monkeypatch): + monkeypatch.setattr(config, "ENABLE_XHS_CREATOR_ID_CAPTURE", False) + monkeypatch.setattr(config, "XHS_CREATOR_ID_CAPTURE_DIR", str(tmp_path)) + detail = {"note_id": "note-1", "user": {"user_id": VALID_ID}} + + await capture_creator_id(detail) + assert list(tmp_path.iterdir()) == [] + + monkeypatch.setattr(config, "ENABLE_XHS_CREATOR_ID_CAPTURE", True) + await capture_creator_id(detail) + await capture_creator_id(detail) + records = json.loads(next(tmp_path.iterdir()).read_text(encoding="utf-8")) + assert len(records) == 1 + assert records[0]["public_user_id"] == VALID_ID + + +def test_invalid_user_id_is_not_captured(): + assert build_capture_record({"note_id": "note-1", "user": {"user_id": "short"}}) is None +``` + +- [ ] **Step 2: Run the tests and verify RED** + +Run: `uv run pytest tests/test_xhs_creator_id_capture.py -v` + +Expected: collection fails with `ModuleNotFoundError: No module named 'tools.xhs_creator_id_capture'`. + +- [ ] **Step 3: Add default-off configuration and minimal capture module** + +Add to `config/base_config.py`: + +```python +ENABLE_XHS_CREATOR_ID_CAPTURE = False +XHS_CREATOR_ID_CAPTURE_DIR = "" +``` + +Create `tools/xhs_creator_id_capture.py` with these concrete behaviors: + +```python +import asyncio +import json +import os +import re +import tempfile +from datetime import datetime +from pathlib import Path + +import config +from tools.user_hash import anonymize_user_id +from tools.utils import utils + +_USER_ID = re.compile(r"^[0-9a-f]{24}$") +_LOCK = asyncio.Lock() + + +def build_capture_record(note_detail: dict, captured_at: str | None = None) -> dict | None: + note_id = str(note_detail.get("note_id") or "").strip() + user_id = str((note_detail.get("user") or {}).get("user_id") or "").strip() + if not note_id or not _USER_ID.fullmatch(user_id): + return None + return { + "note_id": note_id, + "creator_hash": anonymize_user_id(user_id), + "public_user_id": user_id, + "profile_url": f"https://www.xiaohongshu.com/user/profile/{user_id}", + "captured_at": captured_at or datetime.now().astimezone().isoformat(timespec="seconds"), + } + + +def _capture_path() -> Path: + base = Path(config.XHS_CREATOR_ID_CAPTURE_DIR) if config.XHS_CREATOR_ID_CAPTURE_DIR else Path("data/xhs/private") + return base / f"search_creator_ids_{utils.get_current_date()}.json" + + +def _write_record(path: Path, record: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + records = json.loads(path.read_text(encoding="utf-8")) if path.exists() else [] + keyed = {(item["note_id"], item["public_user_id"]): item for item in records} + keyed[(record["note_id"], record["public_user_id"])] = record + fd, temporary_name = tempfile.mkstemp(dir=path.parent, prefix=path.name, suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as output: + json.dump(list(keyed.values()), output, ensure_ascii=False, indent=2) + os.replace(temporary_name, path) + finally: + if os.path.exists(temporary_name): + os.unlink(temporary_name) + + +async def capture_creator_id(note_detail: dict) -> None: + if not config.ENABLE_XHS_CREATOR_ID_CAPTURE: + return + record = build_capture_record(note_detail) + if record is None: + return + async with _LOCK: + await asyncio.to_thread(_write_record, _capture_path(), record) +``` + +Add `/data/xhs/private/` to `.gitignore`. + +- [ ] **Step 4: Hook capture before anonymous storage** + +In `media_platform/xhs/core.py`, import `capture_creator_id`, then change the search result loop to: + +```python +for note_detail in note_details: + if note_detail: + await capture_creator_id(note_detail) + await xhs_store.update_xhs_note(note_detail) + await self.get_notice_media(note_detail) + note_ids.append(note_detail.get("note_id")) + xsec_tokens.append(note_detail.get("xsec_token")) +``` + +- [ ] **Step 5: Run focused and anonymity tests** + +Run: `uv run pytest tests/test_xhs_creator_id_capture.py tests/test_no_user_info.py -v` + +Expected: all tests pass; no raw ID appears in ordinary XHS content storage. + +- [ ] **Step 6: Commit Task 1 in the MediaCrawler repository** + +```powershell +git add -- .gitignore config/base_config.py media_platform/xhs/core.py tools/xhs_creator_id_capture.py tests/test_xhs_creator_id_capture.py +git commit -m "feat: capture XHS creator ids during initial search" +``` + +### Task 2: Thread the capture switch through CLI, API, and keyword runner + +**Files:** +- Modify: `cmd_arg/arg.py` +- Modify: `api/schemas/crawler.py` +- Modify: `api/services/crawler_manager.py` +- Modify: `tests/test_api_limits.py` +- Modify: `../keyword_crawl_runner/run_keywords.py` +- Modify: `../keyword_crawl_runner/test_run_keywords.py` + +**Interfaces:** +- Consumes: API request field `capture_creator_ids: bool = False`. +- Produces: CLI option `--capture_creator_ids true|false`, assigned to `config.ENABLE_XHS_CREATOR_ID_CAPTURE`. +- Keyword runner search requests always set `capture_creator_ids` to `true`. + +- [ ] **Step 1: Write failing MediaCrawler propagation tests** + +Add to `tests/test_api_limits.py`: + +```python +@pytest.mark.asyncio +async def test_capture_creator_ids_cli_defaults_off_and_can_be_enabled(): + original = config.ENABLE_XHS_CREATOR_ID_CAPTURE + try: + await parse_cmd(["--platform", "xhs", "--capture_creator_ids", "true"]) + assert config.ENABLE_XHS_CREATOR_ID_CAPTURE is True + finally: + config.ENABLE_XHS_CREATOR_ID_CAPTURE = original + + +def test_crawler_manager_only_passes_capture_switch_when_enabled(): + manager = CrawlerManager() + disabled = manager._build_command(CrawlerStartRequest(platform=PlatformEnum.XHS)) + enabled = manager._build_command(CrawlerStartRequest( + platform=PlatformEnum.XHS, + capture_creator_ids=True, + )) + assert "--capture_creator_ids" not in disabled + index = enabled.index("--capture_creator_ids") + assert enabled[index + 1] == "true" +``` + +- [ ] **Step 2: Run MediaCrawler tests and verify RED** + +Run: `uv run pytest tests/test_api_limits.py -v` + +Expected: failures report the missing Pydantic field or missing CLI option. + +- [ ] **Step 3: Implement the MediaCrawler propagation** + +Add `capture_creator_ids: bool = False` to `CrawlerStartRequest`. Add a Typer string option named +`--capture_creator_ids`, normalize it with the existing `_to_bool()`, and assign the result to +`config.ENABLE_XHS_CREATOR_ID_CAPTURE`. In `CrawlerManager._build_command()` append: + +```python +if config.capture_creator_ids: + cmd.extend(["--capture_creator_ids", "true"]) +``` + +- [ ] **Step 4: Verify MediaCrawler GREEN** + +Run: `uv run pytest tests/test_api_limits.py tests/test_xhs_creator_id_capture.py -v` + +Expected: all tests pass. + +- [ ] **Step 5: Write the failing keyword-runner payload assertion** + +In `../keyword_crawl_runner/test_run_keywords.py`, extend +`test_start_payload_disables_comments_and_limits_notes` with: + +```python +self.assertTrue(payload["capture_creator_ids"]) +``` + +- [ ] **Step 6: Run the keyword-runner test and verify RED** + +Run from `D:\red_note_rich_search\keyword_crawl_runner`: +`python -m unittest test_run_keywords.HttpPayloadTests.test_start_payload_disables_comments_and_limits_notes -v` + +Expected: failure with `KeyError: 'capture_creator_ids'`. + +- [ ] **Step 7: Enable capture in keyword searches** + +Add this field to `CrawlerApi.start()`'s POST payload in `run_keywords.py`: + +```python +"capture_creator_ids": True, +``` + +- [ ] **Step 8: Verify and commit Task 2 in both repositories** + +Run MediaCrawler: `uv run pytest tests/test_api_limits.py -v` + +Run keyword runner: `python -m unittest test_run_keywords.HttpPayloadTests -v` + +Commit MediaCrawler: + +```powershell +git add -- cmd_arg/arg.py api/schemas/crawler.py api/services/crawler_manager.py tests/test_api_limits.py +git commit -m "feat: expose XHS creator id capture switch" +``` + +Commit keyword runner without staging unrelated files: + +```powershell +git add -- run_keywords.py test_run_keywords.py +git commit -m "feat: capture creator ids in keyword crawls" +``` + +### Task 3: Keep only strong/medium candidates and attach captured IDs + +**Files:** +- Modify: `../keyword_crawl_runner/screening/features.py` +- Create: `../keyword_crawl_runner/screening/creator_ids.py` +- Modify: `../keyword_crawl_runner/screen_data.py` +- Modify: `../keyword_crawl_runner/test_screening_features.py` +- Modify: `../keyword_crawl_runner/test_screen_data.py` + +**Interfaces:** +- Consumes: sidecar JSON arrays matching `search_creator_ids_*.json`. +- Produces: `load_creator_id_records(path: Path) -> list[dict]` and + `attach_creator_ids(authors: list[dict], records: list[dict]) -> tuple[list[dict], list[dict]]`. +- Produces: `candidate_creators.json` containing only qualified authors plus `public_user_id` and `profile_url`; `candidate_creator_errors.json` contains missing-map and hash-conflict records. + +- [ ] **Step 1: Change aggregation expectations first** + +Update `test_candidate_score_boundaries_are_assigned_to_tiers` to expect only authors `a` and `b`. +Replace provider-only, evidence-none, and weak-only tests with assertions that +`aggregate_authors(notes) == []`. These tests must fail because current aggregation emits C-tier authors. + +- [ ] **Step 2: Run feature tests and verify RED** + +Run from `keyword_crawl_runner`: +`python -m unittest test_screening_features.AggregateAuthorsTests -v` + +Expected: three failures showing unqualified authors are still returned. + +- [ ] **Step 3: Apply the minimal candidate filter** + +In `aggregate_authors()`, skip groups immediately after computing `qualified_notes`: + +```python +qualified_notes = [note for note in notes if _is_qualified_potential_customer(note)] +if not qualified_notes: + continue +``` + +- [ ] **Step 4: Add failing ID join tests** + +Add tests to `test_screen_data.py` that call the wished-for interface: + +```python +from screening.creator_ids import attach_creator_ids + + +def test_attach_creator_ids_links_unique_hash_and_reports_missing_or_conflict(): + authors = [ + {"creator_hash": "ok"}, + {"creator_hash": "missing"}, + {"creator_hash": "conflict"}, + ] + records = [ + {"creator_hash": "ok", "public_user_id": "5eb8e1d400000000010075ae"}, + {"creator_hash": "conflict", "public_user_id": "5eb8e1d400000000010075af"}, + {"creator_hash": "conflict", "public_user_id": "5eb8e1d400000000010075b0"}, + ] + + linked, errors = attach_creator_ids(authors, records) + + assert linked == [{ + "creator_hash": "ok", + "public_user_id": "5eb8e1d400000000010075ae", + "profile_url": "https://www.xiaohongshu.com/user/profile/5eb8e1d400000000010075ae", + }] + assert [item["error"] for item in errors] == ["missing_mapping", "hash_conflict"] +``` + +- [ ] **Step 5: Run the ID join test and verify RED** + +Run: `python -m unittest test_screen_data -v` + +Expected: import fails for missing `screening.creator_ids`. + +- [ ] **Step 6: Implement sidecar loading and collision-safe join** + +Create `screening/creator_ids.py` using only `json`, `re`, and `pathlib`. It must: + +```python +def load_creator_id_records(path: Path) -> list[dict]: + paths = [path] if path.is_file() else sorted(path.glob("search_creator_ids_*.json")) + records = [] + for source in paths: + value = json.loads(source.read_text(encoding="utf-8")) + if not isinstance(value, list): + raise ValueError(f"{source}: creator ID map must be an array") + records.extend(item for item in value if isinstance(item, dict)) + return records + + +def attach_creator_ids(authors: list[dict], records: list[dict]) -> tuple[list[dict], list[dict]]: + by_hash: dict[str, set[str]] = {} + for record in records: + creator_hash = record.get("creator_hash") + user_id = record.get("public_user_id") + if isinstance(creator_hash, str) and re.fullmatch(r"[0-9a-f]{24}", str(user_id)): + by_hash.setdefault(creator_hash, set()).add(user_id) + linked, errors = [], [] + for author in authors: + ids = sorted(by_hash.get(author["creator_hash"], set())) + if len(ids) != 1: + errors.append({ + "creator_hash": author["creator_hash"], + "error": "missing_mapping" if not ids else "hash_conflict", + }) + continue + user_id = ids[0] + linked.append({ + **author, + "public_user_id": user_id, + "profile_url": f"https://www.xiaohongshu.com/user/profile/{user_id}", + }) + return linked, errors +``` + +- [ ] **Step 7: Integrate the join after AI completion** + +Add `--creator-id-map` to `screen_data.parse_args()`, defaulting to +`../MediaCrawler/data/xhs/private`. Add `"creator_errors": "candidate_creator_errors.json"` to +`OUTPUT_FILES`. After `aggregate_authors(evidence_notes)`, load the sidecar and call +`attach_creator_ids()`. Write linked authors to `candidate_creators.json` and errors to the new file. +Do not add either value to `featured_notes`, `results`, the AI call, or `ScreeningCache`. + +Extend `_atomic_write()` with a keyword-only `sanitize: bool = True`. Keep the current sanitizer for +all existing outputs, but call it with `sanitize=False` only for `candidate_creators.json`, +`candidate_creator_errors.json`, and later `candidate_creator_crawl.json`. These three local files are +the intentional private research outputs; every value written to them must first be constructed by the +whitelisting functions in `screening.creator_ids.py` or the creator crawl status builder. + +- [ ] **Step 8: Update integration expectations and verify** + +Update `test_screen_data.py` fixtures to create a sidecar mapping for `author-1`; expect six output +files and assert raw IDs occur only in `candidate_creators.json`. Add one test where `screen_note()` +inspects its input and confirms `public_user_id` is absent. + +Run: +`python -m unittest test_screening_features test_screen_data -v` + +Expected: all tests pass. + +- [ ] **Step 9: Commit Task 3 in keyword_crawl_runner** + +```powershell +git add -- screening/features.py screening/creator_ids.py screen_data.py test_screening_features.py test_screen_data.py +git commit -m "feat: attach ids to strong and medium candidates" +``` + +### Task 4: Optional automatic creator crawl + +**Files:** +- Modify: `../keyword_crawl_runner/run_keywords.py` +- Modify: `../keyword_crawl_runner/screen_data.py` +- Modify: `../keyword_crawl_runner/test_run_keywords.py` +- Modify: `../keyword_crawl_runner/test_screen_data.py` +- Modify: `../keyword_crawl_runner/README.md` + +**Interfaces:** +- Produces: `CrawlerApi.start_creator(user_id: str, max_notes_count: int) -> str`. +- Consumes: `--crawl-creators`, `--crawler-api-base`, and `--creator-max-notes`. +- Produces: `candidate_creator_crawl.json` with one status record per deduplicated candidate ID. + +- [ ] **Step 1: Write failing creator API payload test** + +Add to `test_run_keywords.HttpPayloadTests`: + +```python +def test_start_creator_disables_comments_and_uses_one_public_id(self): + api = CrawlerApi("http://127.0.0.1:8080/api") + with patch.object(api, "_request", return_value={"task_id": "creator-task"}) as request_call: + task_id = api.start_creator("5eb8e1d400000000010075ae", 50) + assert task_id == "creator-task" + payload = request_call.call_args.args[2] + assert payload["crawler_type"] == "creator" + assert payload["creator_ids"] == "5eb8e1d400000000010075ae" + assert payload["max_notes_count"] == 50 + assert payload["enable_comments"] is False +``` + +- [ ] **Step 2: Run the test and verify RED** + +Run: `python -m unittest test_run_keywords.HttpPayloadTests.test_start_creator_disables_comments_and_uses_one_public_id -v` + +Expected: failure with missing `start_creator`. + +- [ ] **Step 3: Implement `start_creator` with the existing `_request` helper** + +```python +def start_creator(self, user_id: str, max_notes_count: int) -> str: + response = self._request("POST", "/crawler/start", { + "platform": "xhs", + "login_type": "qrcode", + "crawler_type": "creator", + "creator_ids": user_id, + "enable_comments": False, + "enable_sub_comments": False, + "save_option": "json", + "cookies": "", + "headless": False, + "max_notes_count": max_notes_count, + }) + task_id = response.get("task_id") + if not isinstance(task_id, str) or not task_id: + raise ApiError("启动响应缺少 task_id") + return task_id +``` + +- [ ] **Step 4: Write failing orchestration tests** + +Add tests to `test_screen_data.py` for a helper with this interface: + +```python +from screen_data import crawl_candidate_creators + + +def test_crawl_candidate_creators_deduplicates_and_continues_after_failure(): + creators = [ + {"public_user_id": "5eb8e1d400000000010075ae"}, + {"public_user_id": "5eb8e1d400000000010075ae"}, + {"public_user_id": "5eb8e1d400000000010075af"}, + ] + api = FakeCreatorApi(fail_ids={"5eb8e1d400000000010075ae"}) + statuses = crawl_candidate_creators(creators, api, 50, sleep=lambda _: None) + assert api.started == [ + "5eb8e1d400000000010075ae", + "5eb8e1d400000000010075af", + ] + assert [item["status"] for item in statuses] == ["failed", "succeeded"] +``` + +Also assert `run_screening()` never constructs `CrawlerApi` without `--crawl-creators`. + +- [ ] **Step 5: Run orchestration tests and verify RED** + +Run: `python -m unittest test_screen_data -v` + +Expected: import fails for missing `crawl_candidate_creators`. + +- [ ] **Step 6: Implement explicit creator crawling** + +Add CLI options: + +```python +parser.add_argument("--crawl-creators", action="store_true") +parser.add_argument("--crawler-api-base", default="http://127.0.0.1:8080/api") +parser.add_argument("--creator-max-notes", type=int, default=50) +``` + +Reject `creator_max_notes < 1`. Implement `crawl_candidate_creators()` to preserve input order, +deduplicate IDs, call `start_creator()` once per author, poll `api.status()` until the matching task is +idle, and catch `ApiError` per author. Return only: + +```python +{ + "public_user_id": user_id, + "task_id": task_id_or_none, + "status": "succeeded" | "failed" | "needs_review", + "error": safe_short_error_or_none, +} +``` + +When `args.crawl_creators` is true, call the helper only after all screening files are successfully +written, then atomically write `candidate_creator_crawl.json`. When false, do not create this file and +do not contact the crawler API. + +- [ ] **Step 7: Update README with the exact two-stage commands** + +Document: + +```powershell +python screen_data.py ` + --input ..\MediaCrawler\data\xhs\json ` + --output results\first_pass_20260731 ` + --creator-id-map ..\MediaCrawler\data\xhs\private ` + --crawl-creators ` + --creator-max-notes 50 + +python screen_data.py ` + --input ..\MediaCrawler\data\xhs\json ` + --pattern "creator_contents_*.json" ` + --output results\second_pass_20260731 +``` + +- [ ] **Step 8: Run keyword-runner regression tests** + +Run: `python -m unittest discover -v` + +Expected: all tests pass with no network calls from tests. + +- [ ] **Step 9: Commit Task 4 in keyword_crawl_runner** + +```powershell +git add -- run_keywords.py screen_data.py test_run_keywords.py test_screen_data.py README.md +git commit -m "feat: crawl qualified XHS creator candidates" +``` + +### Task 5: Cross-repository verification + +**Files:** +- Verify only; no new production files. + +**Interfaces:** +- Confirms the API contract and sidecar schema match across both repositories. + +- [ ] **Step 1: Run MediaCrawler focused regression** + +Run from `D:\red_note_rich_search\MediaCrawler`: + +```powershell +uv run pytest tests/test_xhs_creator_id_capture.py tests/test_api_limits.py tests/test_no_user_info.py tests/test_crawler_task_status.py -v +``` + +Expected: all selected tests pass. + +- [ ] **Step 2: Run keyword runner full suite** + +Run from `D:\red_note_rich_search\keyword_crawl_runner`: + +```powershell +python -m unittest discover -v +``` + +Expected: all tests pass. + +- [ ] **Step 3: Run static and diff checks** + +Run in each repository: + +```powershell +git diff --check +git status --short +``` + +Expected: no whitespace errors; only the user's pre-existing unrelated changes remain unstaged. + +- [ ] **Step 4: Perform one controlled manual smoke test** + +Start the existing MediaCrawler API, run one keyword through `run_keywords.py`, and verify: + +1. A new `data/xhs/private/search_creator_ids_YYYY-MM-DD.json` exists. +2. Ordinary search content still contains `creator_hash` and no raw `user_id`. +3. AI request inspection shows no raw ID. +4. `candidate_creators.json` contains IDs only for strong/medium candidates. +5. With `--crawl-creators`, creator notes are saved without requesting comments or media. + +Do not commit generated data or smoke-test results. diff --git a/docs/superpowers/specs/2026-07-24-public-account-id-extractor-design.md b/docs/superpowers/specs/2026-07-24-public-account-id-extractor-design.md new file mode 100644 index 000000000..5884b02d1 --- /dev/null +++ b/docs/superpowers/specs/2026-07-24-public-account-id-extractor-design.md @@ -0,0 +1,222 @@ +# 公开笔记作者 ID 提取器设计 + +> 日期：2026-07-24 +> +> 状态：已确认，待实施 +> 范围：阶段一，仅处理人工确认后的公开笔记 + +## 1. 目标 + +在 `D:\red_note_rich_search` 下新增一个独立的小工具，从人工审核通过的公开小红书笔记 JSON 中提取笔记作者的公开 `user_id`，并生成公开主页链接，供后续 MediaCrawler 创作者模式使用。 + +本阶段只打通以下链路： + +```text +人工审核后的笔记 JSON + -> 读取 accepted 记录 + -> 复用 MediaCrawler 获取笔记详情 + -> 提取公开作者 user_id + -> 生成公开主页链接 + -> 输出 JSON +``` + +## 2. 非目标 + +本阶段不实现： + +- AI 初筛或复筛。 +- 创作者全部笔记采集。 +- 目标账号数据库。 +- WebUI 审核页面。 +- 作者主页详情采集。 +- 评论用户 ID 采集。 +- 跨平台身份匹配、联系方式提取或自动营销触达。 +- 修改 MediaCrawler 当前匿名内容存储逻辑。 + +## 3. 目录与运行边界 + +新增独立目录： + +```text +D:\red_note_rich_search\public_account_id_extractor +├── extract_ids.py +├── reviewed_notes.json +├── extracted_ids.json +└── README.md +``` + +工具代码与输入、输出文件放在独立目录中，但运行时复用相邻 `MediaCrawler` 项目的代码和虚拟环境，不复制小红书登录、签名、Cookie、CDP 或请求实现。 + +推荐运行方式： + +```powershell +cd D:\red_note_rich_search\MediaCrawler + +uv run python ` + ..\public_account_id_extractor\extract_ids.py ` + --input ..\public_account_id_extractor\reviewed_notes.json ` + --output ..\public_account_id_extractor\extracted_ids.json +``` + +## 4. 输入格式 + +输入文件为 UTF-8 JSON 数组。每条记录至少包含： + +```json +[ + { + "note_id": "公开笔记 ID", + "note_url": "https://www.xiaohongshu.com/explore/{note_id}?xsec_token={token}&xsec_source=pc_search", + "source_keyword": "重庆企业主资产配置", + "review_status": "accepted", + "suspected_marketing": false + } +] +``` + +处理条件： + +```text +review_status == "accepted" +AND suspected_marketing == false +AND note_url 非空 +``` + +其他记录必须跳过，不尝试获取作者 ID。 + +`note_url` 应保留搜索结果中的 `xsec_token` 和 `xsec_source` 参数，以复用 MediaCrawler 现有笔记详情获取逻辑。 + +## 5. 输出格式 + +输出文件为 UTF-8 JSON 对象： + +```json +{ + "results": [ + { + "note_id": "公开笔记 ID", + "public_user_id": "公开作者 ID", + "profile_url": "https://www.xiaohongshu.com/user/profile/公开作者ID", + "source_keyword": "重庆企业主资产配置", + "evidence_note_url": "公开笔记链接", + "extracted_at": "2026-07-24T16:00:00+08:00" + } + ], + "errors": [ + { + "note_id": "失败的笔记 ID", + "note_url": "失败的笔记链接", + "error": "简短错误原因" + } + ] +} +``` + +本阶段的输出只是后续流程的中间文件，不承担正式账号库、人工终审或长期保存职责。 + +## 6. 复用 MediaCrawler 的方式 + +新工具定义一个继承 `media_platform.xhs.core.XiaoHongShuCrawler` 的轻量适配器。 + +适配器复用： + +- `CDPBrowserManager` 连接 Chrome 9222 调试端口。 +- MediaCrawler 的浏览器上下文和 Cookie 转换。 +- `XiaoHongShuClient` 的请求签名和笔记详情接口。 +- `parse_note_info_from_note_url()` 解析笔记链接。 +- `get_note_detail_async_task()` 获取原始笔记详情。 +- 现有登录状态检查和二维码登录回退流程。 + +适配器只覆盖详情模式的结果处理：获取原始笔记详情后读取 `note_detail["user"]["user_id"]`，生成主页链接并写入本工具的结果集合，不调用 `store.xhs.update_xhs_note()`。 + +因此，MediaCrawler 当前将原始 ID 转为 `creator_hash` 的匿名存储流程及其测试保持不变。 + +## 7. 数据流 + +1. 解析命令行中的输入、输出路径。 +2. 校验输入文件存在且顶层为 JSON 数组。 +3. 过滤出人工审核通过且非营销的记录。 +4. 对 `note_url` 去重，避免重复访问同一笔记。 +5. 将 MediaCrawler 配置为小红书详情模式、关闭评论和媒体下载。 +6. 通过 MediaCrawler 连接已启用远程调试的 Chrome。 +7. 对每条通过审核的笔记调用现有详情获取方法。 +8. 从原始详情读取公开作者 ID。 +9. 生成公开主页链接并保留来源关键词和证据笔记链接。 +10. 将成功和失败结果一次性写入输出 JSON。 +11. 正常关闭本工具创建的页面和 MediaCrawler 客户端；不主动关闭用户已有的 Chrome。 + +## 8. 访问控制与节流 + +- 只访问输入中已人工确认的公开笔记。 +- 默认串行处理，避免新增并发压力。 +- 沿用 MediaCrawler 的请求间隔配置。 +- 不访问生成的作者主页链接。 +- 不请求评论接口。 +- 不下载图片或视频。 +- 不输出 Cookie、请求头、签名参数或登录凭证。 + +## 9. 错误处理 + +以下错误只记录到 `errors`，不终止整个批次： + +- 输入记录缺少 `note_url`。 +- 链接无法解析笔记 ID。 +- 笔记已删除、不可见或接口返回空结果。 +- 原始详情缺少 `user.user_id`。 +- 单条请求超时或平台临时拒绝。 + +以下错误应立即失败并返回非零退出码： + +- 输入文件不存在或不是合法 JSON。 +- 输入 JSON 顶层不是数组。 +- MediaCrawler 项目无法导入。 +- 无法连接 Chrome 9222 且登录流程无法建立可用浏览器上下文。 +- 输出目录不可写。 + +即使批次中存在单条错误，只要初始化和输出成功，程序仍以成功状态结束，并在控制台汇总成功、跳过和失败数量。 + +## 10. 去重与幂等 + +本阶段按 `note_url` 去重输入，并按 `(note_id, public_user_id)` 去重结果。 + +重复运行时重新生成完整输出文件，而不是追加旧结果。这样不需要引入数据库、状态文件或增量同步逻辑，也避免一次失败留下半写入文件。 + +输出应先写入同目录临时文件，写入成功后再原子替换目标文件，防止中断导致 JSON 损坏。 + +## 11. 测试与验收 + +至少提供以下自动化检查： + +1. 只选择 `accepted` 且非营销的记录。 +2. 拒绝非数组 JSON 输入。 +3. 重复笔记链接只处理一次。 +4. 能从模拟笔记详情提取 `user.user_id` 并生成主页链接。 +5. 缺少 `user_id` 时写入单条错误而不中断后续记录。 +6. 输出不包含 Cookie、手机号、微信、住址或评论用户信息。 + +人工验收使用一条已确认的公开笔记： + +1. Chrome 9222 连接成功。 +2. 工具仅请求该笔记详情。 +3. 输出包含正确的 `public_user_id`、`profile_url` 和证据笔记链接。 +4. MediaCrawler 原有匿名存储测试继续通过。 +5. 工具不会打开或采集作者主页详情。 + +## 12. 后续接口 + +阶段一输出的 `public_user_id` 可在后续阶段转换为 MediaCrawler 创作者模式输入： + +```text +https://www.xiaohongshu.com/user/profile/{public_user_id} +``` + +后续阶段可以读取 `extracted_ids.json`，调用现有创作者模式抓取公开笔记，再进行 AI 复筛和人工终审。该能力不在本阶段实现。 + +## 13. 使用边界 + +- 仅用于个人、非商业研究。 +- 仅处理公开笔记及其公开发布者标识。 +- 不依据笔记内容断言作者的真实职业、资产、身份或意图。 +- 不用于自动营销、联系人挖掘或跨平台真人匹配。 +- 输入和输出仅保存在本机，不提交 Git，不上传公共云盘。 +- 研究结束后删除不再需要的账号级中间数据。 diff --git a/docs/superpowers/specs/2026-07-27-keyword-crawl-runner-design.md b/docs/superpowers/specs/2026-07-27-keyword-crawl-runner-design.md new file mode 100644 index 000000000..098344363 --- /dev/null +++ b/docs/superpowers/specs/2026-07-27-keyword-crawl-runner-design.md @@ -0,0 +1,356 @@ +# 小红书关键词自动采集队列设计 + +> 日期：2026-07-27 +> +> 状态：已确认，待实施 + +## 1. 目标 + +新增一个独立的关键词队列程序，读取《首批可立即使用的搜索词.md》中“建议首先执行的20个词”，逐个调用 MediaCrawler 本地 API 完成小红书公开笔记采集。 + +程序必须在每个关键词完成后持久化进度。中途失败或程序重启后，不重复提交已经成功的关键词，并能继续处理未完成项。 + +本阶段采用以下固定配置： + +- 平台：小红书。 +- 类型：关键词搜索。 +- 登录方式：二维码或现有 Chrome 登录态。 +- 每个关键词：请求20条笔记。 +- 一级评论：关闭。 +- 二级评论：关闭。 +- 保存格式：JSON。 +- 关键词失败：记录后继续下一个。 + +## 2. 非目标 + +本阶段不实现： + +- AI 内容筛选。 +- 作者 ID 提取或创作者模式采集。 +- 数据库存储。 +- WebUI 队列管理页面。 +- Windows 服务、定时任务或常驻调度器。 +- 全量运行词表中的其他批次。 +- 评论、图片或视频采集。 +- 自动修改、扩展或组合搜索词。 + +## 3. 目录 + +新增独立目录： + +```text +D:\red_note_rich_search\keyword_crawl_runner +├── run_keywords.py +├── crawl_progress.json +└── README.md +``` + +`run_keywords.py` 使用 Python 标准库调用本机 HTTP API，不引入新的第三方依赖。 + +`crawl_progress.json` 为运行时状态文件，不应提交 Git。首次运行时自动创建。 + +## 4. 输入词表解析 + +默认输入文件： + +```text +C:\Users\15377\Documents\民生银行课题\首批可立即使用的搜索词.md +``` + +程序只读取二级标题： + +```text +## 建议首先执行的20个词 +``` + +并只解析该标题下紧随的 `text` 代码块。后续章节不在本阶段处理。 + +解析规则： + +1. 去除每行首尾空白。 +2. 忽略空行。 +3. 按首次出现顺序去重。 +4. 最终必须恰好得到20个关键词。 +5. 标题不存在、代码块格式错误或结果不是20个时立即退出，不启动爬虫。 + +程序为解析出的有序关键词列表计算 SHA-256 摘要，并写入进度文件。以后启动时若摘要变化，程序必须停止并提示用户确认，不得把旧进度错误套用到新词表。 + +## 5. API 调用 + +默认 API 地址： + +```text +http://127.0.0.1:8080/api +``` + +程序启动时先调用： + +```text +GET /crawler/status +``` + +确认后端在线。每个关键词使用一次： + +```text +POST /crawler/start +``` + +请求体固定为： + +```json +{ + "platform": "xhs", + "login_type": "qrcode", + "crawler_type": "search", + "keywords": "当前关键词", + "start_page": 1, + "enable_comments": false, + "enable_sub_comments": false, + "save_option": "json", + "cookies": "", + "headless": false, + "max_notes_count": 20 +} +``` + +提交后每2秒调用一次 `GET /crawler/status`，直到对应任务结束。一个关键词结束后等待5秒，再提交下一个关键词。 + +程序不得并发启动多个关键词任务。 + +## 6. API 状态增强 + +当前 MediaCrawler 在子进程结束后只暴露 `status="idle"`，无法区分成功与失败。为支持可靠断点续跑，需要给现有状态增加： + +| 字段 | 含义 | +|---|---| +| `task_id` | 当前或最近一次任务的 UUID | +| `last_exit_code` | 最近一次子进程退出码；运行中为 `null` | +| `finished_at` | 最近一次任务结束时间；运行中为 `null` | + +`POST /crawler/start` 成功响应同时返回本次 `task_id`： + +```json +{ + "status": "ok", + "message": "Crawler started successfully", + "task_id": "UUID" +} +``` + +`GET /crawler/status` 示例： + +```json +{ + "status": "idle", + "platform": null, + "crawler_type": null, + "started_at": "2026-07-27T10:00:00+08:00", + "error_message": null, + "task_id": "UUID", + "last_exit_code": 0, + "finished_at": "2026-07-27T10:02:00+08:00" +} +``` + +任务开始时，服务生成新的 `task_id`，并将 `last_exit_code`、`finished_at` 清空。子进程结束后记录退出码和结束时间。 + +API 重启后不要求保留历史任务结果。跨 API 重启的不确定任务由队列程序标记为 `needs_review`。 + +## 7. 进度状态 + +`crawl_progress.json` 示例： + +```json +{ + "source_file": "C:\\Users\\15377\\Documents\\民生银行课题\\首批可立即使用的搜索词.md", + "keywords_sha256": "词表摘要", + "created_at": "2026-07-27T10:00:00+08:00", + "updated_at": "2026-07-27T10:02:00+08:00", + "keywords": [ + { + "keyword": "重庆企业主资产配置", + "status": "succeeded", + "attempts": 1, + "task_id": "UUID", + "started_at": "2026-07-27T10:00:00+08:00", + "finished_at": "2026-07-27T10:02:00+08:00", + "exit_code": 0, + "error": null + }, + { + "keyword": "重庆公私资产隔离", + "status": "failed", + "attempts": 1, + "task_id": "UUID", + "started_at": "2026-07-27T10:03:00+08:00", + "finished_at": "2026-07-27T10:04:00+08:00", + "exit_code": 1, + "error": "crawler exited with code 1" + } + ] +} +``` + +允许的关键词状态： + +- `pending`：尚未提交。 +- `running`：已提交，等待 API 结果。 +- `succeeded`：对应任务退出码为0。 +- `failed`：对应任务明确返回非零退出码或启动失败。 +- `needs_review`：无法判断上次任务是否完成，禁止自动重跑。 + +每次状态变化后立即保存进度文件。保存采用同目录临时文件加原子替换，避免中断产生半写入 JSON。 + +## 8. 首次运行、继续与重试 + +### 8.1 首次运行 + +1. 解析并验证20个关键词。 +2. 创建全部为 `pending` 的进度文件。 +3. 确认 API 在线且当前没有其他爬虫任务。 +4. 从第一个关键词开始逐个运行。 + +如果 API 已有非本程序提交的任务，程序停止并提示用户，不接管、不终止该任务。 + +### 8.2 中断后继续 + +重新启动时： + +- 跳过 `succeeded`。 +- 继续处理 `pending`。 +- `failed` 默认不在同一轮立即循环重试，留到后续重试模式。 +- 对 `running` 比较进度文件和 API 返回的 `task_id`。 + +当 `running` 的 `task_id` 与 API 相同： + +- API 仍在运行：继续等待。 +- API 已结束：根据 `last_exit_code` 补记为 `succeeded` 或 `failed`。 + +当 API 不在线、已重启或 `task_id` 不匹配时，将该关键词标记为 `needs_review`，避免自动重复爬取。 + +### 8.3 重试失败项 + +提供显式参数： + +```powershell +--retry-failed +``` + +该模式只把 `failed` 重置为 `pending`，仍然跳过 `succeeded` 和 `needs_review`。每次提交前增加 `attempts`。 + +`needs_review` 必须由用户检查数据后手工改为 `succeeded` 或 `failed`，程序不自动猜测。 + +## 9. 数据输出与重复控制 + +MediaCrawler 继续按现有逻辑把笔记写入： + +```text +D:\red_note_rich_search\MediaCrawler\data\xhs\json +``` + +每条记录已有 `source_keyword`，可以追溯到原始搜索词。 + +队列程序通过不重复提交 `succeeded` 关键词，避免整词重复采集。明确失败的任务可能已经写入部分笔记，重试后可能产生重复记录。本阶段不修改 MediaCrawler 的 JSON 写入器；后续 AI 粗筛前按 `(note_id, source_keyword)` 去重。 + +## 10. 控制台输出 + +程序运行时输出紧凑进度： + +```text +[3/20] 开始：重庆公司分红资金安排 +[3/20] 成功：退出码 0，用时 82 秒 +[4/20] 开始：重庆股权退出资产配置 +[4/20] 失败：退出码 1，继续下一项 +``` + +结束时汇总： + +```text +成功 18，失败 2，待处理 0，需要人工确认 0 +``` + +不在控制台重复打印完整爬虫日志。详细日志继续通过 MediaCrawler WebUI 或 `/api/crawler/logs` 查看。 + +## 11. 错误处理 + +以下错误立即停止队列，已保存进度不丢失： + +- 词表文件不存在或格式不符合约定。 +- 词表摘要与已有进度不一致。 +- MediaCrawler API 不在线。 +- API 已有其他任务运行。 +- 进度文件损坏或不可写。 +- API 返回无法解析的数据。 + +以下错误记录当前关键词失败后继续： + +- `/crawler/start` 对当前关键词返回服务端错误。 +- 子进程退出码非0。 +- 单个任务明确报告爬取失败。 + +网络超时不得直接判定任务失败。程序先查询状态；若仍无法确认，则保存为 `needs_review` 并停止，避免重复提交。 + +## 12. 命令行接口 + +默认运行： + +```powershell +cd D:\red_note_rich_search\MediaCrawler + +uv run python ` + ..\keyword_crawl_runner\run_keywords.py ` + --source "C:\Users\15377\Documents\民生银行课题\首批可立即使用的搜索词.md" +``` + +重试明确失败项： + +```powershell +uv run python ` + ..\keyword_crawl_runner\run_keywords.py ` + --source "C:\Users\15377\Documents\民生银行课题\首批可立即使用的搜索词.md" ` + --retry-failed +``` + +可选参数仅包括： + +- `--source`：词表路径。 +- `--api-base`：本地 API 基础地址，默认 `http://127.0.0.1:8080/api`。 +- `--progress`：进度文件路径，默认使用程序目录下的 `crawl_progress.json`。 +- `--retry-failed`：重试明确失败项。 + +不为当前固定研究流程增加通用平台、评论、并发或保存格式参数。 + +## 13. 测试与验收 + +自动化测试至少覆盖： + +1. 只解析“建议首先执行的20个词”。 +2. 去重后不足或超过20个时拒绝运行。 +3. 词表摘要变化时拒绝复用旧进度。 +4. `succeeded` 关键词不会再次提交。 +5. 某关键词失败后继续提交下一项。 +6. 进度文件在每次状态变化后更新。 +7. `running` 任务能通过相同 `task_id` 恢复等待。 +8. 无法确认的任务转为 `needs_review`，不自动重跑。 +9. `--retry-failed` 只重试 `failed`。 +10. API 状态正确暴露退出码和结束时间。 + +测试使用 Python 标准库的临时目录和本地假 HTTP 服务，不连接小红书。 + +人工验收： + +1. 后端和 Chrome 9222 正常运行。 +2. 用前两个关键词进行受控试跑。 +3. 人工中断队列程序，但保留后端任务。 +4. 重新启动后能恢复当前任务并跳过已成功关键词。 +5. 完成20个关键词后，进度文件显示20项终态。 +6. 输出笔记中 `source_keyword` 与任务关键词一致。 + +## 14. 使用边界 + +- 仅用于个人、非商业研究。 +- 仅采集公开笔记。 +- 控制请求频率，不并发运行关键词。 +- 不采集评论、联系方式或非公开信息。 +- 不用于自动营销触达或个人画像定性。 +- 遵守 MediaCrawler 许可证和目标平台规则。 diff --git a/docs/superpowers/specs/2026-07-27-keyword-json-cdp-resilience-design.md b/docs/superpowers/specs/2026-07-27-keyword-json-cdp-resilience-design.md new file mode 100644 index 000000000..89ea9326e --- /dev/null +++ b/docs/superpowers/specs/2026-07-27-keyword-json-cdp-resilience-design.md @@ -0,0 +1,237 @@ +# 关键词 JSON 队列与 Chrome 稳定连接升级设计 + +> 日期：2026-07-27 +> +> 状态：设计已确认，待用户审核书面规格 + +## 1. 背景与目标 + +现有外部工具（GitHub 仓库 `mediaClawerTooler`，本地目录 `keyword_crawl_runner`）已能从固定 Markdown 读取20个关键词，逐词调用 MediaCrawler API，并通过 JSON 进度文件断点恢复。第一轮运行暴露了三个限制：输入被固定为20个词、Chrome 136+ 直接 CDP 连接会为每个爬虫子进程重复请求人工授权、验证码出现后现有三次短重试会在用户完成验证前退出。 + +本次升级采用双仓库最小改造： + +- 外部工具 `mediaClawerTooler` 负责读取任意数量的 JSON 关键词、生成独立进度文件和恢复队列。 +- `MediaCrawler` 继续负责真正的小红书搜索、浏览器控制和原始 JSON 落盘。 +- `start_chrome.ps1` 负责用专用用户目录启动可复用的 9222 Chrome。 +- 验证码仅提供有上限的人工处理时间，不识别、点击或绕过验证码。 + +保持以下现有规则不变：关键词串行执行、每词最多20条、一级和二级评论关闭、保存格式为 JSON。 + +## 2. 非目标 + +本次不实现： + +- 三关键词并行、分组或合并搜索。 +- AI 内容粗筛。 +- 用户 ID、个人主页或创作者模式采集。 +- 数据库入库。 +- 自动识别、点击、破解或绕过验证码。 +- 自动启动或关闭 MediaCrawler API。 +- 自动关闭 Chrome 或接管用户的普通 Chrome。 + +## 3. 总体架构与数据流 + +```text +关键词 JSON + -> 外部工具校验、去空白、保持顺序去重 + -> 创建或读取 progress/<输入文件名>.progress.json + -> 逐个调用本地 MediaCrawler API + -> MediaCrawler 连接同一个 9222 专用 Chrome + -> 小红书关键词搜索 + -> MediaCrawler/data/xhs/json 写入原始笔记 + -> 外部工具原子更新当前关键词状态 + -> 继续下一个关键词 +``` + +两个 Python 项目通过现有 HTTP API 通信，不把 MediaCrawler 的爬虫逻辑复制到外部工具，也不新增第三方依赖。 + +## 4. JSON 输入 + +### 4.1 文件格式 + +`--source` 指向 UTF-8 JSON 文件，结构固定为： + +```json +{ + "keywords": [ + "重庆企业主资产配置", + "重庆公私资产隔离", + "重庆家族信托" + ] +} +``` + +关键词数量不限。解析规则： + +1. 顶层必须为对象，`keywords` 必须为数组。 +2. 数组元素必须全部为字符串；出现其他类型时拒绝整个输入。 +3. 去除每个关键词首尾空白。 +4. 忽略处理后为空的字符串。 +5. 按首次出现顺序去重。 +6. 最终没有有效关键词时停止，不调用 API。 + +本次删除对 Markdown 标题、代码块和“必须恰好20个词”的依赖。 + +### 4.2 命令行 + +正常运行： + +```powershell +uv run python run_keywords.py --source .\inputs\keywords.json +``` + +`--source` 为必填参数，避免误用旧的固定 Markdown。保留现有 `--api-base`、`--progress` 和 `--retry-failed`。 + +## 5. 独立进度文件 + +未传 `--progress` 时，程序按输入文件名生成： + +```text +inputs/重庆客户.json +-> progress/重庆客户.progress.json +``` + +`progress` 目录位于外部工具仓库根目录并自动创建，运行时进度文件不提交 Git。若不同目录存在同名输入，用户可通过 `--progress` 指定不同文件。 + +进度文件继续保存输入文件路径、关键词有序列表摘要、时间戳和逐词状态。允许状态保持为： + +- `pending`：尚未执行。 +- `running`：已经获得 MediaCrawler `task_id`，正在等待。 +- `succeeded`：任务明确成功。 +- `failed`：任务明确失败，可显式重试。 +- `needs_review`：结果不确定，禁止自动重跑。 + +每次状态变化后，仍使用同目录临时文件加原子替换写盘。 + +如果同名进度文件已经存在，但输入的有序关键词摘要发生变化，程序拒绝覆盖或混用旧进度，并提示用户更换输入文件名或通过 `--progress` 使用新文件。 + +## 6. 恢复与重试 + +再次执行相同命令时： + +1. 跳过 `succeeded`。 +2. 对 `running` 使用原 `task_id` 查询 MediaCrawler 状态；任务仍存在则继续等待，结果明确则补记终态。 +3. `failed` 默认跳过，只有传入 `--retry-failed` 才重置为 `pending`。 +4. `needs_review` 保持不变并停止队列，提示人工检查，避免重复提交。 + +开始任务时若网络响应不确定，仍标记 `needs_review`；不得把无法确认的启动请求当作普通失败自动重试。 + +重试明确失败项： + +```powershell +uv run python run_keywords.py ` + --source .\inputs\keywords.json ` + --retry-failed +``` + +## 7. 专用 Chrome 启动脚本 + +外部工具中的 `start_chrome.ps1` 负责： + +1. 检查 `http://127.0.0.1:9222/json/version`。 +2. 接口已可用时提示 Chrome 已运行并成功退出，不重复启动实例。 +3. 接口不可用时从 Windows 常见 Chrome 安装位置查找 `chrome.exe`。 +4. 使用 `--remote-debugging-port=9222` 和专用 `--user-data-dir` 启动可见 Chrome。 +5. 默认专用目录为 `D:\red_note_rich_search\chrome_crawler_profile`，允许通过脚本参数覆盖。 + +第一次启动专用 Chrome 后，用户人工登录小红书；以后复用该目录中的登录状态。脚本不结束任何现有浏览器进程。若 9222 被其他程序占用但 `/json/version` 不是 Chrome 调试接口，脚本报错并停止。 + +## 8. MediaCrawler CDP 连接顺序 + +当 `CDP_CONNECT_EXISTING = True` 时，连接逻辑调整为： + +1. 优先请求 `http://127.0.0.1:9222/json/version`；端口取自现有 `CDP_DEBUG_PORT` 配置。 +2. 返回有效 `webSocketDebuggerUrl` 时，通过该地址连接命令行启动的专用 Chrome。 +3. HTTP 接口不可用时，回退到现有 Chrome 136+ 直接 CDP 连接方式。 +4. 进入回退模式时输出明确警告，说明浏览器可能继续要求人工授权。 + +这样每个关键词仍可使用现有 MediaCrawler 子进程，但都连接同一个已授权的专用 Chrome，不需要重写爬虫或改成长驻工作进程。 + +## 9. 验证码人工等待 + +小红书 HTTP 状态 `461` 或 `471` 进入专用验证码流程： + +1. 记录一次清晰日志，提示用户在当前 Chrome 中人工完成验证。 +2. 保持当前爬虫任务存活，固定每10秒重试当前请求。 +3. 最长等待5分钟。 +4. 验证通过后继续原请求和原关键词任务。 +5. 超过5分钟仍返回 `461/471` 时，让任务以明确失败结束；runner 保存 `failed`，以后可用 `--retry-failed` 重跑。 + +验证码等待只匹配 `461/471`。其他网络异常、业务错误和笔记不存在继续沿用现有短重试或异常处理，不进入五分钟等待。实现不得模拟人工验证动作。 + +## 10. 错误处理 + +以下情况在提交新关键词前停止队列，已保存进度不丢失： + +- JSON 文件不存在、编码错误或结构不合法。 +- 输入摘要与已有进度不一致。 +- MediaCrawler API 不在线或返回无法解释的状态。 +- API 已有不属于当前进度文件的任务。 +- 进度文件损坏或不可写。 +- `running` 的 `task_id` 与 API 当前或最近任务不匹配。 + +单个关键词明确返回非零退出码时记为 `failed`，并继续下一项。验证码等待超时也属于明确失败。启动请求是否成功无法确认时记为 `needs_review` 并停止。 + +Chrome 调试接口在任务中途消失时，由 MediaCrawler 返回明确失败；runner 保存当前状态，后续关键词是否继续遵循现有“明确失败后继续”规则。 + +## 11. 测试设计 + +### 11.1 外部工具自动化测试 + +- 任意数量关键词能够读取。 +- 非法 JSON、缺少 `keywords`、错误字段类型和空结果被拒绝。 +- 空白词被忽略，重复词按首次出现顺序去重。 +- 默认进度路径按输入文件名生成，`--progress` 能覆盖。 +- 同名输入内容变化时拒绝复用旧进度。 +- `succeeded` 被跳过，`running` 能按相同 `task_id` 恢复。 +- `--retry-failed` 只重置 `failed`。 +- `needs_review` 不自动重跑。 + +### 11.2 MediaCrawler 自动化测试 + +- `/json/version` 可用时优先使用其中的 WebSocket 地址。 +- HTTP 探测失败时回退现有直接 CDP 模式。 +- 无效 `/json/version` 响应不会被当作有效连接。 +- `461/471` 验证后成功时继续原请求。 +- `461/471` 持续5分钟时明确失败。 +- 普通错误不进入长等待。 + +测试使用替身响应和可控时钟，不访问小红书，不在自动化测试中真实等待5分钟。 + +### 11.3 人工验收 + +1. 执行 `start_chrome.ps1`，在专用 Chrome 首次登录小红书。 +2. 启动 MediaCrawler API。 +3. 用至少两个测试关键词运行 JSON 队列，确认第二个任务不再弹出 CDP 授权。 +4. 中断 runner 后重新运行，确认成功项被跳过、运行项能够恢复。 +5. 在受控情况下出现验证码时人工完成，确认任务在五分钟窗口内继续。 +6. 检查 `MediaCrawler/data/xhs/json` 和独立进度文件。 + +## 12. 最终使用顺序 + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +.\start_chrome.ps1 +``` + +首次使用时在打开的专用 Chrome 中登录小红书。然后启动 API： + +```powershell +cd D:\red_note_rich_search\MediaCrawler +uv run uvicorn api.main:app --port 8080 +``` + +最后在另一终端运行队列： + +```powershell +cd D:\red_note_rich_search\keyword_crawl_runner +uv run python run_keywords.py --source .\inputs\keywords.json +``` + +## 13. 使用边界 + +- 仅用于个人、非商业研究和公开内容。 +- 控制请求频率，关键词保持串行。 +- 不采集评论、联系方式或非公开信息。 +- 不把验证码处理自动化。 +- 遵守 MediaCrawler 许可证、目标平台规则及适用要求。 diff --git a/docs/superpowers/specs/2026-07-29-xhs-needs-review-partial-save-design.md b/docs/superpowers/specs/2026-07-29-xhs-needs-review-partial-save-design.md new file mode 100644 index 000000000..e91dbac25 --- /dev/null +++ b/docs/superpowers/specs/2026-07-29-xhs-needs-review-partial-save-design.md @@ -0,0 +1,227 @@ +# 小红书风控暂停与部分结果保存设计 + +## 1. 背景 + +关键词队列当前通过本地 API 为每个关键词启动一个独立的 MediaCrawler 子进程。runner 只根据子进程退出码判断结果：退出码为 `0` 时标记为 `succeeded`，其他退出码一律标记为 `failed`。 + +小红书搜索列表请求成功后，MediaCrawler 会并发获取笔记详情。当前实现等待全部详情请求完成后才统一写入 JSON。当任意详情请求持续返回 `461/471` 时，程序最多等待人工验证 5 分钟；如果风控仍未解除，`CaptchaRequiredError` 会中断整批请求。即使部分详情已经获取成功，它们也可能尚未进入保存循环。子进程随后以非零退出码结束，runner 将整个关键词标记为普通失败。 + +另外，`461/471` 只表示后台接口要求验证或触发风险控制，并不保证普通搜索页面会呈现可操作的滑块或验证码。当前打开搜索页的行为只能帮助人工观察，不能被视为验证一定可完成。 + +## 2. 目标 + +本次改造采用“结构化风控状态 + 增量保存”方案，目标如下： + +1. 每条笔记详情成功获取后尽快写入 JSON，避免后续风控导致已获取结果丢失。 +2. 将风控超时与普通程序失败区分开，使用 `needs_review` 表示需要人工确认。 +3. runner 遇到 `needs_review` 后立即保存进度并暂停整个队列，不再启动后续关键词。 +4. API 返回本轮实际保存数量、目标数量和停止原因，不再只暴露退出码。 +5. 保留现有每词 20 条、关闭评论、串行执行关键词、JSON 输出和断点续跑行为。 +6. 不自动识别、点击或绕过验证码，也不尝试规避平台风险控制。 + +## 3. 非目标 + +本次不实现以下内容： + +- 不把 MediaCrawler 攚造成长期驻留的关键词工作进程。 +- 不让多个关键词并行采集。 +- 不自动处理滑块、扫码或其他验证。 +- 不根据输出文件内容猜测任务状态。 +- 不改变关键词 JSON 格式。 +- 不增加数据库或 Excel 输出。 +- 不解决关键词重跑时的业务去重；继续沿用现有 `(note_id, source_keyword)` 去重约定。 + +## 4. 状态模型 + +### 4.1 子进程结果 + +子进程结果分为三类： + +| 结果 | 含义 | 退出码 | +|---|---|---:| +| `succeeded` | 本轮正常完成 | `0` | +| `needs_review` | 遇到 `461/471`，人工等待结束后仍未恢复 | `2` | +| `failed` | 参数错误、启动失败、未处理异常等普通失败 | 其他非零值 | + +退出码 `2` 只用于受控的人工复核结果，不用于一般异常。 + +### 4.2 API 任务状态 + +API 的运行状态继续使用 `running`、`stopping` 和 `idle`。子进程结束后 API 回到 `idle`，并通过新增的结果字段描述刚刚结束的任务： + +```json +{ + "status": "idle", + "task_id": "c3e6ff24-3811-47fb-9a80-07224b0aa4dc", + "last_exit_code": 2, + "outcome": "needs_review", + "saved_count": 8, + "expected_count": 20, + "stop_reason": "captcha_or_risk_control", + "error_message": "后台接口持续返回 461/471，人工验证等待超时", + "finished_at": "2026-07-29T15:22:10+08:00" +} +``` + +API 保持 `idle` 是为了表明子进程已经结束，避免把“需要人工复核”误解为仍有任务占用 API。runner 负责根据 `outcome` 暂停队列。 + +### 4.3 runner 进度状态 + +runner 为每个关键词继续使用 `pending`、`running`、`succeeded`、`failed` 和 `needs_review`。`needs_review` 项新增以下可选字段： + +```json +{ + "keyword": "重庆别墅装修", + "status": "needs_review", + "attempts": 3, + "saved_count": 8, + "expected_count": 20, + "stop_reason": "captcha_or_risk_control", + "error": "检测到风控，已保存 8/20 条，队列已暂停" +} +``` + +这些字段必须进行类型校验。布尔值不能作为整数数量接受；数量不得为负数，且 `saved_count` 不得大于 `expected_count`。 + +## 5. MediaCrawler 数据流 + +### 5.1 搜索与增量保存 + +搜索列表仍一次获取最多 20 个候选笔记。详情请求保留当前并发限制，但每个详情任务完成后必须在保存锁内执行以下操作： + +1. 验证详情结果非空。 +2. 调用现有 `xhs_store.update_xhs_note` 写入 JSON。 +3. 完成媒体保存逻辑（仅在现有配置开启时）。 +4. 将本轮 `saved_count` 增加 1。 + +保存锁只保护写入和计数，不包围网络请求，避免把现有详情并发退化为串行。不得引入新的第三方依赖。 + +### 5.2 并发任务收尾 + +当某个详情任务在 5 分钟等待后仍抛出 `CaptchaRequiredError` 时： + +1. 取消本页尚未完成的其他详情任务。 +2. 等待这些任务完成取消，防止后台遗留任务继续请求。 +3. 保留此前已经成功写入的记录。 +4. 生成受控的 `needs_review` 结果。 +5. 执行现有浏览器和数据库清理。 +6. 子进程以退出码 `2` 结束。 + +如果单条笔记是已删除、不可见或普通 `DataFetchError`，沿用现有“记录并跳过”的行为，不把它升级为整词 `needs_review`。只有明确的 `CaptchaRequiredError` 才触发整队暂停。 + +### 5.3 验证页面行为 + +首次收到 `461/471` 时继续执行以下行为： + +- 将专用 Chrome 页面置前； +- 打开当前关键词的官方搜索页； +- 日志明确说明“后台接口触发风控，页面不一定显示验证码”； +- 记录响应中的 `Verifytype` 和 `Verifyuuid`（若存在）； +- 每次重试前继续刷新 Cookie、重新生成签名。 + +仍保留最长 5 分钟人工处理窗口。程序只等待用户自行完成平台提供的验证，不自动操作验证控件。5 分钟内接口恢复则继续本轮；超时则进入 `needs_review`。 + +## 6. 子进程到 API 的结果传递 + +采用标准输出中的单行机器结果，不新增临时文件或数据库表。子进程在受控结束前输出一个固定前缀的 JSON 行，例如： + +```text +MEDIACRAWLER_RESULT:{"outcome":"needs_review","saved_count":8,"expected_count":20,"stop_reason":"captcha_or_risk_control","error_message":"..."} +``` + +约束如下： + +- JSON 使用单行格式；中文按 UTF-8 输出。 +- API 进程只解析精确匹配固定前缀的行，其他日志保持原样。 +- API 对字段进行严格类型和值域校验，不信任子进程输出。 +- 合法结果保存到当前任务状态；畸形结果忽略并记录警告。 +- 退出码 `2` 但缺少合法机器结果时，API 仍返回 `outcome = needs_review`，数量字段为 `null`，避免错误地降级为普通失败。 +- 其他非零退出码即使携带未知结果，也按 `failed` 处理。 + +该方式复用现有 stdout 读取链路，避免增加跨进程结果文件的创建、清理和并发命名问题。 + +## 7. runner 行为 + +runner 轮询到 API `status = idle` 后按以下顺序处理： + +1. 校验 `task_id` 与进度文件一致。 +2. 校验 `last_exit_code`、`outcome` 和数量字段。 +3. `outcome = succeeded`：标记 `succeeded`，继续下一个关键词。 +4. `outcome = needs_review`：写入数量与停止原因，标记 `needs_review`，原子保存进度，立即退出队列，返回现有的人工复核退出码 `2`。 +5. `outcome = failed`：标记 `failed`，沿用当前“记录失败后继续”的策略。 + +兼容规则：如果 API 暂未返回 `outcome`，runner 继续按退出码判断；退出码 `0` 视为成功，退出码 `2` 视为 `needs_review`，其他非零值视为失败。 + +`--retry-failed` 仍然只重置 `failed`，不得自动重置 `needs_review`。人工处理后继续沿用现有安全流程：检查输出与进度文件，再明确决定将该项改为 `succeeded` 或 `failed`；只有改为 `failed` 后才可通过 `--retry-failed` 重跑。 + +## 8. 异常与边界条件 + +- 搜索列表请求本身触发风控且尚未保存任何详情：`saved_count = 0`，状态仍为 `needs_review`。 +- 已保存部分详情后触发风控：保留数据并准确报告数量。 +- 20 条全部保存后，清理阶段发生普通异常：不得误报为 `needs_review`，按现有失败规则处理并保留数据。 +- API 在任务结束前重启或连接中断：runner 沿用现有 `needs_review` 安全停队规则，不猜测结果。 +- 进度文件中的数量字段畸形：拒绝加载，防止错误续跑。 +- 多个并发详情任务同时收到风控：只生成一次任务级 `needs_review` 结果和一次用户提示。 +- 用户中断 runner 不会自动停止 API 中已启动的爬虫任务；恢复时仍先按 `task_id` 接管正在运行的任务。 + +## 9. 测试策略 + +### 9.1 MediaCrawler + +新增或调整聚焦测试，至少覆盖： + +1. 部分详情成功后另一个详情触发 `CaptchaRequiredError`，成功详情已经写入。 +2. 风控发生后未完成任务被取消并回收。 +3. 风控超时生成退出码 `2` 和合法机器结果。 +4. 机器结果只输出一次。 +5. `Verifytype`、`Verifyuuid` 缺失时仍能进入 `needs_review`。 +6. 普通 `DataFetchError` 不触发整词暂停。 +7. 完全成功仍返回退出码 `0`。 +8. 现有 CAPTCHA 等待、Cookie 刷新、签名刷新和页面呈现测试继续通过。 + +### 9.2 API + +至少覆盖： + +1. 解析合法 `MEDIACRAWLER_RESULT` 行。 +2. 拒绝畸形 JSON、负数、布尔数量和未知 outcome。 +3. 退出码 `2` 映射为 `needs_review`。 +4. 退出码 `2` 且机器结果缺失时使用安全回退。 +5. 退出码 `0` 与其他非零退出码保持原行为。 + +### 9.3 runner + +至少覆盖： + +1. `needs_review` 保存进度并停止后续关键词。 +2. 部分数量正确写入进度文件。 +3. `--retry-failed` 不重置 `needs_review`。 +4. 缺少 `outcome` 时按退出码兼容。 +5. 畸形结果进入 `needs_review` 安全停队，而不是继续执行。 +6. 原有 40 个 runner 回归测试继续通过。 + +## 10. 验收标准 + +在一个包含至少两个关键词的测试队列中，模拟第一个关键词保存 8 条后触发持续风控，应满足： + +1. JSON 中保留第一个关键词已成功获取的 8 条详情。 +2. API 最终为 `idle`，且最后结果为 `outcome = needs_review`、`saved_count = 8`、`expected_count = 20`。 +3. runner 将第一个关键词标记为 `needs_review` 并原子保存进度。 +4. 第二个关键词保持 `pending`，没有调用启动 API。 +5. runner 以人工复核退出码 `2` 结束。 +6. 日志不声称一定存在可见验证码，并包含可用的风控诊断信息。 +7. 全部现有聚焦测试与新增测试通过。 + +## 11. 实施范围 + +预计只修改以下现有区域，不引入新服务或第三方依赖： + +- `media_platform/xhs/client.py`：风控诊断与受控异常信息。 +- `media_platform/xhs/core.py`：详情增量保存、取消与任务结果计数。 +- `main.py`：输出受控机器结果并使用退出码 `2`。 +- `api/services/crawler_manager.py`：解析子进程结果并扩展最后任务状态。 +- `api/schemas.py` 或对应响应模型：增加可选结果字段。 +- `keyword_crawl_runner/run_keywords.py`：识别 `needs_review`、保存数量并暂停。 +- 两个项目的相关测试和中文使用说明。 + +实现必须测试先行，且不得修改用户现有的未跟踪研究文档。 diff --git a/docs/superpowers/specs/2026-07-31-public-account-creator-pipeline-design.md b/docs/superpowers/specs/2026-07-31-public-account-creator-pipeline-design.md new file mode 100644 index 000000000..f654ac662 --- /dev/null +++ b/docs/superpowers/specs/2026-07-31-public-account-creator-pipeline-design.md @@ -0,0 +1,158 @@ +# 首次捕获作者 ID 与候选 Creator 自动采集设计 + +> 日期：2026-07-31 +> +> 状态：已确认，待实施 + +## 1. 目标 + +在第一次关键词搜索成功取得笔记详情时、匿名化存储之前，旁路捕获公开作者的 24 位 +`user_id`。AI 粗筛完成后，只保留 `potential_customer` 且证据等级为 `strong` 或 +`medium` 的候选作者，将其与旁路 ID 映射关联，并自动调用现有 creator 模式采集候选 +作者主页下的公开笔记。 + +```text +关键词搜索取得原始 note_detail + ├─ 现有匿名存储：creator_hash + 笔记内容 + └─ 本地临时映射：note_id + creator_hash + user_id + ↓ +AI 粗筛（不接收 user_id） + -> potential_customer + strong/medium + -> 关联本地临时映射 + -> candidate_creators.json（含 user_id） + -> 自动调用 creator 模式 + -> 保存候选作者的公开笔记，供第二轮 AI 筛选 +``` + +主流程不依赖粗筛后重新获取旧笔记详情，因此不会因原笔记 `xsec_token` 过期而丢失 +作者 ID。 + +## 2. 候选规则 + +候选证据笔记必须同时满足： + +```text +status in {"screened", "cached"} +AND label == "potential_customer" +AND evidence_level in {"strong", "medium"} +AND evidence_quotes 为非空字符串数组 +``` + +只有至少包含一条上述证据笔记的作者才进入 `candidate_creators.json`。`weak`、`none`、 +`service_provider`、`media_or_educator`、`unclear` 和 `irrelevant` 不进入候选列表。 + +这仍是内容证据驱动的研究候选，不是对作者真实身份、资产、职业或意图的事实判断。 + +## 3. 首次抓取时捕获 ID + +新增一个默认关闭的研究开关。关键词运行器通过 API 启动小红书搜索时显式开启；普通 +MediaCrawler 命令、详情模式、creator 模式和其他平台不捕获原始 ID。 + +捕获点位于 `media_platform/xhs/core.py`：`get_note_detail_async_task()` 已返回完整 +`note_detail`，但尚未调用 `store.xhs.update_xhs_note()`。捕获记录只包含： + +```json +{ + "note_id": "公开笔记 ID", + "creator_hash": "与匿名笔记一致的 SHA-256 截断值", + "public_user_id": "24 位公开用户 ID", + "profile_url": "https://www.xiaohongshu.com/user/profile/公开用户ID", + "captured_at": "带时区的 ISO 8601 时间" +} +``` + +不保存昵称、头像、简介、IP、Cookie、请求头、签名、评论用户或联系方式。记录按 +`(note_id, public_user_id)` 幂等写入本地私有旁路文件。写入失败应终止该次关键词任务， +避免出现“笔记已保存但 ID 静默丢失”的不一致状态。 + +现有 `store/xhs/__init__.py` 不修改，搜索内容仍只保存 `creator_hash`。 + +## 4. API 与关键词运行器 + +MediaCrawler API 的搜索请求新增布尔字段 `capture_creator_ids`，默认 `false`。进程管理器 +只在该值为真时向 `main.py` 传递对应命令行开关。 + +`keyword_crawl_runner/run_keywords.py` 启动小红书搜索时固定传递 +`capture_creator_ids: true`，从而确保本研究流程每次首次取得详情时同步捕获 ID。 + +旁路文件保存在 MediaCrawler 的本地数据目录中，并按日期生成，便于和 +`search_contents_YYYY-MM-DD.json` 对齐。旁路文件加入 `.gitignore`，不得提交 Git。 + +## 5. AI 粗筛与 ID 关联 + +`keyword_crawl_runner/screen_data.py` 继续只把匿名笔记内容发送给 AI。原始 `user_id`、 +主页链接和旁路映射不得进入提示词、缓存或 `screened_notes.jsonl`。 + +模型响应全部完成且通过结构校验后： + +1. 使用现有 strong/medium 合格判断聚合作者。 +2. 丢弃没有合格证据笔记的作者。 +3. 通过 `creator_hash` 将候选作者与本地旁路映射关联。 +4. 一个哈希对应多个不同 `user_id` 时记录冲突并不自动采集。 +5. 缺少映射的候选写入 `candidate_creator_errors.json`，不尝试用旧 token 重抓详情。 +6. 成功关联的候选在 `candidate_creators.json` 中增加 `public_user_id` 和 `profile_url`。 + +候选输出是本地研究中间文件，不上传 AI，不提交 Git。 + +## 6. 自动 Creator 采集 + +粗筛命令新增显式开关 `--crawl-creators`。启用时，`screen_data.py` 写完全部筛选输出后： + +1. 按 `public_user_id` 去重。 +2. 通过现有本地 MediaCrawler API 提交 creator 任务。 +3. 关闭评论和媒体下载。 +4. 使用 `--creator-max-notes` 控制每位作者的最大公开笔记数，该值必须为正整数。 +5. 等待任务结束并把成功、失败和需要人工复核的状态写入运行清单。 + +未指定 `--crawl-creators` 时只输出候选及 ID，不访问作者主页。显式开关既支持全自动 +运行，也保留用户对主页采集时机的控制。 + +Creator 模式使用新登录态和已捕获的 `user_id`，不再依赖候选证据笔记的旧 +`xsec_token`。平台仍可能因登录状态、频率限制或风控拒绝 creator 请求；这种失败应记录 +并可重试，但不能回退为手工打开候选笔记恢复 ID。 + +## 7. 第二轮 AI 筛选 + +Creator 输出保持 MediaCrawler 现有匿名笔记格式。第二轮 AI 筛选复用现有 +`screen_data.py`，通过输入路径或文件匹配模式指向 creator 输出。本阶段不引入新的模型 +提示词或对个人身份作额外推断。 + +## 8. 错误处理与恢复 + +- 捕获开关关闭：完全保持现有行为。 +- 原始详情缺少合法 24 位 `user_id`：记录捕获错误，笔记仍可按匿名流程保存。 +- 旁路文件不可写：关键词任务失败，保留已写入的原子记录供重试审计。 +- 候选缺少 ID 映射或发生哈希冲突：写入错误文件，不启动该作者的 creator 任务。 +- AI 粗筛存在单条失败：不影响其他已验证 strong/medium 候选。 +- Creator 单个任务失败：记录状态，后续候选继续处理。 +- 输出目录非空等现有安全检查继续生效。 + +不新增数据库、WebUI、联系人提取、跨平台匹配或自动营销功能。 + +## 9. 测试与验收 + +自动测试覆盖： + +1. 捕获开关默认关闭，关闭时不写旁路数据。 +2. 开启时从原始详情捕获合法 24 位 `user_id`，并生成与现有逻辑一致的 + `creator_hash`。 +3. 捕获输出不包含昵称、Cookie、请求头、评论用户等字段。 +4. API 只在请求开启捕获时传递命令行参数。 +5. 关键词运行器固定开启捕获。 +6. 候选只包含 `potential_customer` 的 strong/medium 证据作者。 +7. ID 映射不会进入 AI 请求或筛选缓存。 +8. 缺失映射和哈希冲突不会启动 creator。 +9. `--crawl-creators` 关闭时不访问主页，开启时只提交去重后的候选 ID。 +10. Creator 失败被记录且不阻止后续候选。 +11. 原有匿名存储与关键词运行器测试继续通过。 + +人工验收至少执行一条新关键词搜索，确认首次详情阶段生成旁路映射；随后执行粗筛， +确认候选仅含 strong/medium、候选 ID 正确、旧笔记 token 不再被请求，并确认 creator +输出可作为第二轮 AI 筛选输入。 + +## 10. 使用与数据边界 + +- 仅用于个人、非商业研究和公开内容分析。 +- 候选结论只表示公开内容证据满足筛选规则，不表示个人事实。 +- 不采集评论用户 ID、联系方式、非公开主页信息或跨平台身份。 +- 旁路映射和候选 ID 仅保存在本机，不提交 Git；研究结束后删除不再需要的数据。 diff --git a/media_platform/xhs/client.py b/media_platform/xhs/client.py index 1e07b0802..61d7cf818 100644 --- a/media_platform/xhs/client.py +++ b/media_platform/xhs/client.py @@ -19,7 +19,8 @@ import asyncio import json -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union +import time +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Union from urllib.parse import quote, urlencode import httpx @@ -35,13 +36,59 @@ if TYPE_CHECKING: from proxy.proxy_ip_pool import ProxyIpPool -from .exception import DataFetchError, IPBlockError, NoteNotFoundError +from .exception import CaptchaRequiredError, DataFetchError, IPBlockError, NoteNotFoundError from .field import SearchNoteType, SearchSortType from .help import get_search_id from .extractor import XiaoHongShuExtractor from .playwright_sign import sign_with_xhshow +CAPTCHA_RETRY_SECONDS = 10 +CAPTCHA_MAX_WAIT_SECONDS = 300 + + +def raise_for_captcha(response: httpx.Response) -> None: + if response.status_code not in {461, 471}: + return + + verify_type = response.headers.get("Verifytype", "unknown") + verify_uuid = response.headers.get("Verifyuuid", "unknown") + raise CaptchaRequiredError( + "CAPTCHA appeared; please verify manually in Chrome. " + f"Verifytype: {verify_type}, Verifyuuid: {verify_uuid}" + ) + + +async def wait_for_captcha( + operation: Callable[[], Awaitable[Any]], + *, + on_captcha: Optional[Callable[[], Awaitable[None]]] = None, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + monotonic: Callable[[], float] = time.monotonic, +) -> Any: + deadline = None + presented = False + while True: + try: + return await operation() + except CaptchaRequiredError: + now = monotonic() + if deadline is None: + deadline = now + CAPTCHA_MAX_WAIT_SECONDS + if not presented: + presented = True + if on_captcha is not None: + await on_captcha() + remaining = deadline - now + if remaining <= 0: + raise + utils.logger.warning( + "[XiaoHongShuClient] CAPTCHA appeared; complete verification " + "in Chrome within 5 minutes." + ) + await sleep(min(CAPTCHA_RETRY_SECONDS, remaining)) + + class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): def __init__( @@ -71,6 +118,7 @@ def __init__( self.NOTE_ABNORMAL_CODE = -510001 self.playwright_page = playwright_page self.cookie_dict = cookie_dict + self._captcha_keyword: Optional[str] = None self._extractor = XiaoHongShuExtractor() # Initialize proxy pool (from ProxyRefreshMixin) self.init_proxy_pool(proxy_ip_pool) @@ -112,7 +160,6 @@ async def _pre_headers(self, url: str, params: Optional[Dict] = None, payload: O self.headers.update(headers) return self.headers - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1), retry=retry_if_not_exception_type(NoteNotFoundError)) async def request(self, method, url, **kwargs) -> Union[str, Any]: """ Wrapper for httpx common request method, processes request response @@ -124,34 +171,40 @@ async def request(self, method, url, **kwargs) -> Union[str, Any]: Returns: """ + return_response = kwargs.pop("return_response", False) + + async def operation(): + return await self._request_with_short_retry( + method, url, return_response=return_response, **kwargs + ) + + return await wait_for_captcha(operation, on_captcha=self._present_captcha) + + @retry( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + retry=retry_if_not_exception_type((NoteNotFoundError, CaptchaRequiredError)), + ) + async def _request_with_short_retry( + self, method, url, *, return_response=False, **kwargs + ) -> Union[str, Any]: # Check if proxy is expired before each request await self._refresh_proxy_if_expired() - - # return response.text - return_response = kwargs.pop("return_response", False) async with make_async_client(proxy=self.proxy) as client: response = await client.request(method, url, timeout=self.timeout, **kwargs) - if response.status_code == 471 or response.status_code == 461: - # someday someone maybe will bypass captcha - verify_type = response.headers["Verifytype"] - verify_uuid = response.headers["Verifyuuid"] - msg = f"CAPTCHA appeared, request failed, Verifytype: {verify_type}, Verifyuuid: {verify_uuid}, Response: {response}" - utils.logger.error(msg) - raise Exception(msg) - + raise_for_captcha(response) if return_response: return response.text data: Dict = response.json() if data["success"]: return data.get("data", data.get("success", {})) - elif data["code"] == self.IP_ERROR_CODE: + if data["code"] == self.IP_ERROR_CODE: raise IPBlockError(self.IP_ERROR_STR) - elif data["code"] in (self.NOTE_NOT_FOUND_CODE, self.NOTE_ABNORMAL_CODE): + if data["code"] in (self.NOTE_NOT_FOUND_CODE, self.NOTE_ABNORMAL_CODE): raise NoteNotFoundError(f"Note not found or abnormal, code: {data['code']}") - else: - err_msg = data.get("msg", None) or f"{response.text}" - raise DataFetchError(err_msg) + err_msg = data.get("msg", None) or response.text + raise DataFetchError(err_msg) @staticmethod def _build_query_string(params: Dict) -> str: @@ -162,6 +215,14 @@ def _build_query_string(params: Dict) -> str: parts.append(f"{key}={quote(value_str, safe=',')}") return "&".join(parts) + async def _present_captcha(self, keyword: Optional[str] = None) -> None: + await self.playwright_page.bring_to_front() + keyword = keyword or getattr(self, "_captcha_keyword", None) + if keyword: + await self.playwright_page.goto( + f"{self._domain}/search_result?keyword={quote(keyword, safe='')}" + ) + async def get(self, uri: str, params: Optional[Dict] = None) -> Dict: """ GET request, signs request headers @@ -172,16 +233,29 @@ async def get(self, uri: str, params: Optional[Dict] = None) -> Dict: Returns: """ - headers = await self._pre_headers(uri, params) - # Build URL manually to ensure query string encoding matches the sign string - # (httpx's default params encoding differs from browser/XHS frontend behavior) - if params: - full_url = f"{self._host}{uri}?{self._build_query_string(params)}" - else: - full_url = f"{self._host}{uri}" + async def operation(): + await self.update_cookies(self.playwright_page.context) + headers = await self._pre_headers(uri, params) + # Build URL manually to ensure query string encoding matches the sign string + # (httpx's default params encoding differs from browser/XHS frontend behavior) + if params: + full_url = f"{self._host}{uri}?{self._build_query_string(params)}" + else: + full_url = f"{self._host}{uri}" + return await self._request_with_short_retry( + method="GET", url=full_url, headers=headers + ) - return await self.request( - method="GET", url=full_url, headers=headers + keyword = ( + params.get("keyword") + if uri == "/api/sns/web/v1/search/notes" and isinstance(params, dict) + else None + ) + if isinstance(keyword, str) and keyword: + self._captcha_keyword = keyword + return await wait_for_captcha( + operation, + on_captcha=lambda: self._present_captcha(keyword), ) async def post(self, uri: str, data: dict, **kwargs) -> Dict: @@ -194,14 +268,28 @@ async def post(self, uri: str, data: dict, **kwargs) -> Dict: Returns: """ - headers = await self._pre_headers(uri, payload=data) - json_str = json.dumps(data, separators=(",", ":"), ensure_ascii=False) - return await self.request( - method="POST", - url=f"{self._host}{uri}", - data=json_str, - headers=headers, - **kwargs, + async def operation(): + await self.update_cookies(self.playwright_page.context) + headers = await self._pre_headers(uri, payload=data) + json_str = json.dumps(data, separators=(",", ":"), ensure_ascii=False) + return await self._request_with_short_retry( + method="POST", + url=f"{self._host}{uri}", + data=json_str, + headers=headers, + **kwargs, + ) + + keyword = ( + data.get("keyword") + if uri == "/api/sns/web/v1/search/notes" and isinstance(data, dict) + else None + ) + if isinstance(keyword, str) and keyword: + self._captcha_keyword = keyword + return await wait_for_captcha( + operation, + on_captcha=lambda: self._present_captcha(keyword), ) async def get_note_media(self, url: str) -> Union[bytes, None]: diff --git a/media_platform/xhs/core.py b/media_platform/xhs/core.py index 334fef395..a8e4376e6 100644 --- a/media_platform/xhs/core.py +++ b/media_platform/xhs/core.py @@ -39,6 +39,7 @@ from store import xhs as xhs_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.xhs_creator_id_capture import capture_creator_id from var import crawler_type_var, source_keyword_var from .client import XiaoHongShuClient @@ -154,7 +155,10 @@ async def search(self) -> None: page=page, sort=(SearchSortType(config.SORT_TYPE) if config.SORT_TYPE != "" else SearchSortType.GENERAL), ) - utils.logger.info(f"[XiaoHongShuCrawler.search] Search notes response: {notes_res}") + utils.logger.info( + f"[XiaoHongShuCrawler.search] Search returned " + f"{len(notes_res.get('items', [])) if isinstance(notes_res, dict) else 0} items" + ) if not notes_res or not notes_res.get("has_more", False): utils.logger.info("[XiaoHongShuCrawler.search] No more content!") break @@ -170,12 +174,13 @@ async def search(self) -> None: note_details = await asyncio.gather(*task_list) for note_detail in note_details: if note_detail: + await capture_creator_id(note_detail) await xhs_store.update_xhs_note(note_detail) await self.get_notice_media(note_detail) note_ids.append(note_detail.get("note_id")) xsec_tokens.append(note_detail.get("xsec_token")) page += 1 - utils.logger.info(f"[XiaoHongShuCrawler.search] Note details: {note_details}") + utils.logger.info(f"[XiaoHongShuCrawler.search] Processed note IDs: {note_ids}") await self.batch_get_note_comments(note_ids, xsec_tokens) # Sleep after each page navigation diff --git a/media_platform/xhs/exception.py b/media_platform/xhs/exception.py index a956d936b..57a84fbe2 100644 --- a/media_platform/xhs/exception.py +++ b/media_platform/xhs/exception.py @@ -31,3 +31,7 @@ class IPBlockError(RequestError): class NoteNotFoundError(RequestError): """Note does not exist or is abnormal""" + + +class CaptchaRequiredError(RuntimeError): + """Xiaohongshu requires the user to complete CAPTCHA manually.""" diff --git a/tests/test_api_limits.py b/tests/test_api_limits.py index 0cb65bc8d..f870b5bb4 100644 --- a/tests/test_api_limits.py +++ b/tests/test_api_limits.py @@ -26,6 +26,33 @@ async def test_cmd_arg_crawler_max_notes_count(): config.CRAWLER_MAX_NOTES_COUNT = orig_notes config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = orig_comments + +@pytest.mark.asyncio +async def test_capture_creator_ids_cli_defaults_off_and_can_be_enabled(): + original = config.ENABLE_XHS_CREATOR_ID_CAPTURE + try: + config.ENABLE_XHS_CREATOR_ID_CAPTURE = False + await parse_cmd(["--platform", "xhs"]) + assert config.ENABLE_XHS_CREATOR_ID_CAPTURE is False + + await parse_cmd(["--platform", "xhs", "--capture_creator_ids", "true"]) + assert config.ENABLE_XHS_CREATOR_ID_CAPTURE is True + finally: + config.ENABLE_XHS_CREATOR_ID_CAPTURE = original + + +def test_crawler_manager_only_passes_capture_switch_when_enabled(): + manager = CrawlerManager() + disabled = manager._build_command(CrawlerStartRequest(platform=PlatformEnum.XHS)) + enabled = manager._build_command(CrawlerStartRequest( + platform=PlatformEnum.XHS, + capture_creator_ids=True, + )) + assert "--capture_creator_ids" not in disabled + index = enabled.index("--capture_creator_ids") + assert enabled[index + 1] == "true" + + def test_crawler_manager_build_command(): cm = CrawlerManager() @@ -66,7 +93,7 @@ def test_api_start_crawler_with_limits(): client = TestClient(app) with patch("api.routers.crawler.crawler_manager.start", new_callable=AsyncMock) as mock_start: - mock_start.return_value = True + mock_start.return_value = "task-123" # Test case 1: with limits response = client.post("/api/crawler/start", json={ @@ -79,7 +106,11 @@ def test_api_start_crawler_with_limits(): }) assert response.status_code == 200 - assert response.json() == {"status": "ok", "message": "Crawler started successfully"} + assert response.json() == { + "status": "ok", + "message": "Crawler started successfully", + "task_id": "task-123", + } mock_start.assert_called_once() called_request = mock_start.call_args[0][0] @@ -91,7 +122,7 @@ def test_api_start_crawler_without_limits(): client = TestClient(app) with patch("api.routers.crawler.crawler_manager.start", new_callable=AsyncMock) as mock_start: - mock_start.return_value = True + mock_start.return_value = "task-123" # Test case 2: without limits response = client.post("/api/crawler/start", json={ @@ -102,6 +133,11 @@ def test_api_start_crawler_without_limits(): }) assert response.status_code == 200 + assert response.json() == { + "status": "ok", + "message": "Crawler started successfully", + "task_id": "task-123", + } mock_start.assert_called_once() called_request = mock_start.call_args[0][0] assert called_request.platform == PlatformEnum.XHS diff --git a/tests/test_cdp_browser.py b/tests/test_cdp_browser.py index 66a4b3f41..145def2f6 100644 --- a/tests/test_cdp_browser.py +++ b/tests/test_cdp_browser.py @@ -8,14 +8,14 @@ @pytest.mark.asyncio -async def test_existing_browser_connects_directly_to_devtools_browser(monkeypatch): +async def test_existing_browser_prefers_discovered_websocket_url(monkeypatch, caplog): monkeypatch.setattr(config, "CDP_CONNECT_EXISTING", True) monkeypatch.setattr(config, "BROWSER_LAUNCH_TIMEOUT", 60) manager = CDPBrowserManager() manager.debug_port = 9222 manager._get_browser_websocket_url = AsyncMock( # type: ignore[method-assign] - side_effect=AssertionError("existing browser mode must not call /json/version") + return_value="ws://localhost:9222/devtools/browser/generated-id" ) browser = MagicMock() @@ -27,21 +27,23 @@ async def test_existing_browser_connects_directly_to_devtools_browser(monkeypatc await manager._connect_via_cdp(playwright) + manager._get_browser_websocket_url.assert_awaited_once_with(9222) playwright.chromium.connect_over_cdp.assert_awaited_once_with( - "ws://localhost:9222/devtools/browser", + "ws://localhost:9222/devtools/browser/generated-id", timeout=60000, ) + assert "Please check your browser for a confirmation dialog" not in caplog.text @pytest.mark.asyncio -async def test_existing_browser_falls_back_to_discovered_websocket_url(monkeypatch): +async def test_existing_browser_falls_back_to_direct_when_discovery_fails(monkeypatch): monkeypatch.setattr(config, "CDP_CONNECT_EXISTING", True) monkeypatch.setattr(config, "BROWSER_LAUNCH_TIMEOUT", 60) manager = CDPBrowserManager() manager.debug_port = 9222 manager._get_browser_websocket_url = AsyncMock( # type: ignore[method-assign] - return_value="ws://localhost:9222/devtools/browser/generated-id" + side_effect=RuntimeError("no endpoint") ) browser = MagicMock() @@ -49,25 +51,133 @@ async def test_existing_browser_falls_back_to_discovered_websocket_url(monkeypat browser.contexts = [] playwright = MagicMock() - playwright.chromium.connect_over_cdp = AsyncMock( - side_effect=[RuntimeError("direct websocket failed"), browser] - ) + playwright.chromium.connect_over_cdp = AsyncMock(return_value=browser) await manager._connect_via_cdp(playwright) - manager._get_browser_websocket_url.assert_awaited_once_with(9222) - assert playwright.chromium.connect_over_cdp.await_args_list[0].args == ( + playwright.chromium.connect_over_cdp.assert_awaited_once_with( "ws://localhost:9222/devtools/browser", + timeout=60000, ) - assert playwright.chromium.connect_over_cdp.await_args_list[0].kwargs == { - "timeout": 60000, + + +@pytest.mark.asyncio +async def test_get_browser_websocket_url_uses_ipv4_loopback(monkeypatch): + monkeypatch.setattr(config, "CDP_CONNECT_EXISTING", True) + response = MagicMock() + response.status_code = 200 + response.json.return_value = { + "webSocketDebuggerUrl": "ws://127.0.0.1:9222/devtools/browser/generated-id" } - assert playwright.chromium.connect_over_cdp.await_args_list[1].args == ( - "ws://localhost:9222/devtools/browser/generated-id", + client = MagicMock() + client.get = AsyncMock(return_value=response) + async_client = MagicMock() + async_client.__aenter__ = AsyncMock(return_value=client) + async_client.__aexit__ = AsyncMock(return_value=None) + monkeypatch.setattr("tools.cdp_browser.httpx.AsyncClient", lambda: async_client) + + ws_url = await CDPBrowserManager()._get_browser_websocket_url(9222) + + assert ws_url == "ws://127.0.0.1:9222/devtools/browser/generated-id" + client.get.assert_awaited_once_with( + "http://127.0.0.1:9222/json/version", timeout=10 ) - assert playwright.chromium.connect_over_cdp.await_args_list[1].kwargs == { - "timeout": 60000, + + +@pytest.mark.asyncio +async def test_launched_browser_get_browser_websocket_url_uses_localhost(monkeypatch): + monkeypatch.setattr(config, "CDP_CONNECT_EXISTING", False) + response = MagicMock() + response.status_code = 200 + response.json.return_value = { + "webSocketDebuggerUrl": "ws://localhost:9223/devtools/browser/generated-id" } + client = MagicMock() + client.get = AsyncMock(return_value=response) + async_client = MagicMock() + async_client.__aenter__ = AsyncMock(return_value=client) + async_client.__aexit__ = AsyncMock(return_value=None) + monkeypatch.setattr("tools.cdp_browser.httpx.AsyncClient", lambda: async_client) + + await CDPBrowserManager()._get_browser_websocket_url(9223) + + client.get.assert_awaited_once_with( + "http://localhost:9223/json/version", timeout=10 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "invalid_url", + [ + "ws://", + "ws:///path", + "ws:// host", + "ws://127.0.0.1:9222/dev tools", + "ws://127.0.0.1:9222/path?query=has value", + "ws://127.0.0.1:notaport/path", + "ws://127.0.0.1:65536/path", + ], +) +async def test_existing_browser_falls_back_to_direct_for_malformed_websocket_url( + monkeypatch, invalid_url +): + monkeypatch.setattr(config, "CDP_CONNECT_EXISTING", True) + monkeypatch.setattr(config, "BROWSER_LAUNCH_TIMEOUT", 60) + + response = MagicMock() + response.status_code = 200 + response.json.return_value = {"webSocketDebuggerUrl": invalid_url} + client = MagicMock() + client.get = AsyncMock(return_value=response) + async_client = MagicMock() + async_client.__aenter__ = AsyncMock(return_value=client) + async_client.__aexit__ = AsyncMock(return_value=None) + monkeypatch.setattr("tools.cdp_browser.httpx.AsyncClient", lambda: async_client) + + manager = CDPBrowserManager() + manager.debug_port = 9222 + browser = MagicMock() + browser.is_connected.return_value = True + browser.contexts = [] + playwright = MagicMock() + playwright.chromium.connect_over_cdp = AsyncMock(return_value=browser) + + await manager._connect_via_cdp(playwright) + + playwright.chromium.connect_over_cdp.assert_awaited_once_with( + "ws://localhost:9222/devtools/browser", timeout=60000 + ) + + +@pytest.mark.asyncio +async def test_existing_browser_falls_back_to_direct_for_invalid_discovered_url(monkeypatch): + monkeypatch.setattr(config, "CDP_CONNECT_EXISTING", True) + monkeypatch.setattr(config, "BROWSER_LAUNCH_TIMEOUT", 60) + + response = MagicMock() + response.status_code = 200 + response.json.return_value = {"webSocketDebuggerUrl": "http://127.0.0.1:9222/not-cdp"} + client = MagicMock() + client.get = AsyncMock(return_value=response) + async_client = MagicMock() + async_client.__aenter__ = AsyncMock(return_value=client) + async_client.__aexit__ = AsyncMock(return_value=None) + monkeypatch.setattr("tools.cdp_browser.httpx.AsyncClient", lambda: async_client) + + manager = CDPBrowserManager() + manager.debug_port = 9222 + browser = MagicMock() + browser.is_connected.return_value = True + browser.contexts = [] + playwright = MagicMock() + playwright.chromium.connect_over_cdp = AsyncMock(return_value=browser) + + await manager._connect_via_cdp(playwright) + + playwright.chromium.connect_over_cdp.assert_awaited_once_with( + "ws://localhost:9222/devtools/browser", timeout=60000 + ) @pytest.mark.asyncio diff --git a/tests/test_crawler_task_status.py b/tests/test_crawler_task_status.py new file mode 100644 index 000000000..bff9ab5a9 --- /dev/null +++ b/tests/test_crawler_task_status.py @@ -0,0 +1,289 @@ +import asyncio +from contextlib import suppress +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from api.main import app +from api.schemas import CrawlerStartRequest, PlatformEnum +from api.services.crawler_manager import CrawlerManager + + +def request() -> CrawlerStartRequest: + return CrawlerStartRequest(platform=PlatformEnum.XHS, keywords="重庆企业主资产配置") + + +@pytest.mark.asyncio +async def test_start_returns_task_id_and_clears_completion_state(): + manager = CrawlerManager() + manager.last_exit_code = 1 + manager.finished_at = datetime.now() + process = MagicMock() + process.poll.return_value = None + + with patch("api.services.crawler_manager.subprocess.Popen", return_value=process): + with patch( + "api.services.crawler_manager.asyncio.create_task", + side_effect=lambda coroutine: coroutine.close(), + ): + task_id = await manager.start(request()) + + assert isinstance(task_id, str) + assert task_id == manager.task_id + assert manager.last_exit_code is None + assert manager.finished_at is None + assert manager.status == "running" + + +@pytest.mark.asyncio +async def test_read_output_records_exit_metadata(): + manager = CrawlerManager() + manager.status = "running" + manager.current_config = request() + manager.task_id = "task-123" + process = MagicMock() + process.poll.side_effect = [0, 0] + process.returncode = 0 + process.stdout.readline.return_value = "" + process.stdout.read.return_value = "" + manager.process = process + + await manager._read_output() + + status = manager.get_status() + assert status["status"] == "idle" + assert status["task_id"] == "task-123" + assert status["last_exit_code"] == 0 + assert status["finished_at"] is not None + assert status["platform"] is None + + +@pytest.mark.asyncio +async def test_stop_failure_keeps_active_process_and_reader(): + manager = CrawlerManager() + manager.status = "running" + active_config = request() + manager.current_config = active_config + manager.task_id = "old-task" + process = MagicMock() + process.poll.return_value = None + process.send_signal.side_effect = OSError("terminate failed") + process.kill.side_effect = OSError("kill failed") + manager.process = process + reader = MagicMock() + reader.done.return_value = False + manager._read_task = reader + + assert await manager.stop() is False + + process.kill.assert_called_once_with() + assert manager.status == "error" + assert manager.error_message is not None + assert "terminate failed" in manager.error_message + assert "kill failed" in manager.error_message + assert manager.process is process + assert manager._read_task is reader + assert manager.current_config is active_config + assert manager.last_exit_code is None + assert manager.finished_at is None + reader.cancel.assert_not_called() + + +@pytest.mark.asyncio +async def test_stop_waits_for_killed_process_and_reader_before_restart(): + manager = CrawlerManager() + manager.status = "running" + manager.current_config = request() + manager.task_id = "old-task" + process = MagicMock() + process.returncode = None + process.poll.side_effect = lambda: process.returncode + + def delayed_exit(timeout): + process.returncode = -9 + return process.returncode + + process.wait.side_effect = delayed_exit + manager.process = process + + async def monitor_output(): + await asyncio.Event().wait() + + reader = asyncio.create_task(monitor_output()) + manager._read_task = reader + + with patch( + "api.services.crawler_manager.asyncio.sleep", new_callable=AsyncMock + ): + stopped = await manager.stop() + + try: + assert stopped is True + process.kill.assert_called_once_with() + process.wait.assert_called_once_with(timeout=5) + assert manager.status == "idle" + assert manager.current_config is None + assert manager.last_exit_code == -9 + assert manager.finished_at is not None + assert manager.error_message is None + assert reader.done() + assert reader.cancelled() + assert manager._read_task is None + + new_process = MagicMock() + new_process.poll.return_value = None + with patch( + "api.services.crawler_manager.subprocess.Popen", return_value=new_process + ): + with patch( + "api.services.crawler_manager.asyncio.create_task", + side_effect=lambda coroutine: coroutine.close(), + ): + task_id = await manager.start(request()) + + assert task_id is not None + assert manager.task_id == task_id + finally: + if not reader.done(): + reader.cancel() + with suppress(asyncio.CancelledError): + await reader + + +@pytest.mark.asyncio +async def test_old_reader_uses_its_process_and_cannot_overwrite_new_task(): + manager = CrawlerManager() + manager.status = "running" + manager.current_config = request() + manager.task_id = "old-task" + old_process = MagicMock() + old_process.poll.side_effect = [None, 0] + old_process.returncode = 7 + old_process.stdout.readline.return_value = "old output\n" + old_process.stdout.read.return_value = "" + manager.process = old_process + + new_process = MagicMock() + new_process.poll.return_value = 0 + new_process.returncode = 0 + new_config = request() + + async def switch_to_new_task(_entry): + manager.process = new_process + manager.task_id = "new-task" + manager.status = "running" + manager.current_config = new_config + manager.last_exit_code = None + manager.finished_at = None + + manager._push_log = AsyncMock(side_effect=switch_to_new_task) + + await manager._read_output() + + old_process.stdout.read.assert_called_once_with() + new_process.stdout.read.assert_not_called() + assert manager.process is new_process + assert manager.task_id == "new-task" + assert manager.status == "running" + assert manager.current_config is new_config + assert manager.last_exit_code is None + assert manager.finished_at is None + + +@pytest.mark.asyncio +async def test_read_output_error_records_terminal_metadata(): + manager = CrawlerManager() + manager.status = "running" + manager.current_config = request() + manager.task_id = "task-123" + process = MagicMock() + process.poll.return_value = None + process.returncode = 17 + process.stdout.readline.side_effect = RuntimeError("read failed") + manager.process = process + + await manager._read_output() + + status = manager.get_status() + assert status["status"] == "error" + assert status["task_id"] == "task-123" + assert status["last_exit_code"] == 17 + assert status["finished_at"] is not None + assert status["error_message"] == "Error reading output: read failed" + + +@pytest.mark.asyncio +async def test_old_reader_error_cannot_overwrite_new_task(): + manager = CrawlerManager() + manager.status = "running" + manager.current_config = request() + manager.task_id = "old-task" + old_process = MagicMock() + old_process.poll.return_value = None + manager.process = old_process + + new_process = MagicMock() + new_config = request() + + def fail_after_switch(): + manager.process = new_process + manager.task_id = "new-task" + manager.status = "running" + manager.current_config = new_config + manager.last_exit_code = None + manager.finished_at = None + manager._logs = [] + raise RuntimeError("old reader failed") + + old_process.stdout.readline.side_effect = fail_after_switch + + await manager._read_output() + + assert manager.logs == [] + assert manager.process is new_process + assert manager.task_id == "new-task" + assert manager.status == "running" + assert manager.current_config is new_config + assert manager.last_exit_code is None + assert manager.finished_at is None + + +def test_start_endpoint_returns_task_id(): + client = TestClient(app) + with patch( + "api.routers.crawler.crawler_manager.start", + new_callable=AsyncMock, + return_value="task-123", + ): + response = client.post( + "/api/crawler/start", + json={"platform": "xhs", "keywords": "重庆企业主资产配置"}, + ) + + assert response.status_code == 200 + assert response.json() == { + "status": "ok", + "message": "Crawler started successfully", + "task_id": "task-123", + } + + +def test_status_endpoint_serializes_completion_fields(): + client = TestClient(app) + status = { + "status": "idle", + "platform": None, + "crawler_type": None, + "started_at": "2026-07-27T10:00:00+08:00", + "error_message": None, + "task_id": "task-123", + "last_exit_code": 0, + "finished_at": "2026-07-27T10:02:00+08:00", + } + with patch("api.routers.crawler.crawler_manager.get_status", return_value=status): + response = client.get("/api/crawler/status") + + assert response.status_code == 200 + assert response.json() == status diff --git a/tests/test_xhs_captcha_wait.py b/tests/test_xhs_captcha_wait.py new file mode 100644 index 000000000..e8db4c3a4 --- /dev/null +++ b/tests/test_xhs_captcha_wait.py @@ -0,0 +1,418 @@ +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest +from tenacity import RetryError, wait_none + +from media_platform.xhs.client import XiaoHongShuClient, raise_for_captcha, wait_for_captcha +from media_platform.xhs.exception import CaptchaRequiredError, DataFetchError, NoteNotFoundError + + +async def record_sleep(values, seconds): + values.append(seconds) + + +@pytest.mark.parametrize("status", [461, 471]) +def test_captcha_status_raises_specific_error(status): + response = httpx.Response( + status, + headers={"Verifytype": "slider", "Verifyuuid": "uuid-1"}, + request=httpx.Request("GET", "https://example.test"), + ) + + with pytest.raises(CaptchaRequiredError, match="slider"): + raise_for_captcha(response) + + +@pytest.mark.asyncio +async def test_wait_for_captcha_continues_after_manual_verification(): + operation = AsyncMock( + side_effect=[CaptchaRequiredError("verify"), {"notes": []}] + ) + sleeps = [] + times = iter([0.0, 1.0]) + + result = await wait_for_captcha( + operation, + sleep=lambda seconds: record_sleep(sleeps, seconds), + monotonic=lambda: next(times), + ) + + assert result == {"notes": []} + assert sleeps == [10] + assert operation.await_count == 2 + + +@pytest.mark.asyncio +async def test_wait_for_captcha_calls_presenter_once_for_repeated_captcha(): + operation = AsyncMock( + side_effect=[ + CaptchaRequiredError("verify"), + CaptchaRequiredError("verify"), + {"notes": []}, + ] + ) + presenter = AsyncMock() + sleeps = [] + times = iter([0.0, 1.0]) + + result = await wait_for_captcha( + operation, + on_captcha=presenter, + sleep=lambda seconds: record_sleep(sleeps, seconds), + monotonic=lambda: next(times), + ) + + assert result == {"notes": []} + presenter.assert_awaited_once_with() + assert sleeps == [10, 10] + assert operation.await_count == 3 + + +@pytest.mark.asyncio +async def test_wait_for_captcha_stops_after_five_minutes(): + operation = AsyncMock(side_effect=CaptchaRequiredError("verify")) + times = iter([0.0, 300.0]) + sleep = AsyncMock() + + with pytest.raises(CaptchaRequiredError): + await wait_for_captcha( + operation, sleep=sleep, monotonic=lambda: next(times) + ) + + assert operation.await_count == 2 + sleep.assert_awaited_once_with(10) + + +@pytest.mark.asyncio +async def test_captcha_deadline_starts_when_captcha_is_first_seen(): + clock = {"now": 0.0} + attempts = 0 + sleeps = [] + + async def operation(): + nonlocal attempts + attempts += 1 + if attempts == 1: + clock["now"] = 301.0 + raise CaptchaRequiredError("verify") + return {"notes": []} + + result = await wait_for_captcha( + operation, + sleep=lambda seconds: record_sleep(sleeps, seconds), + monotonic=lambda: clock["now"], + ) + + assert result == {"notes": []} + assert attempts == 2 + assert sleeps == [10] + + +@pytest.mark.asyncio +async def test_wait_for_captcha_does_not_retry_other_errors(): + operation = AsyncMock(side_effect=RuntimeError("network")) + sleep = AsyncMock() + presenter = AsyncMock() + + with pytest.raises(RuntimeError, match="network"): + await wait_for_captcha(operation, sleep=sleep, on_captcha=presenter) + + sleep.assert_not_awaited() + presenter.assert_not_awaited() + assert operation.await_count == 1 + + +class _ResponseClient: + def __init__(self, response): + self.response = response + self.calls = 0 + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return False + + async def request(self, *args, **kwargs): + self.calls += 1 + return self.response + + +class _SequenceResponseClient: + def __init__(self, responses): + self.responses = list(responses) + self.requests = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return False + + async def request(self, *args, **kwargs): + self.requests.append((args, deepcopy(kwargs))) + return self.responses.pop(0) + + +class _CookieContext: + def __init__(self, values): + self.values = list(values) + self.calls = 0 + + async def cookies(self, urls=None): + self.calls += 1 + value = self.values.pop(0) + return [{"name": "a1", "value": value}] + + +def _client_for_request_tests(): + client = XiaoHongShuClient.__new__(XiaoHongShuClient) + client.proxy = None + client.timeout = 1 + client.IP_ERROR_CODE = 300012 + client.NOTE_NOT_FOUND_CODE = -510000 + client.NOTE_ABNORMAL_CODE = -510001 + client.IP_ERROR_STR = "Network connection error" + client._refresh_proxy_if_expired = AsyncMock() + client.playwright_page = SimpleNamespace( + bring_to_front=AsyncMock(), + goto=AsyncMock(), + ) + return client + + +def _client_for_presenter_flow(monkeypatch, responses): + client = _client_for_request_tests() + client._host = "https://example.test" + client._domain = "https://www.xiaohongshu.com" + client.playwright_page.context = object() + client.update_cookies = AsyncMock() + client._pre_headers = AsyncMock(return_value={}) + client._request_with_short_retry = AsyncMock(side_effect=responses) + monkeypatch.setattr( + "media_platform.xhs.client.wait_for_captcha", + lambda operation, **kwargs: wait_for_captcha( + operation, + on_captcha=kwargs.get("on_captcha"), + sleep=AsyncMock(), + monotonic=lambda: 0.0, + ), + ) + return client + + +@pytest.mark.asyncio +async def test_request_waits_for_captcha_instead_of_short_retry(monkeypatch): + captcha = httpx.Response( + 461, + headers={"Verifytype": "slider", "Verifyuuid": "uuid-1"}, + request=httpx.Request("GET", "https://example.test"), + ) + success = httpx.Response( + 200, + json={"success": True, "data": {"notes": []}}, + request=httpx.Request("GET", "https://example.test"), + ) + responses = iter([captcha, success]) + + def make_client(**kwargs): + return _ResponseClient(next(responses)) + + sleeps = [] + times = iter([0.0, 1.0]) + monkeypatch.setattr("media_platform.xhs.client.make_async_client", make_client) + monkeypatch.setattr( + "media_platform.xhs.client.wait_for_captcha", + lambda operation, **kwargs: wait_for_captcha( + operation, + on_captcha=kwargs.get("on_captcha"), + sleep=lambda seconds: record_sleep(sleeps, seconds), + monotonic=lambda: next(times), + ), + ) + + client = _client_for_request_tests() + result = await client.request("GET", "https://example.test") + + assert result == {"notes": []} + assert sleeps == [10] + client.playwright_page.bring_to_front.assert_awaited_once_with() + client.playwright_page.goto.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["get", "post"]) +async def test_signed_requests_refresh_cookies_and_signature_after_captcha( + monkeypatch, method +): + captcha = httpx.Response( + 461, + headers={"Verifytype": "slider", "Verifyuuid": "uuid-1"}, + request=httpx.Request(method.upper(), "https://example.test"), + ) + success = httpx.Response( + 200, + json={"success": True, "data": {"notes": []}}, + request=httpx.Request(method.upper(), "https://example.test"), + ) + response_client = _SequenceResponseClient([captcha, success]) + context = _CookieContext(["old", "new"]) + client = _client_for_request_tests() + client._host = "https://example.test" + client._domain = "https://www.xiaohongshu.com" + client.cookie_urls = [client._domain] + client.headers = {"Cookie": "a1=initial"} + client.cookie_dict = {"a1": "initial"} + client.playwright_page = SimpleNamespace( + context=context, + bring_to_front=AsyncMock(), + goto=AsyncMock(), + ) + + def sign_with_cookie(*, uri, data, cookie_str, method): + return { + "x-s": f"xs-{cookie_str}", + "x-t": f"xt-{cookie_str}", + "x-s-common": f"common-{cookie_str}", + "x-b3-traceid": f"trace-{cookie_str}", + } + + sleeps = [] + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", lambda **kwargs: response_client + ) + monkeypatch.setattr("media_platform.xhs.client.sign_with_xhshow", sign_with_cookie) + monkeypatch.setattr( + "media_platform.xhs.client.wait_for_captcha", + lambda operation, **kwargs: wait_for_captcha( + operation, + on_captcha=kwargs.get("on_captcha"), + sleep=lambda seconds: record_sleep(sleeps, seconds), + monotonic=lambda: 0.0, + ), + ) + + if method == "get": + result = await client.get( + "/api/sns/web/v1/search/notes", {"keyword": "重庆 火锅"} + ) + else: + result = await client.post( + "/api/sns/web/v1/search/notes", {"keyword": "重庆 火锅"} + ) + + request_headers = [call[1]["headers"] for call in response_client.requests] + assert result == {"notes": []} + assert context.calls == 2 + assert client.cookie_dict == {"a1": "new"} + assert [headers["Cookie"] for headers in request_headers] == ["a1=old", "a1=new"] + assert [headers["X-T"] for headers in request_headers] == ["xt-a1=old", "xt-a1=new"] + client.playwright_page.bring_to_front.assert_awaited_once_with() + client.playwright_page.goto.assert_awaited_once_with( + "https://www.xiaohongshu.com/search_result?keyword=%E9%87%8D%E5%BA%86%20%E7%81%AB%E9%94%85" + ) + assert sleeps == [10] + + +@pytest.mark.asyncio +async def test_detail_captcha_reuses_latest_search_keyword(monkeypatch): + client = _client_for_presenter_flow( + monkeypatch, + [ + {"notes": []}, + CaptchaRequiredError("verify"), + {"items": []}, + ], + ) + + await client.post( + "/api/sns/web/v1/search/notes", {"keyword": "重庆 亲子游"} + ) + await client.post("/api/sns/web/v1/feed", {"source_note_id": "note-1"}) + + client.playwright_page.bring_to_front.assert_awaited_once_with() + client.playwright_page.goto.assert_awaited_once_with( + "https://www.xiaohongshu.com/search_result?keyword=%E9%87%8D%E5%BA%86%20%E4%BA%B2%E5%AD%90%E6%B8%B8" + ) + + +@pytest.mark.asyncio +async def test_new_search_keyword_overwrites_presenter_history(monkeypatch): + client = _client_for_presenter_flow( + monkeypatch, + [ + {"notes": []}, + {"notes": []}, + CaptchaRequiredError("verify"), + {"items": []}, + ], + ) + + await client.post("/api/sns/web/v1/search/notes", {"keyword": "旧词"}) + await client.post("/api/sns/web/v1/search/notes", {"keyword": "新词"}) + await client.post("/api/sns/web/v1/feed", {"source_note_id": "note-1"}) + + client.playwright_page.goto.assert_awaited_once_with( + "https://www.xiaohongshu.com/search_result?keyword=%E6%96%B0%E8%AF%8D" + ) + + +@pytest.mark.asyncio +async def test_detail_captcha_without_search_history_only_brings_page_forward( + monkeypatch, +): + client = _client_for_presenter_flow( + monkeypatch, + [CaptchaRequiredError("verify"), {"items": []}], + ) + + await client.post("/api/sns/web/v1/feed", {"source_note_id": "note-1"}) + + client.playwright_page.bring_to_front.assert_awaited_once_with() + client.playwright_page.goto.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_short_retry_keeps_retry_error_for_data_fetch_errors(monkeypatch): + response = httpx.Response( + 200, + json={"success": False, "code": 500, "msg": "network"}, + request=httpx.Request("GET", "https://example.test"), + ) + response_client = _ResponseClient(response) + client = _client_for_request_tests() + previous_wait = XiaoHongShuClient._request_with_short_retry.retry.wait + XiaoHongShuClient._request_with_short_retry.retry.wait = wait_none() + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", lambda **kwargs: response_client + ) + try: + with pytest.raises(RetryError) as exc_info: + await client._request_with_short_retry("GET", "https://example.test") + finally: + XiaoHongShuClient._request_with_short_retry.retry.wait = previous_wait + + assert isinstance(exc_info.value.last_attempt.exception(), DataFetchError) + assert response_client.calls == 3 + + +@pytest.mark.asyncio +async def test_short_retry_does_not_retry_note_not_found(monkeypatch): + response = httpx.Response( + 200, + json={"success": False, "code": -510000, "msg": "missing"}, + request=httpx.Request("GET", "https://example.test"), + ) + response_client = _ResponseClient(response) + client = _client_for_request_tests() + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", lambda **kwargs: response_client + ) + + with pytest.raises(NoteNotFoundError): + await client._request_with_short_retry("GET", "https://example.test") + + assert response_client.calls == 1 diff --git a/tests/test_xhs_creator_id_capture.py b/tests/test_xhs_creator_id_capture.py new file mode 100644 index 000000000..0517132ee --- /dev/null +++ b/tests/test_xhs_creator_id_capture.py @@ -0,0 +1,142 @@ +import json +from unittest.mock import AsyncMock + +import pytest + +import config +from media_platform.xhs import core as xhs_core +from media_platform.xhs.core import XiaoHongShuCrawler +from tools import xhs_creator_id_capture as capture_module +from tools.xhs_creator_id_capture import build_capture_record, capture_creator_id + + +VALID_ID = "5eb8e1d400000000010075ae" + + +def test_build_capture_record_contains_only_allowed_fields(): + record = build_capture_record({ + "note_id": "note-1", + "user": { + "user_id": VALID_ID, + "nickname": "不得保存", + "avatar": "https://private.invalid/avatar", + }, + "xsec_token": "secret-token", + }, captured_at="2026-07-31T10:00:00+08:00") + + assert record == { + "note_id": "note-1", + "creator_hash": "fb9185b3a79e6291", + "public_user_id": VALID_ID, + "profile_url": f"https://www.xiaohongshu.com/user/profile/{VALID_ID}", + "captured_at": "2026-07-31T10:00:00+08:00", + } + + +@pytest.mark.asyncio +async def test_capture_is_disabled_by_default_and_idempotent_when_enabled(tmp_path, monkeypatch): + monkeypatch.setattr(config, "ENABLE_XHS_CREATOR_ID_CAPTURE", False) + monkeypatch.setattr(config, "XHS_CREATOR_ID_CAPTURE_DIR", str(tmp_path)) + detail = {"note_id": "note-1", "user": {"user_id": VALID_ID}} + + await capture_creator_id(detail) + assert list(tmp_path.iterdir()) == [] + + monkeypatch.setattr(config, "ENABLE_XHS_CREATOR_ID_CAPTURE", True) + await capture_creator_id(detail) + await capture_creator_id(detail) + records = json.loads(next(tmp_path.iterdir()).read_text(encoding="utf-8")) + assert len(records) == 1 + assert records[0]["public_user_id"] == VALID_ID + + +def test_invalid_user_id_is_not_captured(): + assert build_capture_record({"note_id": "note-1", "user": {"user_id": "short"}}) is None + + +@pytest.mark.asyncio +async def test_invalid_user_id_writes_only_allowlisted_error_and_returns_normally( + tmp_path, monkeypatch +): + invalid_id = "INVALID-RAW-ID-MUST-NOT-LEAK" + monkeypatch.setattr(config, "ENABLE_XHS_CREATOR_ID_CAPTURE", True) + monkeypatch.setattr(config, "XHS_CREATOR_ID_CAPTURE_DIR", str(tmp_path)) + monkeypatch.setattr(capture_module.utils, "get_current_date", lambda: "2026-07-31") + + await capture_creator_id({ + "note_id": "note-1", + "user": {"user_id": invalid_id, "nickname": "must-not-leak"}, + "xsec_token": "must-not-leak", + }) + + error_path = tmp_path / "search_creator_id_errors_2026-07-31.json" + records = json.loads(error_path.read_text(encoding="utf-8")) + assert len(records) == 1 + assert records[0]["note_id"] == "note-1" + assert records[0]["error"] == "missing_or_invalid_user_id" + assert isinstance(records[0]["captured_at"], str) + assert set(records[0]) == {"note_id", "error", "captured_at"} + assert invalid_id not in error_path.read_text(encoding="utf-8") + + +@pytest.mark.asyncio +async def test_invalid_user_id_sidecar_io_failure_is_raised(tmp_path, monkeypatch): + monkeypatch.setattr(config, "ENABLE_XHS_CREATOR_ID_CAPTURE", True) + monkeypatch.setattr(config, "XHS_CREATOR_ID_CAPTURE_DIR", str(tmp_path)) + + async def fail_to_thread(*args, **kwargs): + raise OSError("private sidecar unavailable") + + monkeypatch.setattr(capture_module.asyncio, "to_thread", fail_to_thread) + + with pytest.raises(OSError, match="private sidecar unavailable"): + await capture_creator_id({ + "note_id": "note-1", + "user": {"user_id": "INVALID-RAW-ID-MUST-NOT-LEAK"}, + }) + + +@pytest.mark.asyncio +async def test_invalid_user_id_error_does_not_block_anonymous_note_storage( + tmp_path, monkeypatch +): + log_messages = [] + + class Logger: + def info(self, message): + log_messages.append(str(message)) + + detail = { + "note_id": "note-1", + "user": {"user_id": "INVALID-RAW-ID-MUST-NOT-LEAK"}, + "xsec_token": "token", + } + monkeypatch.setattr(config, "ENABLE_XHS_CREATOR_ID_CAPTURE", True) + monkeypatch.setattr(config, "XHS_CREATOR_ID_CAPTURE_DIR", str(tmp_path)) + monkeypatch.setattr(config, "KEYWORDS", "keyword") + monkeypatch.setattr(config, "CRAWLER_MAX_NOTES_COUNT", 20) + monkeypatch.setattr(config, "START_PAGE", 1) + monkeypatch.setattr(config, "SORT_TYPE", "") + monkeypatch.setattr(config, "MAX_CONCURRENCY_NUM", 1) + monkeypatch.setattr(config, "CRAWLER_MAX_SLEEP_SEC", 0) + + crawler = XiaoHongShuCrawler() + crawler.xhs_client = type("Client", (), { + "get_note_by_keyword": AsyncMock(return_value={ + "has_more": True, + "items": [{"id": "note-1", "xsec_source": "search", "xsec_token": "token"}], + }) + })() + crawler.get_note_detail_async_task = AsyncMock(return_value=detail) + crawler.get_notice_media = AsyncMock() + crawler.batch_get_note_comments = AsyncMock() + store_note = AsyncMock() + monkeypatch.setattr(xhs_core.xhs_store, "update_xhs_note", store_note) + monkeypatch.setattr(xhs_core.asyncio, "sleep", AsyncMock()) + monkeypatch.setattr(xhs_core.utils, "logger", Logger()) + + await crawler.search() + + store_note.assert_awaited_once_with(detail) + assert len(list(tmp_path.glob("search_creator_id_errors_*.json"))) == 1 + assert "INVALID-RAW-ID-MUST-NOT-LEAK" not in "\n".join(log_messages) diff --git a/tools/cdp_browser.py b/tools/cdp_browser.py index bc5a76c12..a61b174f8 100644 --- a/tools/cdp_browser.py +++ b/tools/cdp_browser.py @@ -25,6 +25,7 @@ import signal import atexit from typing import Optional, Dict, Any +from urllib.parse import urlparse from playwright.async_api import Browser, BrowserContext, Playwright import config @@ -285,25 +286,45 @@ async def _launch_browser(self, browser_path: str, headless: bool): "[CDPBrowserManager] CDP connection test failed, but will continue to try connecting" ) + @staticmethod + def _is_valid_websocket_url(ws_url: Any) -> bool: + if not isinstance(ws_url, str) or any(character.isspace() for character in ws_url): + return False + + try: + parsed_url = urlparse(ws_url) + port = parsed_url.port + except ValueError: + return False + + if parsed_url.scheme not in ("ws", "wss") or not parsed_url.hostname: + return False + + if any(character.isspace() for character in parsed_url.netloc): + return False + + host_port = parsed_url.netloc.rsplit("@", 1)[-1] + return port is not None or not host_port.endswith(":") + async def _get_browser_websocket_url(self, debug_port: int) -> str: """ Get browser WebSocket connection URL """ try: + discovery_host = "127.0.0.1" if config.CDP_CONNECT_EXISTING else "localhost" async with httpx.AsyncClient() as client: response = await client.get( - f"http://localhost:{debug_port}/json/version", timeout=10 + f"http://{discovery_host}:{debug_port}/json/version", timeout=10 ) if response.status_code == 200: data = response.json() ws_url = data.get("webSocketDebuggerUrl") - if ws_url: + if self._is_valid_websocket_url(ws_url): utils.logger.info( f"[CDPBrowserManager] Got browser WebSocket URL: {ws_url}" ) return ws_url - else: - raise RuntimeError("webSocketDebuggerUrl not found") + raise RuntimeError("valid webSocketDebuggerUrl not found") else: raise RuntimeError(f"HTTP {response.status_code}: {response.text}") except Exception as e: @@ -316,29 +337,21 @@ async def _connect_via_cdp(self, playwright: Playwright): """ try: if config.CDP_CONNECT_EXISTING: - # Existing browser remote debugging in Chrome 136+ does not expose - # /json/version. Connect directly and wait for user confirmation. - ws_url = f"ws://localhost:{self.debug_port}/devtools/browser" - utils.logger.info(f"[CDPBrowserManager] Connecting to existing browser via CDP: {ws_url}") - utils.logger.info( - "[CDPBrowserManager] Please check your browser for a confirmation dialog and accept it" - ) try: - self.browser = await playwright.chromium.connect_over_cdp( - ws_url, timeout=config.BROWSER_LAUNCH_TIMEOUT * 1000 - ) - except Exception as direct_error: - utils.logger.warning( - "[CDPBrowserManager] Direct existing-browser CDP connection failed: " - f"{direct_error}. Trying /json/version discovery..." - ) ws_url = await self._get_browser_websocket_url(self.debug_port) utils.logger.info( - f"[CDPBrowserManager] Connecting to existing browser via discovered CDP: {ws_url}" + f"[CDPBrowserManager] Connecting via discovered CDP URL: {ws_url}" ) - self.browser = await playwright.chromium.connect_over_cdp( - ws_url, timeout=config.BROWSER_LAUNCH_TIMEOUT * 1000 + except Exception as discovery_error: + ws_url = f"ws://localhost:{self.debug_port}/devtools/browser" + utils.logger.warning( + "[CDPBrowserManager] /json/version unavailable; falling back to " + "Chrome 136+ direct CDP mode, which may require browser approval: " + f"{discovery_error}" ) + self.browser = await playwright.chromium.connect_over_cdp( + ws_url, timeout=config.BROWSER_LAUNCH_TIMEOUT * 1000 + ) else: # For launched browser, get WebSocket URL first ws_url = await self._get_browser_websocket_url(self.debug_port) diff --git a/tools/xhs_creator_id_capture.py b/tools/xhs_creator_id_capture.py new file mode 100644 index 000000000..1efb5e7cc --- /dev/null +++ b/tools/xhs_creator_id_capture.py @@ -0,0 +1,80 @@ +import asyncio +import json +import os +import re +import tempfile +from datetime import datetime +from pathlib import Path + +import config +from tools import utils +from tools.user_hash import anonymize_user_id + + +_USER_ID = re.compile(r"^[0-9a-f]{24}$") +_LOCK = asyncio.Lock() + + +def build_capture_record(note_detail: dict, captured_at: str | None = None) -> dict | None: + note_id = str(note_detail.get("note_id") or "").strip() + user = note_detail.get("user") + user_id = str(user.get("user_id") or "").strip() if isinstance(user, dict) else "" + if not note_id or not _USER_ID.fullmatch(user_id): + return None + return { + "note_id": note_id, + "creator_hash": anonymize_user_id(user_id), + "public_user_id": user_id, + "profile_url": f"https://www.xiaohongshu.com/user/profile/{user_id}", + "captured_at": captured_at or datetime.now().astimezone().isoformat(timespec="seconds"), + } + + +def _capture_path() -> Path: + base = Path(config.XHS_CREATOR_ID_CAPTURE_DIR) if config.XHS_CREATOR_ID_CAPTURE_DIR else Path("data/xhs/private") + return base / f"search_creator_ids_{utils.get_current_date()}.json" + + +def _capture_error_path() -> Path: + return _capture_path().with_name( + f"search_creator_id_errors_{utils.get_current_date()}.json" + ) + + +def _write_record( + path: Path, record: dict, key_fields: tuple[str, ...] = ("note_id", "public_user_id") +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + records = json.loads(path.read_text(encoding="utf-8")) if path.exists() else [] + keyed = {tuple(item[field] for field in key_fields): item for item in records} + keyed[tuple(record[field] for field in key_fields)] = record + fd, temporary_name = tempfile.mkstemp(dir=path.parent, prefix=path.name, suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as output: + json.dump(list(keyed.values()), output, ensure_ascii=False, indent=2) + os.replace(temporary_name, path) + finally: + if os.path.exists(temporary_name): + os.unlink(temporary_name) + + +async def capture_creator_id(note_detail: dict) -> None: + if not config.ENABLE_XHS_CREATOR_ID_CAPTURE: + return + record = build_capture_record(note_detail) + if record is None: + note_id = str(note_detail.get("note_id") or "").strip() + if not note_id: + return + record = { + "note_id": note_id, + "error": "missing_or_invalid_user_id", + "captured_at": datetime.now().astimezone().isoformat(timespec="seconds"), + } + path = _capture_error_path() + key_fields = ("note_id", "error") + else: + path = _capture_path() + key_fields = ("note_id", "public_user_id") + async with _LOCK: + await asyncio.to_thread(_write_record, path, record, key_fields)