From 6c0094c9c63ea9e0d6f5f69ec0acd9e278758e36 Mon Sep 17 00:00:00 2001 From: GdoongMathew Date: Wed, 5 Aug 2026 16:00:05 +0800 Subject: [PATCH 1/6] refactor: update git repository cloning mechanism. --- api/poetry.lock | 50 ++++- api/pyproject.toml | 1 + api/repository.py | 524 ++++++++++----------------------------------- 3 files changed, 166 insertions(+), 409 deletions(-) diff --git a/api/poetry.lock b/api/poetry.lock index 153ecd4a5..8af77dff8 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1003,6 +1003,40 @@ files = [ {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, ] +[[package]] +name = "gitdb" +version = "4.0.12" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, + {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.58" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "gitpython-3.1.58-py3-none-any.whl", hash = "sha256:d331e722577f0fd7fc1f857419b3ecc07af66282b933d2a4d95f84a042fdd50f"}, + {file = "gitpython-3.1.58.tar.gz", hash = "sha256:621416df10ef3fd0e19fabf9172ddeed0fa704d353d04f194eec56a625a95b22"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + +[package.extras] +doc = ["sphinx (>=7.4.7,<8)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] +test = ["basedpyright (==1.39.9) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy (==1.18.2) ; python_version >= \"3.9\"", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] + [[package]] name = "google-ai-generativelanguage" version = "0.6.15" @@ -1071,7 +1105,7 @@ googleapis-common-protos = ">=1.56.2,<2.0.0" grpcio = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\" and python_version < \"3.14\""} grpcio-status = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} proto-plus = [ - {version = ">=1.22.3,<2.0.0", markers = "python_version < \"3.13\""}, + {version = ">=1.24.0,<2.0.0", markers = "python_version < \"3.13\""}, {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" @@ -2828,6 +2862,18 @@ files = [ {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] +[[package]] +name = "smmap" +version = "5.0.3" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f"}, + {file = "smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c"}, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -3452,4 +3498,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "86ee62820be3180e14bcd1bb48607994e24ba6648e13a346b8cee26fcdc89a06" +content-hash = "530d6dc9f317228da098c01ccf1e0fee602cb197b115a272cfa171fb8ad3f2f7" diff --git a/api/pyproject.toml b/api/pyproject.toml index ad1f5d32c..5cdf552b7 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -29,6 +29,7 @@ websockets = ">=11.0.3" azure-identity = ">=1.12.0" azure-core = ">=1.24.0" anthropic = "^0.117.1" +gitpython = "^3.1.58" [build-system] diff --git a/api/repository.py b/api/repository.py index a4b4fc0dc..9f7c4738d 100644 --- a/api/repository.py +++ b/api/repository.py @@ -1,11 +1,10 @@ -import base64 -import json import os import subprocess +from functools import wraps +from collections.abc import Callable from urllib.parse import quote, urlparse, urlunparse -import requests -from requests.exceptions import RequestException +from git import Repo as GitRepo, GIT_OK from api.logger import get_logger from api.utils import deepwiki_root @@ -16,416 +15,102 @@ CLONE_REPO_ROOT = os.path.join(deepwiki_root(), "repo") -def _get_github_file_content( - repo_url: str, file_path: str, access_token: str = None -) -> str: - """ - Retrieves the content of a file from a GitHub repository using the GitHub API. - Supports both public GitHub (github.com) and GitHub Enterprise (custom domains). - - Args: - repo_url (str): The URL of the GitHub repository - (e.g., "https://github.com/username/repo" or "https://github.company.com/username/repo") - file_path (str): The path to the file within the repository (e.g., "src/main.py") - access_token (str, optional): GitHub personal access token for private repositories - - Returns: - str: The content of the file as a string - - Raises: - ValueError: If the file cannot be fetched or if the URL is not a valid GitHub URL - """ - try: - # Parse the repository URL to support both github.com and enterprise GitHub - parsed_url = urlparse(repo_url) - if not parsed_url.scheme or not parsed_url.netloc: - raise ValueError("Not a valid GitHub repository URL") - - # Check if it's a GitHub-like URL structure - path_parts = parsed_url.path.strip("/").split("/") - if len(path_parts) < 2: - raise ValueError( - "Invalid GitHub URL format - expected format: https://domain/owner/repo" - ) - - owner = path_parts[-2] - repo = path_parts[-1].replace(".git", "") - - # Determine the API base URL - if parsed_url.netloc == "github.com": - # Public GitHub - api_base = "https://api.github.com" - else: - # GitHub Enterprise - API is typically at https://domain/api/v3/ - api_base = f"{parsed_url.scheme}://{parsed_url.netloc}/api/v3" - - # Use GitHub API to get file content - # The API endpoint for getting file content is: /repos/{owner}/{repo}/contents/{path} - api_url = f"{api_base}/repos/{owner}/{repo}/contents/{file_path}" - - # Fetch file content from GitHub API - headers = {} - if access_token: - headers["Authorization"] = f"token {access_token}" - logger.info(f"Fetching file content from GitHub API: {api_url}") +def _exception_cleanup(func: Callable) -> Callable: + @wraps + def wrapper(*args, **kwargs): try: - response = requests.get(api_url, headers=headers) - response.raise_for_status() - except RequestException as e: - raise ValueError(f"Error fetching file content: {e}") - try: - content_data = response.json() - except json.JSONDecodeError: - raise ValueError("Invalid response from GitHub API") - - # Check if we got an error response - if "message" in content_data and "documentation_url" in content_data: - raise ValueError(f"GitHub API error: {content_data['message']}") - - # GitHub API returns file content as base64 encoded string - if "content" in content_data and "encoding" in content_data: - if content_data["encoding"] == "base64": - # The content might be split into lines, so join them first - content_base64 = content_data["content"].replace("\n", "") - content = base64.b64decode(content_base64).decode("utf-8") - return content - else: - raise ValueError(f"Unexpected encoding: {content_data['encoding']}") - else: - raise ValueError("File content not found in GitHub API response") - - except Exception as e: - raise ValueError(f"Failed to get file content: {str(e)}") - - -def _get_gitlab_file_content( - repo_url: str, file_path: str, access_token: str = None -) -> str: - """ - Retrieves the content of a file from a GitLab repository (cloud or self-hosted). - - Args: - repo_url (str): The GitLab repo URL (e.g., "https://gitlab.com/username/repo" or "http://localhost/group/project") - file_path (str): File path within the repository (e.g., "src/main.py") - access_token (str, optional): GitLab personal access token - - Returns: - str: File content - - Raises: - ValueError: If anything fails - """ - try: - # Parse and validate the URL - parsed_url = urlparse(repo_url) - if not parsed_url.scheme or not parsed_url.netloc: - raise ValueError("Not a valid GitLab repository URL") - - gitlab_domain = f"{parsed_url.scheme}://{parsed_url.netloc}" - if parsed_url.port not in (None, 80, 443): - gitlab_domain += f":{parsed_url.port}" - path_parts = parsed_url.path.strip("/").split("/") - if len(path_parts) < 2: - raise ValueError( - "Invalid GitLab URL format — expected something like https://gitlab.domain.com/group/project" - ) - - # Build project path and encode for API - project_path = "/".join(path_parts).replace(".git", "") - encoded_project_path = quote(project_path, safe="") - - # Encode file path - encoded_file_path = quote(file_path, safe="") - - # Try to get the default branch from the project info - default_branch = None - try: - project_info_url = f"{gitlab_domain}/api/v4/projects/{encoded_project_path}" - project_headers = {} - if access_token: - project_headers["PRIVATE-TOKEN"] = access_token - - project_response = requests.get(project_info_url, headers=project_headers) - if project_response.status_code == 200: - project_data = project_response.json() - default_branch = project_data.get("default_branch", "main") - logger.info(f"Found default branch: {default_branch}") - else: - logger.warning( - "Could not fetch project info, using 'main' as default branch" - ) - default_branch = "main" - except Exception as e: - logger.warning( - f"Error fetching project info: {e}, using 'main' as default branch" + return func(*args, **kwargs) + except subprocess.CalledProcessError as e: + err_msg = e.stderr.decode("utf-8") + token = kwargs.get("access_token", None) + if token: + token_mask = "***TOKEN***" + err_msg.replace(token, token_mask) + encoded_token = quote(token, safe="") + err_msg = err_msg.replace(encoded_token, token_mask) + raise ValueError(err_msg) + + return wrapper + + +@_exception_cleanup +def _clone_from_gitlab( + remote_url: str, + local_path: str, + *, + access_token: str | None = None, +) -> GitRepo: + if access_token: + parsed = urlparse(remote_url) + + remote_url = urlunparse( + ( + parsed.scheme, + f"oauth2:{quote(access_token, safe="")}@{parsed.netloc}", + parsed.path, + "", + "", + "", ) - default_branch = "main" - - api_url = f"{gitlab_domain}/api/v4/projects/{encoded_project_path}/repository/files/{encoded_file_path}/raw?ref={default_branch}" - # Fetch file content from GitLab API - headers = {} - if access_token: - headers["PRIVATE-TOKEN"] = access_token - logger.info(f"Fetching file content from GitLab API: {api_url}") - try: - response = requests.get(api_url, headers=headers) - response.raise_for_status() - content = response.text - except RequestException as e: - raise ValueError(f"Error fetching file content: {e}") - - # Check for GitLab error response (JSON instead of raw file) - if content.startswith("{") and '"message":' in content: - try: - error_data = json.loads(content) - if "message" in error_data: - raise ValueError(f"GitLab API error: {error_data['message']}") - except json.JSONDecodeError: - pass - - return content - - except Exception as e: - raise ValueError(f"Failed to get file content: {str(e)}") - - -def _get_bitbucket_file_content( - repo_url: str, file_path: str, access_token: str = None -) -> str: - """ - Retrieves the content of a file from a Bitbucket repository using the Bitbucket API. - - Args: - repo_url (str): The URL of the Bitbucket repository (e.g., "https://bitbucket.org/username/repo") - file_path (str): The path to the file within the repository (e.g., "src/main.py") - access_token (str, optional): Bitbucket personal access token for private repositories + ) - Returns: - str: The content of the file as a string - """ - try: - # Extract owner and repo name from Bitbucket URL - if not ( - repo_url.startswith("https://bitbucket.org/") - or repo_url.startswith("http://bitbucket.org/") - ): - raise ValueError("Not a valid Bitbucket repository URL") - - parts = repo_url.rstrip("/").split("/") - if len(parts) < 5: - raise ValueError("Invalid Bitbucket URL format") - - owner = parts[-2] - repo = parts[-1].replace(".git", "") - - # Try to get the default branch from the repository info - default_branch = None - try: - repo_info_url = f"https://api.bitbucket.org/2.0/repositories/{owner}/{repo}" - repo_headers = {} - if access_token: - repo_headers["Authorization"] = f"Bearer {access_token}" - - repo_response = requests.get(repo_info_url, headers=repo_headers) - if repo_response.status_code == 200: - repo_data = repo_response.json() - default_branch = repo_data.get("mainbranch", {}).get("name", "main") - logger.info(f"Found default branch: {default_branch}") - else: - logger.warning( - "Could not fetch repository info, using 'main' as default branch" - ) - default_branch = "main" - except Exception as e: - logger.warning( - f"Error fetching repository info: {e}, using 'main' as default branch" + return GitRepo.clone_from(url=remote_url, to_path=local_path) + + +@_exception_cleanup +def _clone_from_github( + remote_url: str, + local_path: str, + *, + access_token: str | None = None, +) -> GitRepo: + if access_token: + parsed = urlparse(remote_url) + + remote_url = urlunparse( + ( + parsed.scheme, + f"{quote(access_token, safe="")}@{parsed.netloc}", + parsed.path, + "", + "", + "", ) - default_branch = "main" - - # Use Bitbucket API to get file content - # The API endpoint for getting file content is: /2.0/repositories/{owner}/{repo}/src/{branch}/{path} - api_url = f"https://api.bitbucket.org/2.0/repositories/{owner}/{repo}/src/{default_branch}/{file_path}" - - # Fetch file content from Bitbucket API - headers = {} - if access_token: - headers["Authorization"] = f"Bearer {access_token}" - logger.info(f"Fetching file content from Bitbucket API: {api_url}") - try: - response = requests.get(api_url, headers=headers) - if response.status_code == 200: - content = response.text - elif response.status_code == 404: - raise ValueError( - "File not found on Bitbucket. Please check the file path and repository." - ) - elif response.status_code == 401: - raise ValueError( - "Unauthorized access to Bitbucket. Please check your access token." - ) - elif response.status_code == 403: - raise ValueError( - "Forbidden access to Bitbucket. You might not have permission to access this file." - ) - elif response.status_code == 500: - raise ValueError( - "Internal server error on Bitbucket. Please try again later." - ) - else: - response.raise_for_status() - content = response.text - return content - except RequestException as e: - raise ValueError(f"Error fetching file content: {e}") - - except Exception as e: - raise ValueError(f"Failed to get file content: {str(e)}") - - -def get_repo_content( - repo_url: str, file_path: str, repo_type: str = None, access_token: str = None -) -> str: - """ - Retrieves the content of a file from a Git repository (GitHub or GitLab). - - Args: - repo_type (str): Type of repository - repo_url (str): The URL of the repository - file_path (str): The path to the file within the repository - access_token (str, optional): Access token for private repositories - - Returns: - str: The content of the file as a string - - Raises: - ValueError: If the file cannot be fetched or if the URL is not valid - """ - if repo_type == "github": - return _get_github_file_content(repo_url, file_path, access_token) - elif repo_type == "gitlab": - return _get_gitlab_file_content(repo_url, file_path, access_token) - elif repo_type == "bitbucket": - return _get_bitbucket_file_content(repo_url, file_path, access_token) - else: - raise ValueError( - "Unsupported repository type. Only GitHub, GitLab, and Bitbucket are supported." ) - -def download_repo( - repo_url: str, local_path: str, repo_type: str = None, access_token: str = None -) -> str: - """ - Downloads a Git repository (GitHub, GitLab, or Bitbucket) to a specified local path. - - Args: - repo_type(str): Type of repository - repo_url (str): The URL of the Git repository to clone. - local_path (str): The local directory where the repository will be cloned. - access_token (str, optional): Access token for private repositories. - - Returns: - str: The output message from the `git` command. - """ - try: - # Check if Git is installed - logger.info(f"Preparing to clone repository to {local_path}") - subprocess.run( - ["git", "--version"], - check=True, - capture_output=True, + return GitRepo.clone_from(url=remote_url, to_path=local_path) + + +@_exception_cleanup +def _clone_from_bitbucket( + remote_url: str, + local_path: str, + *, + access_token: str | None = None, +) -> GitRepo: + if access_token: + parsed = urlparse(remote_url) + # Bitbucket has two token formats with different auth schemes: + # - HTTP access tokens (prefix "ATCTT") use x-bitbucket-api-token-auth + # - App passwords (deprecated, EOL June 2026) use x-token-auth + # Detect by token prefix so existing app password users keep working. + auth_scheme = ( + "x-bitbucket-api-token-auth" + if access_token.startswith("ATCTT") + else "x-token-auth" ) - # Check if repository already exists - if os.path.exists(local_path) and os.listdir(local_path): - # Directory exists and is not empty - logger.warning( - f"Repository already exists at {local_path}. Using existing repository." + remote_url = urlunparse( + ( + parsed.scheme, + f"{auth_scheme}:{quote(access_token, safe="")}@{parsed.netloc}", + parsed.path, + "", + "", + "", ) - return f"Using existing repository at {local_path}" - - # Ensure the local path exists - os.makedirs(local_path, exist_ok=True) - - # Prepare the clone URL with access token if provided - clone_url = repo_url - if access_token: - parsed = urlparse(repo_url) - # URL-encode the token to handle special characters - encoded_token = quote(access_token, safe="") - # Determine the repository type and format the URL accordingly - if repo_type == "github": - # Format: https://{token}@{domain}/owner/repo.git - # Works for both github.com and enterprise GitHub domains - clone_url = urlunparse( - ( - parsed.scheme, - f"{encoded_token}@{parsed.netloc}", - parsed.path, - "", - "", - "", - ) - ) - elif repo_type == "gitlab": - # Format: https://oauth2:{token}@gitlab.com/owner/repo.git - clone_url = urlunparse( - ( - parsed.scheme, - f"oauth2:{encoded_token}@{parsed.netloc}", - parsed.path, - "", - "", - "", - ) - ) - elif repo_type == "bitbucket": - # Bitbucket has two token formats with different auth schemes: - # - HTTP access tokens (prefix "ATCTT") use x-bitbucket-api-token-auth - # - App passwords (deprecated, EOL June 2026) use x-token-auth - # Detect by token prefix so existing app password users keep working. - if access_token.startswith("ATCTT"): - auth_scheme = "x-bitbucket-api-token-auth" - else: - auth_scheme = "x-token-auth" - # Format: https://{auth_scheme}:{token}@bitbucket.org/owner/repo.git - clone_url = urlunparse( - ( - parsed.scheme, - f"{auth_scheme}:{encoded_token}@{parsed.netloc}", - parsed.path, - "", - "", - "", - ) - ) - - logger.info("Using access token for authentication") - - # Clone the repository - logger.info(f"Cloning repository from {repo_url} to {local_path}") - # We use repo_url in the log to avoid exposing the token in logs - result = subprocess.run( - ["git", "clone", "--depth=1", "--single-branch", clone_url, local_path], - check=True, - capture_output=True, ) - - logger.info("Repository cloned successfully") - return result.stdout.decode("utf-8") - - except subprocess.CalledProcessError as e: - error_msg = e.stderr.decode("utf-8") - # Sanitize error message to remove any tokens (both raw and URL-encoded) - if access_token: - # Remove raw token - error_msg = error_msg.replace(access_token, "***TOKEN***") - # Also remove URL-encoded token to prevent leaking encoded version - encoded_token = quote(access_token, safe="") - error_msg = error_msg.replace(encoded_token, "***TOKEN***") - raise ValueError(f"Error during cloning: {error_msg}") - except Exception as e: - raise ValueError(f"An unexpected error occurred: {str(e)}") + return GitRepo.clone_from(url=remote_url, to_path=local_path) def _path_is_url(path: str) -> bool: @@ -501,9 +186,34 @@ def _extract_repo_name(repo_url: str, repo_type: str | None) -> str: def download(self, force: bool = False) -> None: if force or (not self.downloaded and not self.is_local): os.makedirs(self.save_path, exist_ok=True) - download_repo( - self.repo_url, self.save_path, self.repo_type, self.access_token - ) + + if not GIT_OK: + raise RuntimeError("Missing `git` in current environment") + + if self.repo_type == "github": + _clone_from_github( + remote_url=self.repo_url, + local_path=self.save_path, + access_token=self.access_token, + ) + + elif self.repo_type == "gitlab": + _clone_from_gitlab( + remote_url=self.repo_url, + local_path=self.save_path, + access_token=self.access_token, + ) + + elif self.repo_type == "bitbucket": + _clone_from_bitbucket( + remote_url=self.repo_url, + local_path=self.save_path, + access_token=self.access_token, + ) + else: + raise NotImplementedError(f"Unknown repo type: {self.repo_type}") + + logger.info("Repository %s cloned successfully", self.name) @property def save_path(self) -> str: From 190957d8a5e2ea96d077d9d5e3ebae0ca778bd93 Mon Sep 17 00:00:00 2001 From: GdoongMathew Date: Wed, 5 Aug 2026 17:31:55 +0800 Subject: [PATCH 2/6] fix: restore original git shallow clone option --- api/repository.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/api/repository.py b/api/repository.py index 9f7c4738d..2cce509ad 100644 --- a/api/repository.py +++ b/api/repository.py @@ -39,6 +39,7 @@ def _clone_from_gitlab( local_path: str, *, access_token: str | None = None, + **kwargs, ) -> GitRepo: if access_token: parsed = urlparse(remote_url) @@ -53,8 +54,7 @@ def _clone_from_gitlab( "", ) ) - - return GitRepo.clone_from(url=remote_url, to_path=local_path) + return GitRepo.clone_from(url=remote_url, to_path=local_path, **kwargs) @_exception_cleanup @@ -63,6 +63,7 @@ def _clone_from_github( local_path: str, *, access_token: str | None = None, + **kwargs, ) -> GitRepo: if access_token: parsed = urlparse(remote_url) @@ -77,8 +78,7 @@ def _clone_from_github( "", ) ) - - return GitRepo.clone_from(url=remote_url, to_path=local_path) + return GitRepo.clone_from(url=remote_url, to_path=local_path, **kwargs) @_exception_cleanup @@ -87,6 +87,7 @@ def _clone_from_bitbucket( local_path: str, *, access_token: str | None = None, + **kwargs, ) -> GitRepo: if access_token: parsed = urlparse(remote_url) @@ -110,7 +111,7 @@ def _clone_from_bitbucket( "", ) ) - return GitRepo.clone_from(url=remote_url, to_path=local_path) + return GitRepo.clone_from(url=remote_url, to_path=local_path, **kwargs) def _path_is_url(path: str) -> bool: @@ -190,26 +191,21 @@ def download(self, force: bool = False) -> None: if not GIT_OK: raise RuntimeError("Missing `git` in current environment") + kwargs = { + "remote_url": self.repo_url, + "local_path": self.save_path, + "access_token": self.access_token, + "multi_options": ["--depth=1", "--single-branch"], + } + if self.repo_type == "github": - _clone_from_github( - remote_url=self.repo_url, - local_path=self.save_path, - access_token=self.access_token, - ) + _clone_from_github(**kwargs) elif self.repo_type == "gitlab": - _clone_from_gitlab( - remote_url=self.repo_url, - local_path=self.save_path, - access_token=self.access_token, - ) + _clone_from_gitlab(**kwargs) elif self.repo_type == "bitbucket": - _clone_from_bitbucket( - remote_url=self.repo_url, - local_path=self.save_path, - access_token=self.access_token, - ) + _clone_from_bitbucket(**kwargs) else: raise NotImplementedError(f"Unknown repo type: {self.repo_type}") From a18b40aa4bf05b4b2bb2ab5d129d4a3958817394 Mon Sep 17 00:00:00 2001 From: GdoongMathew Date: Wed, 5 Aug 2026 17:39:50 +0800 Subject: [PATCH 3/6] apply suggestions --- api/repository.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/repository.py b/api/repository.py index 2cce509ad..c70ee16cb 100644 --- a/api/repository.py +++ b/api/repository.py @@ -16,7 +16,7 @@ def _exception_cleanup(func: Callable) -> Callable: - @wraps + @wraps(func) def wrapper(*args, **kwargs): try: return func(*args, **kwargs) @@ -67,11 +67,12 @@ def _clone_from_github( ) -> GitRepo: if access_token: parsed = urlparse(remote_url) + access_token = quote(access_token, safe="") remote_url = urlunparse( ( parsed.scheme, - f"{quote(access_token, safe="")}@{parsed.netloc}", + f"{access_token}@{parsed.netloc}", parsed.path, "", "", @@ -100,11 +101,12 @@ def _clone_from_bitbucket( if access_token.startswith("ATCTT") else "x-token-auth" ) + access_token = quote(access_token, safe="") remote_url = urlunparse( ( parsed.scheme, - f"{auth_scheme}:{quote(access_token, safe="")}@{parsed.netloc}", + f"{auth_scheme}:{access_token}@{parsed.netloc}", parsed.path, "", "", From 362b23066691e7f4e728692ab566fbb300f7bb02 Mon Sep 17 00:00:00 2001 From: GdoongMathew Date: Wed, 5 Aug 2026 18:25:33 +0800 Subject: [PATCH 4/6] fix exception cleanup, and add unittests --- api/repository.py | 8 ++-- tests/test_repository.py | 85 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 tests/test_repository.py diff --git a/api/repository.py b/api/repository.py index c70ee16cb..dc8f58787 100644 --- a/api/repository.py +++ b/api/repository.py @@ -4,7 +4,7 @@ from collections.abc import Callable from urllib.parse import quote, urlparse, urlunparse -from git import Repo as GitRepo, GIT_OK +from git import Repo as GitRepo, GIT_OK, GitCommandError from api.logger import get_logger from api.utils import deepwiki_root @@ -20,8 +20,10 @@ def _exception_cleanup(func: Callable) -> Callable: def wrapper(*args, **kwargs): try: return func(*args, **kwargs) - except subprocess.CalledProcessError as e: - err_msg = e.stderr.decode("utf-8") + except (subprocess.CalledProcessError, GitCommandError) as e: + err_msg: str | bytes = e.stderr + if isinstance(err_msg, bytes): + err_msg = err_msg.decode("utf-8") token = kwargs.get("access_token", None) if token: token_mask = "***TOKEN***" diff --git a/tests/test_repository.py b/tests/test_repository.py new file mode 100644 index 000000000..c36cdfa5d --- /dev/null +++ b/tests/test_repository.py @@ -0,0 +1,85 @@ +import pytest +import re +import os + +import git + +from api.repository import Repo + + +def test_repo_is_local(): + repo = Repo(repo_url="./", repo_type="local") + assert repo.is_local + + +def test_repo_is_remote(tmpdir): + repo = Repo( + repo_url="https://github.com/AsyncFuncAI/deepwiki-open", + repo_type="github", + root_path=tmpdir, + ) + assert not repo.is_local + assert not repo.downloaded + + +def test_repo_download_no_git(tmpdir, monkeypatch): + repo = Repo( + repo_url="https://github.com/AsyncFuncAI/deepwiki-open", + repo_type="github", + root_path=tmpdir, + ) + from api import repository + monkeypatch.setattr(repository, "GIT_OK", value=False) + + with pytest.raises(RuntimeError, match="Missing `git` in current environment"): + repo.download() + + +def test_repo_download_path_exists(tmpdir, mocker): + repo = Repo( + repo_url="https://github.com/AsyncFuncAI/deepwiki-open", + repo_type="github", + root_path=tmpdir, + ) + + def touch_file(*args, **kwargs): + tmp_file = os.path.join(repo.save_path, "touch") + with open(tmp_file, "w") as f: + f.write("") + mocker.patch.object(git.Repo, "clone_from", return_value=None, side_effect=touch_file) + + repo.download() + assert repo.downloaded + assert os.path.exists(repo.save_path) + + +def test_repo_git_clone_message_masking(tmpdir, mocker): + repo = Repo( + repo_url="https://github.com/AsyncFuncAI/deepwiki-open", + repo_type="github", + root_path=tmpdir, + access_token="123456789" + ) + + def raise_error(*args, **kwargs): + raise git.GitCommandError(command="git clone", stderr="123456789 is not a valid token") + + mocker.patch.object(git.Repo, "clone_from", return_value=None, side_effect=raise_error) + + with pytest.raises(ValueError, match=re.escape("***TOKEN*** is not a valid token")): + repo.download() + + +@pytest.mark.network +@pytest.mark.parametrize( + "repo_url, repo_type", + [ + ("https://github.com/AsyncFuncAI/deepwiki-open", "github"), + ("https://gitlab.com/gitlab-org/gitlab-pages", "gitlab"), + ] +) +def test_repo_download(repo_url, repo_type, tmpdir): + repo = Repo(repo_url, repo_type, root_path=tmpdir) + repo.download() + + assert repo.downloaded From 246ca00cc4ad624012ae07c02f79a7a3d5ebd8fa Mon Sep 17 00:00:00 2001 From: GdoongMathew Date: Wed, 5 Aug 2026 18:32:07 +0800 Subject: [PATCH 5/6] fix cleanup --- api/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/repository.py b/api/repository.py index dc8f58787..070efda18 100644 --- a/api/repository.py +++ b/api/repository.py @@ -27,7 +27,7 @@ def wrapper(*args, **kwargs): token = kwargs.get("access_token", None) if token: token_mask = "***TOKEN***" - err_msg.replace(token, token_mask) + err_msg = err_msg.replace(token, token_mask) encoded_token = quote(token, safe="") err_msg = err_msg.replace(encoded_token, token_mask) raise ValueError(err_msg) From c0aba9f32f673504ee8e3688ef44514480acbf26 Mon Sep 17 00:00:00 2001 From: GdoongMathew Date: Fri, 7 Aug 2026 08:41:46 +0800 Subject: [PATCH 6/6] fix access_token parsing --- api/repository.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/repository.py b/api/repository.py index 070efda18..b87b8ff5b 100644 --- a/api/repository.py +++ b/api/repository.py @@ -45,11 +45,12 @@ def _clone_from_gitlab( ) -> GitRepo: if access_token: parsed = urlparse(remote_url) + access_token = quote(access_token, safe="") remote_url = urlunparse( ( parsed.scheme, - f"oauth2:{quote(access_token, safe="")}@{parsed.netloc}", + f"oauth2:{access_token}@{parsed.netloc}", parsed.path, "", "", @@ -69,7 +70,6 @@ def _clone_from_github( ) -> GitRepo: if access_token: parsed = urlparse(remote_url) - access_token = quote(access_token, safe="") remote_url = urlunparse( ( @@ -224,3 +224,6 @@ def save_path(self) -> str: @property def downloaded(self) -> bool: return os.path.exists(self.save_path) and bool(os.listdir(self.save_path)) + + def __repr__(self) -> str: + return f"{self.repo_type}: {self.name}"