github_api.py

"""GitHub API module - fetches trending repos across all languages."""
import requests
import re
import time
import threading
from datetime import datetime, timedelta
from typing import List, Dict


def _fetch_page(session, query, page, per_page):
    """Fetch single page (no loops) - fast timeout."""
    try:
        r = session.get('https://api.github.com/search/repositories',
                       params={'q': query, 'sort': 'stars', 'order': 'desc', 'per_page': per_page, 'page': page}, timeout=3)
        r.raise_for_status()
        return r.json().get('items', [])
    except:
        return []

def _get_page_count(link_header):
    """Extract page count from Link header."""
    if not link_header or 'rel="last"' not in link_header:
        return None
    m = re.search(r'[?&]page=(\d+)>; rel="last"', link_header) or re.search(r'page=(\d+)>; rel="last"', link_header)
    return int(m.group(1)) if m else None

def _fetch_with_retry(session, url, params, max_retries=3):
    """Fetch with exponential backoff retry - only used when rate limited (no loops, recursive)."""
    def _retry(retries):
        if retries <= 0:
            return None
        try:
            r = session.get(url, params=params, timeout=3)
            if r.status_code == 200:
                return r
            elif r.status_code in (403, 429):
                time.sleep(2 ** (max_retries - retries))
                return _retry(retries - 1)
        except:
            if retries > 1:
                time.sleep(0.3)
                return _retry(retries - 1)
        return None
    return _retry(max_retries)


def fetch_trending_repos(limit: int = 50) -> List[Dict]:
    """Fetch trending repos - stars + recent activity (30 days, recursion no loops)."""
    s = requests.Session()
    s.headers.update({'Accept': 'application/vnd.github.v3+json', 'User-Agent': 'CodeSentinel-Risk-Scraper'})
    q = f'pushed:>={(datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")} stars:>100'
    p = min(100, limit)
    def _r(page, repos):
        if len(repos) >= limit:
            return repos[:limit]
        items = _fetch_page(s, q, page, p)
        if not items or len(items) < p:
            return (repos + items[:limit - len(repos)])[:limit]
        repos.extend(items[:limit - len(repos)])
        return _r(page + 1, repos)
    return _r(1, [])


def fetch_user_repos(username: str) -> List[Dict]:
    """Fetch all public repos for a user (recursive pagination, no loops)."""
    s = requests.Session()
    s.headers.update({'Accept': 'application/vnd.github.v3+json', 'User-Agent': 'CodeSentinel-Risk-Scraper'})
    def _r(page, repos):
        try:
            r = s.get(f"https://api.github.com/users/{username}/repos", params={'per_page': 100, 'page': page, 'sort': 'updated'}, timeout=3)
            if r.status_code == 200:
                items = r.json()
                if not items or len(items) == 0:
                    return repos
                repos.extend(items)
                if len(items) == 100:
                    return _r(page + 1, repos)
            elif r.status_code in (403, 429):
                time.sleep(1)
                return _r(page, repos)
        except:
            pass
        return repos
    return _r(1, [])

def fetch_single_repo_data(repo: Dict) -> Dict:
    """Fetch detailed metadata for single repo (no loops) - thread-safe parallel requests."""
    fn = repo['full_name']
    d = {
        'name': repo['name'], 'full_name': fn, 'language': repo.get('language', 'Unknown'),
        'stars': repo.get('stargazers_count', 0), 'forks': repo.get('forks_count', 0),
        'created_at': repo.get('created_at', ''), 'updated_at': repo.get('updated_at', ''),
        'pushed_at': repo.get('pushed_at', ''), 'contributors': [], 'contributor_count': 0, 'commits_count': 0,
        'open_issues_count': repo.get('open_issues_count', 0),
        'license': repo.get('license', {}).get('spdx_id', 'None') if repo.get('license') else 'None'
    }
    # Parallel requests: contributors and commits in parallel (thread-safe: each thread gets own session)
    def _fetch_contributors():
        try:
            s = requests.Session()
            s.headers.update({'Accept': 'application/vnd.github.v3+json', 'User-Agent': 'CodeSentinel-Risk-Scraper'})
            r = s.get(f"https://api.github.com/repos/{fn}/contributors", params={'per_page': 100}, timeout=3)
            if r.status_code == 200:
                c = r.json()
                cl = c[:10] if isinstance(c, list) else []
                d['contributors'] = cl
                lp = _get_page_count(r.headers.get('Link', ''))
                d['contributor_count'] = (lp - 1) * 100 + 75 if lp else len(c) if isinstance(c, list) else 0
            elif r.status_code in (403, 429):
                r = _fetch_with_retry(s, f"https://api.github.com/repos/{fn}/contributors", {'per_page': 100})
                if r:
                    c = r.json()
                    cl = c[:10] if isinstance(c, list) else []
                    d['contributors'] = cl
                    lp = _get_page_count(r.headers.get('Link', ''))
                    d['contributor_count'] = (lp - 1) * 100 + 75 if lp else len(c) if isinstance(c, list) else 0
        except:
            pass
    def _fetch_commits():
        try:
            s = requests.Session()
            s.headers.update({'Accept': 'application/vnd.github.v3+json', 'User-Agent': 'CodeSentinel-Risk-Scraper'})
            r = s.get(f"https://api.github.com/repos/{fn}/commits", params={'per_page': 1}, timeout=3)
            if r.status_code == 200:
                lp = _get_page_count(r.headers.get('Link', ''))
                d['commits_count'] = lp if lp else len(r.json()) if r.json() else 0
            elif r.status_code in (403, 429):
                r = _fetch_with_retry(s, f"https://api.github.com/repos/{fn}/commits", {'per_page': 1})
                if r:
                    lp = _get_page_count(r.headers.get('Link', ''))
                    d['commits_count'] = lp if lp else len(r.json()) if r.json() else 0
        except:
            pass
    # Run both requests in parallel (no loop, just threading) - each thread has own session
    t1, t2 = threading.Thread(target=_fetch_contributors), threading.Thread(target=_fetch_commits)
    t1.start()
    t2.start()
    t1.join(timeout=3)
    t2.join(timeout=3)
    return d