|
| 1 | +"""Utility to create a trimmed ZIP archive of the repository for LLM analysis. |
| 2 | +
|
| 3 | +Features: |
| 4 | + - Excludes virtual environments, caches, build artifacts, logs, large/local DB files (.accdb, .mdb, .laccdb), coverage outputs, and removed legacy assets. |
| 5 | + - Reads .gitignore (basic parsing) to extend exclusions (optional flag). |
| 6 | + - Provides --dry-run mode to list counts without writing the ZIP. |
| 7 | + - Allows overriding output path and adding extra exclude globs. |
| 8 | + - Keeps documentation, source code (src/), tests/, configuration, and examples. |
| 9 | +
|
| 10 | +Usage examples: |
| 11 | + python tools/create_repo_export_zip.py # creates repo_export_YYYYMMDD_HHMM.zip in repo root |
| 12 | + python tools/create_repo_export_zip.py --output export.zip |
| 13 | + python tools/create_repo_export_zip.py --dry-run |
| 14 | + python tools/create_repo_export_zip.py --extra-exclude "examples/**" --dry-run |
| 15 | +
|
| 16 | +Notes: |
| 17 | + - Exclusion globs are matched against POSIX-style relative paths (e.g. "src/module/file.py"). |
| 18 | + - Directory exclusions prevent descent (efficient skipping). |
| 19 | + - If you need to include Access DB schema docs (.md) they will still be included; only the binary DB files are excluded. |
| 20 | +""" |
| 21 | +from __future__ import annotations |
| 22 | + |
| 23 | +import argparse |
| 24 | +import fnmatch |
| 25 | +import os |
| 26 | +from pathlib import Path |
| 27 | +import sys |
| 28 | +import time |
| 29 | +import zipfile |
| 30 | +from typing import Iterable, List |
| 31 | + |
| 32 | +REPO_ROOT = Path(__file__).resolve().parent.parent |
| 33 | + |
| 34 | +DEFAULT_EXCLUDES: List[str] = [ |
| 35 | + # VCS / tooling |
| 36 | + '.git', '.git/**', '.idea', '.vscode', |
| 37 | + # Virtual envs |
| 38 | + 'venv', 'venv/**', '.venv', '.venv/**', 'env', 'env/**', 'ENV', 'ENV/**', |
| 39 | + # Python caches / compiled |
| 40 | + '__pycache__', '**/__pycache__', '*.py[cod]', '*.pyo', '*.pyd', '*.so', |
| 41 | + # Build / packaging artifacts |
| 42 | + 'build', 'dist', 'downloads', 'eggs', '.eggs', '*.egg-info', '*.egg', |
| 43 | + # Test / coverage artifacts |
| 44 | + '.tox', '.pytest_cache', '.pytest_cache/**', 'htmlcov', 'htmlcov/**', '.coverage*', 'coverage.xml', '*.cover', |
| 45 | + # Logs |
| 46 | + 'logs', 'logs/**', '*.log', |
| 47 | + # Legacy removed quarantine |
| 48 | + 'legacy/.removed', 'legacy/.removed/**', |
| 49 | + # Local database / binary data files |
| 50 | + '*.accdb', '*.mdb', '*.laccdb', 'dbs-locales/*.accdb', 'dbs-locales/*.laccdb', |
| 51 | + # Misc OS / editor |
| 52 | + '.DS_Store', 'Thumbs.db', 'ehthumbs.db', |
| 53 | + # Debug / scratch artifacts referenced previously |
| 54 | + 'debug_*.py', 'verify_*.py', 'debug_html', 'debug_html/**', 'quality_output.html', 'setup_local_environment_root.log', |
| 55 | + # Generated exports (avoid nesting self) |
| 56 | + 'repo_export_*.zip', 'repo_export_*.tar.gz', |
| 57 | +] |
| 58 | + |
| 59 | +def load_gitignore_patterns(root: Path) -> List[str]: |
| 60 | + gitignore = root / '.gitignore' |
| 61 | + if not gitignore.exists(): |
| 62 | + return [] |
| 63 | + patterns: List[str] = [] |
| 64 | + for line in gitignore.read_text(encoding='utf-8').splitlines(): |
| 65 | + line = line.strip() |
| 66 | + if not line or line.startswith('#'): |
| 67 | + continue |
| 68 | + if line.startswith('/'): |
| 69 | + line = line[1:] |
| 70 | + patterns.append(line) |
| 71 | + return patterns |
| 72 | + |
| 73 | +def matches_any(path_rel: str, patterns: Iterable[str]) -> bool: |
| 74 | + return any(fnmatch.fnmatch(path_rel, pat) for pat in patterns) |
| 75 | + |
| 76 | +def should_exclude(path: Path, rel_posix: str, patterns: Iterable[str]) -> bool: |
| 77 | + if matches_any(rel_posix, patterns): |
| 78 | + return True |
| 79 | + if path.is_dir() and matches_any(rel_posix.rstrip('/') + '/**', patterns): |
| 80 | + return True |
| 81 | + return False |
| 82 | + |
| 83 | +def build_file_list(exclude_patterns: List[str]): |
| 84 | + included = [] |
| 85 | + excluded = [] |
| 86 | + for root, dirs, files in os.walk(REPO_ROOT, topdown=True): |
| 87 | + root_path = Path(root) |
| 88 | + # Prune dirs |
| 89 | + pruned_dirs = [] |
| 90 | + for d in list(dirs): |
| 91 | + dir_path = root_path / d |
| 92 | + rel = dir_path.relative_to(REPO_ROOT).as_posix() |
| 93 | + if should_exclude(dir_path, rel, exclude_patterns): |
| 94 | + excluded.append(dir_path) |
| 95 | + continue |
| 96 | + pruned_dirs.append(d) |
| 97 | + dirs[:] = pruned_dirs |
| 98 | + for f in files: |
| 99 | + file_path = root_path / f |
| 100 | + rel = file_path.relative_to(REPO_ROOT).as_posix() |
| 101 | + if should_exclude(file_path, rel, exclude_patterns): |
| 102 | + excluded.append(file_path) |
| 103 | + else: |
| 104 | + included.append(file_path) |
| 105 | + return included, excluded |
| 106 | + |
| 107 | +def create_zip(files: List[Path], output: Path) -> None: |
| 108 | + with zipfile.ZipFile(output, 'w', compression=zipfile.ZIP_DEFLATED) as zf: |
| 109 | + for p in files: |
| 110 | + arcname = p.relative_to(REPO_ROOT).as_posix() |
| 111 | + zf.write(p, arcname=arcname) |
| 112 | + |
| 113 | +def parse_args(argv: List[str]) -> argparse.Namespace: |
| 114 | + parser = argparse.ArgumentParser(description="Create a curated ZIP of the repository excluding non-relevant artifacts.") |
| 115 | + parser.add_argument('--output', '-o', type=Path, help='Output zip file path (default: repo_export_YYYYMMDD_HHMM.zip in repo root)') |
| 116 | + parser.add_argument('--include-gitignore', action='store_true', help='Also parse .gitignore to extend exclusion patterns') |
| 117 | + parser.add_argument('--extra-exclude', action='append', default=[], help='Additional exclusion glob(s). Can be provided multiple times.') |
| 118 | + parser.add_argument('--force-include', action='append', default=[], help='Glob pattern(s) to force-include even if excluded. Can be repeated.') |
| 119 | + parser.add_argument('--list', action='store_true', help='List included file paths (verbose).') |
| 120 | + parser.add_argument('--dry-run', action='store_true', help='Do not write the zip, only report statistics.') |
| 121 | + parser.add_argument('--show-excludes', action='store_true', help='Print the final exclusion pattern list.') |
| 122 | + return parser.parse_args(argv) |
| 123 | + |
| 124 | +def human_readable_size(num_bytes: int) -> str: |
| 125 | + units = ['B', 'KB', 'MB', 'GB', 'TB'] |
| 126 | + size = float(num_bytes) |
| 127 | + for unit in units: |
| 128 | + if size < 1024.0 or unit == units[-1]: |
| 129 | + return f'{size:.1f} {unit}' |
| 130 | + size /= 1024.0 |
| 131 | + return f'{size:.1f} PB' |
| 132 | + |
| 133 | +def main(argv: List[str]) -> int: |
| 134 | + args = parse_args(argv) |
| 135 | + patterns: List[str] = list(dict.fromkeys(DEFAULT_EXCLUDES)) |
| 136 | + if args.include_gitignore: |
| 137 | + patterns.extend(load_gitignore_patterns(REPO_ROOT)) |
| 138 | + if args.extra_exclude: |
| 139 | + patterns.extend(args.extra_exclude) |
| 140 | + patterns = [p[2:] if p.startswith('./') else p for p in patterns] |
| 141 | + if args.show_excludes: |
| 142 | + print(f'Exclusion patterns ({len(patterns)}):') |
| 143 | + for p in patterns: |
| 144 | + print(' -', p) |
| 145 | + included, excluded = build_file_list(patterns) |
| 146 | + # Force-include pass |
| 147 | + if args.force_include: |
| 148 | + force = args.force_include |
| 149 | + # Identify candidates among excluded paths (files only) |
| 150 | + readd = [] |
| 151 | + for path in list(excluded): |
| 152 | + if path.is_dir(): |
| 153 | + continue |
| 154 | + rel = path.relative_to(REPO_ROOT).as_posix() |
| 155 | + if matches_any(rel, force): |
| 156 | + readd.append(path) |
| 157 | + if readd: |
| 158 | + for p in readd: |
| 159 | + excluded.remove(p) |
| 160 | + included.append(p) |
| 161 | + print(f'Force-included {len(readd)} file(s) overriding exclusion patterns.') |
| 162 | + timestamp = time.strftime('%Y%m%d_%H%M') |
| 163 | + default_name = f'repo_export_{timestamp}.zip' |
| 164 | + output_path = (args.output if args.output else (REPO_ROOT / default_name)).resolve() |
| 165 | + try: |
| 166 | + if output_path.is_relative_to(REPO_ROOT): # type: ignore[attr-defined] |
| 167 | + included = [f for f in included if f.resolve() != output_path] |
| 168 | + except AttributeError: |
| 169 | + # Python <3.9 fallback not needed here but ensures compatibility |
| 170 | + if str(output_path).startswith(str(REPO_ROOT)): |
| 171 | + included = [f for f in included if f.resolve() != output_path] |
| 172 | + total_size = sum(f.stat().st_size for f in included) |
| 173 | + print(f'Included files: {len(included)} | Excluded paths: {len(excluded)} | Total size: {human_readable_size(total_size)}') |
| 174 | + if args.list: |
| 175 | + for f in included: |
| 176 | + print(' +', f.relative_to(REPO_ROOT).as_posix()) |
| 177 | + if args.dry_run: |
| 178 | + print('Dry-run complete. No ZIP written.') |
| 179 | + return 0 |
| 180 | + output_path.parent.mkdir(parents=True, exist_ok=True) |
| 181 | + create_zip(included, output_path) |
| 182 | + print(f'ZIP written: {output_path} (size: {human_readable_size(output_path.stat().st_size)})') |
| 183 | + return 0 |
| 184 | + |
| 185 | +if __name__ == '__main__': # pragma: no cover |
| 186 | + sys.exit(main(sys.argv[1:])) |
0 commit comments