diff --git a/.github/assets/before-after.png b/.github/assets/before-after.png new file mode 100644 index 0000000..fe58509 Binary files /dev/null and b/.github/assets/before-after.png differ diff --git a/.github/assets/social-preview.png b/.github/assets/social-preview.png new file mode 100644 index 0000000..c68d79f Binary files /dev/null and b/.github/assets/social-preview.png differ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5b527d5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,82 @@ +# Smoke-tests the pipeline on the bundled sample scan and verifies the output is +# genuinely born-digital (extractable text, much smaller than the scan). +# +# The green check on this workflow is what the README's CI badge points at, so keep +# the filename stable once the badge references it. +# +# NOTE: every step below that touches the environment declares +# `shell: bash -el {0}`. GitHub's default Ubuntu shell is `bash --noprofile +# --norc`, which never sources the profile that `conda init` writes, so without +# it `tesseract` and `python` are "command not found" even though the env was +# created successfully. That is the action's documented requirement, not a +# quirk. + +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + smoke: + runs-on: ubuntu-latest + # See the note at the top: the login shell is what loads the conda env. + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + + # Miniforge, not Miniconda: it ships mamba and only ever points at + # conda-forge, so there is no `defaults` channel to ask about and no + # Anaconda ToS prompt to abort the run. That is the whole reason this step + # is not "setup-miniconda + a condarc". + - name: Setup Miniforge + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-variant: Miniforge3 + miniforge-version: latest + activate-environment: pdf-ocr + environment-file: environment.yml + use-mamba: true + auto-activate: false + + - name: Tesseract is on PATH + run: tesseract --version + + - name: CLI is importable and --help works + run: python -m pdfclean --help + + - name: Convert page 1 of the bundled sample + run: python -m pdfclean assets/ -o output/ --max-pages 1 --overwrite + + - name: Output exists + run: ls -la output/ + + - name: Output is born-digital (real extractable text, far smaller than the scan) + run: | + python - <<'PY' + import glob + import os + + import pymupdf + + scan_path = glob.glob("assets/*.pdf")[0] + out_paths = glob.glob("output/**/*.pdf", recursive=True) + assert out_paths, "no output PDF produced" + + out = pymupdf.open(out_paths[0]) + words = out[0].get_text().split() + print(f"words on page 1: {len(words)}") + print(f"sample: {' '.join(words[:25])}") + + assert len(words) > 200, f"only {len(words)} words extracted — OCR looks broken" + + # The point of the tool: the page photo is gone, so the output must be + # much smaller than the scan it came from. + ratio = os.path.getsize(out_paths[0]) / os.path.getsize(scan_path) + print(f"size ratio vs input: {ratio:.3f}") + assert ratio < 0.6, f"output is {ratio:.0%} of the scan — page image was probably kept" + PY \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d58c819 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,59 @@ +# Changelog + +All notable changes to this project are documented here. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/). + +## [Unreleased] + +### Fixed + +- `conda env create` no longer stalls for tens of minutes on macOS 13 Intel. + Recent `opencv-python-headless` releases publish no `macosx_13_0_x86_64` + wheel, so pip fell back to a source build that needs Xcode Command Line + Tools. The dependency is now bounded so every supported platform resolves + to a prebuilt wheel. +- The setup instructions for the Anaconda terms-of-service prompt now give a + command that actually works. The previously documented + `conda env create --override-channels` is not a valid flag for that + subcommand, and neither the `nodefaults` channel entry nor a `.condarc` in + the working directory prevents the prompt. + +## [0.1.0] - 2026-06-24 + +### Added + +- First release: `python -m pdfclean input/ -o output/` turns scanned, + image-only PDFs into born-digital ones — real selectable text, real figures, + no page photo. +- Per page: extract the embedded scan, clean it (deskew, background whitening, + denoise), OCR it with Tesseract, detect figures the OCR didn't claim as text, + then rebuild the page with reflowed text at a font size chosen to fill each + block's rectangle. +- Batch a folder of PDFs; the output tree mirrors the input tree. +- Optional `--engine vision` corrects OCR errors and detects italics using a + hosted multimodal model (Mistral by default, OpenRouter supported). Falls + back to the Tesseract text per page if a call fails, so a batch always + finishes. +- `--no-figures` for text-only output, `--max-pages` to test cheaply, + `--min-conf` and `--psm` to trade OCR quality against speed. +- `--api-key`, or the provider's env var, or a gitignored `.env.local`. + + \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d450333 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,104 @@ +# Contributing + +## Development environment + +### Requirements + +- Miniconda (or any conda/mamba distribution) — the project targets conda, not pip +- Tesseract comes from conda; no system-level install needed + +### Setup + +```bash +git clone https://github.com/killerwolf/pdfclean.git +cd pdfclean +conda env create -f environment.yml +conda activate pdf-ocr +``` + +Verify the toolchain end to end before changing anything: + +```bash +python -m pdfclean assets/ -o output/ --max-pages 1 --overwrite +``` + +> **The Anaconda ToS prompt.** `environment.yml` asks for `conda-forge`, but conda +> still merges the `defaults` channels (repo.anaconda.com) from its own config, and +> those need their Terms of Service accepted. In a non-interactive shell that aborts +> with `CondaToSNonInteractiveError`. You have two options: +> +> **Accept the terms** — one-off, keeps `defaults` available: +> +> ```bash +> conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main +> conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r +> ``` +> +> **Or skip those channels entirely**, which needs no ToS acceptance at all. Point +> conda at a config that replaces `defaults` rather than adding to it: +> +> ```bash +> printf 'channels:\n - conda-forge\ndefault_channels:\n - conda-forge\nchannel_priority: strict\n' > .condarc +> CONDARC=$PWD/.condarc conda env create -f environment.yml +> ``` +> +> Two things that do *not* work, despite looking like they should: the `nodefaults` +> channel entry inside `environment.yml`, and a bare `.condarc` in the working +> directory. The ToS check runs before either is applied — only the `CONDARC` +> environment variable is honoured early enough. Note that `conda env create` does +> not accept `--override-channels` at all (that's `conda create`). + +## Project structure + +``` +pdfclean/ +├── pdfclean/ +│ ├── cli.py # argument parsing, .env.local loading, batch loop +│ ├── pipeline.py # per-document orchestration +│ ├── clean.py # deskew, background whitening, denoise +│ ├── ocr.py # Tesseract + the Block → Paragraph → Line → Word tree +│ ├── figures.py # figure detection, block splitting around figures +│ ├── style.py # bold detection from stroke width +│ ├── vision.py # optional hosted vision OCR +│ └── reconstruct.py # rebuilds the page with real text + figures +├── assets/ # sample scanned PDF +└── environment.yml +``` + +## Making a change + +1. Branch from `main`. +2. Make the change. The pipeline order in `pipeline.py` is the contract everything + else depends on — if you add a step, keep it there and update the README's + flow diagram to match. +3. Run a one-page conversion locally (command above); CI runs a real conversion + on the bundled sample, so a broken pipeline fails the build. +4. Add a `CHANGELOG.md` entry under `## [Unreleased]` if the change is user-visible. +5. Open a pull request describing what changed and why. + +### Things that are easy to get wrong + +- **Don't invent numbers.** The README quotes file sizes, word counts and OCR + confidence. If you change the pipeline, re-measure them on the bundled sample + and update the README rather than leaving stale figures. +- **Privacy.** `--engine vision` uploads page images to a third party. Keep it + opt-in, and never make it the default. +- **Base-14 fonts.** Output uses Times/Helvetica/Courier so the files stay tiny. + Text containing characters those fonts can't map renders as `?` — normalise + punctuation (see `_normalize_punct` in `reconstruct.py`) rather than embedding + a font. +- **Base64 blobs don't belong in the README.** Embedded base64 images don't render + on GitHub and bloat the file. Commit PNGs under `.github/assets/`. + +## Releasing + +1. Move the `## [Unreleased]` entries in `CHANGELOG.md` into a new version section + with today's date. +2. Bump `__version__` in `pdfclean/__init__.py`. +3. Commit, merge to `main`. +4. Tag and push: `git tag v && git push origin v`. +5. Create the GitHub Release from the tag. + +## Code of conduct + +Be decent to people. Report problems as a GitHub issue on the repo. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..bfc4453 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Hamdi LAADHARI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index e2d6e3b..0fa8eb8 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,36 @@ # pdfclean +

+ CI + MIT + Python 3.11 + Changelog +

+ Turn **scanned, image-only PDFs** into **clean, born-digital PDFs** — real embedded text you can select, search and copy, with the full-page scan image thrown away and only true figures re-embedded. +[![in / out](https://raw.githubusercontent.com/killerwolf/pdfclean/main/.github/assets/before-after.png)](https://raw.githubusercontent.com/killerwolf/pdfclean/main/.github/assets/before-after.png) + +*Page 1 of the bundled sample, before and after. Same 593×841 pt page: the +photo of the scan is gone, replaced by selectable text and the real figure.* + This is the aggressive "reconstruct from scratch" path: the output contains no page photo, just positioned text + cropped illustrations. Files come out small and editable. The trade-off is honesty about OCR — recognition mistakes appear as real text, and layout is approximated rather than pixel-perfect. +**Yes, if** you have a pile of image-only scans — a book chapter, a paper, a +French admin form — and want small, searchable, editable PDFs without paying +per page to a hosted service. The default engine is local Tesseract: nothing +leaves your machine. + +**No, if** you need pixel-perfect layout, or the original text layer exactly as +it was, or colour figures. For layout-preserving OCR with invisible text over the +image, `ocrmypdf` is the better tool. For exact extraction of an existing text +layer, you don't need OCR at all — `pdfplumber` or `pypdf` will do. + ## What it does, per page 1. **Extract** the embedded full-page scan at native resolution. @@ -30,8 +52,15 @@ as real text, and layout is approximated rather than pixel-perfect. white-point) and re-embedded as small JPEGs (`pdfclean/reconstruct.py`). On the bundled sample (`assets/…silent-language….pdf`): 10 pages, -2.1 MB image-only scan → **0.31 MB** fully selectable PDF, ~6,900 words, -mean OCR confidence 95. +2,147 kB image-only scan → **310 kB** fully selectable PDF (a 7× reduction), 6 761 +words, mean OCR confidence 95. Measured with: + +```bash +python -m pdfclean assets/ -o output/ --overwrite +``` + +The numbers above are what that run prints — re-measure rather than trusting +them if you change the pipeline. ## How it works @@ -115,11 +144,38 @@ font text. ## Setup +Requires Python 3.11 and Tesseract. The conda route installs both: + ```bash conda env create -f environment.yml conda activate pdf-ocr ``` +
+Conda asks you to accept Anaconda's terms of service + +`environment.yml` only lists `conda-forge`, but conda still consults its +`defaults` channels (repo.anaconda.com), which now require ToS acceptance. Point +conda at a config that replaces those defaults — nothing else is consulted +early enough to prevent the prompt: + +```bash +printf 'channels:\n - conda-forge\ndefault_channels:\n - conda-forge\nchannel_priority: strict\n' > .condarc +CONDARC=$PWD/.condarc conda env create -f environment.yml +``` +
+ +Prefer pip? The Python packages are ordinary wheels; only Tesseract itself is a +system binary (`brew install tesseract`, `apt install tesseract-ocr`): + +```bash +python -m venv .venv && source .venv/bin/activate +pip install pymupdf "opencv-python-headless<4.13.0" pytesseract requests pillow numpy +``` + +> The `opencv-python-headless` bound matters on macOS 13 Intel, where the newer +> releases ship no wheel and pip silently falls back to compiling from source. + ## Usage ```bash @@ -128,8 +184,16 @@ python -m pdfclean input/ -o output/ # a single file python -m pdfclean scan.pdf -o output/ # -> output/scan.clean.pdf + +# every flag, in your terminal +python -m pdfclean --help ``` +Nothing is overwritten unless you pass `--overwrite`, so re-running a batch is +safe. If no PDF is found at the path you gave, it exits `2` with the reason on +stderr; if some files in a batch fail, the rest still run and the exit code is +`1`. + ### Options | flag | default | meaning | @@ -207,3 +271,16 @@ to the Tesseract text so the batch still completes. weight). Local slant detection was too noisy to ship safely. - **Figures are grayscale.** Fine for line art; a `--color-figures` flag could preserve colour photos. +- **Non-Latin scripts.** Tesseract needs language packs installed (`-l fra`, and + so on). Output is reflowed with Times/Helvetica/Courier so the files stay tiny, + which means glyphs those base-14 fonts lack render as `?` — CJK output needs an + embedded font, not just a `--lang` flag. + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for the dev environment and the checks CI +runs. [CHANGELOG.md](CHANGELOG.md) tracks what changed for users. + +## Licence + +MIT — see [LICENSE](LICENSE). diff --git a/environment.yml b/environment.yml index 8494885..dce7b0f 100644 --- a/environment.yml +++ b/environment.yml @@ -8,9 +8,14 @@ dependencies: - tesseract # OCR engine (system binary + language data) - pip - pip: - - pymupdf # read scans + write born-digital PDFs - - opencv-python-headless # image cleaning (deskew/whiten/denoise) - - pytesseract # Tesseract Python bindings - - requests # hosted vision-OCR API calls (--engine vision) + - pymupdf # read scans + write born-digital PDFs + # opencv-python-headless dropped macosx_13_0_x86_64 wheels after 4.12 + # (5.x is macosx_14_0+ only, and 4.13/4.14 publish only arm64 for 13). + # On macOS 13 Intel pip finds no wheel and silently falls back to a very + # slow source build via scikit-build that needs Xcode CLT. Bound the + # version so every platform resolves to a prebuilt wheel instead. + - opencv-python-headless<4.13.0 # image cleaning (deskew/whiten/denoise) + - pytesseract # Tesseract Python bindings + - requests # hosted vision-OCR API calls (--engine vision) - pillow - numpy