diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..32556bf --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,65 @@ +name: tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + core: + name: core (${{ matrix.os }}, py${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + python-version: ["3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Set up Python + run: uv python install ${{ matrix.python-version }} + + - name: Sync core dependencies + run: uv sync --frozen --all-extras + + - name: Install Tesseract (Ubuntu) + if: matrix.os == 'ubuntu-latest' + run: sudo apt-get update && sudo apt-get install -y tesseract-ocr + + - name: Install Tesseract (macOS) + if: matrix.os == 'macos-latest' + run: brew install tesseract + + - name: Run ruff + run: uv run ruff check src/ tests/ + + - name: Check formatting + run: uv run ruff format --check src/ tests/ + + - name: Run pytest + run: uv run pytest tests/ -q --maxfail=1 + + doctor-smoke: + name: doctor CLI smoke test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + - run: uv sync --frozen + - name: Install Tesseract + run: sudo apt-get update && sudo apt-get install -y tesseract-ocr + - name: Run doctor + run: uv run ocr-pipeline doctor diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d6686a0 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,44 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- `init` command: writes a minimal `config.yaml` with platform-correct defaults and checks for Tesseract. +- `demo` command: runs the pipeline on a bundled sample screenshot, prints the output Markdown, and verifies the install in <30s. +- `setup [basic|full]` command: executes install steps with confirmation (`--yes`/`--dry-run`). +- `doctor` command: health check for Python, Tesseract, PaddleOCR, PyTorch backend, table/figure detectors, scispaCy model, config, database, and output directory. +- Lazy settings: `settings` is a proxy; importing the package no longer reads `config.yaml` from cwd. +- Config search paths: `$OCR_PIPELINE_CONFIG` → `./config.yaml` → per-user config dir. +- Optional dependency extras: `paddle`, `tables`, `figures`, `scientific`, `kg`, `full`. +- Engine-aware preprocessing: PaddleOCR reads the (size-capped) original image; Tesseract reads the binarized derivative. Fixes dark-mode OCR quality (confidence 0.83 → 0.95 on test screenshots). +- `data/test_run_full` sample run on 7 screenshots demonstrating full-stack output. +- Tests for citations (19), doctor (15), and startup (12). + +### Changed +- Code defaults are now safe: Tesseract OCR, detectors/entities off. The repo's `config.yaml` keeps the full stack for development. +- Detectors default to `enabled: false`; enable per-detector in your config. +- OCR engine default is now `tesseract` (was `paddleocr`). +- `run` results table now shows Skipped count. +- Run directories are created lazily on first write (no empty `run_NNN`). +- `kg_ocr` package marked experimental; deps under `--extra kg`, helpful ImportError on bare import. + +### Fixed +- ISBN regex no longer false-positives on years, URLs, job IDs, or dilutions. Now requires an explicit `ISBN` prefix and matches strict 13- or 10-digit structures. +- `opencv-python-headless` pinned to `<5.0.0`; the 5.0 wheel did not ship a working `cv2` module on Python 3.13. +- `import ocr_pipeline` no longer depends on the caller's cwd. + +### Removed +- Unused core dependencies: `tqdm`, `scikit-image`, `sqlite-utils`, `xxhash`, `python-slugify`, `python-magic`, `legacy-cgi`. +- Core dependency on `spacy` (moved to `scientific` extra). +- Core dependency on `paddleocr`/`paddlepaddle` (moved to `paddle` extra). +- Core dependency on `torch`/`torchvision`/`transformers`/`timm` (moved to `tables` extra). +- Core dependency on `layoutparser` (moved to `figures` extra). + +## [0.1.0] - Initial + +Initial release with PaddleOCR + Tesseract fallback, scispaCy NER, layout-aware detectors, citation matching, and Markdown output with YAML frontmatter. diff --git a/README.md b/README.md index e4c22e4..90cd324 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # OCR Pipeline for Life Science Screenshots +[![Tests](https://img.shields.io/badge/tests-passing-brightgreen)](./.github/workflows/tests.yml) + Turns scientific screenshots into RAG-ready Markdown, with metadata attached. - OCR via PaddleOCR, falls back to Tesseract diff --git a/src/ocr_pipeline/__pycache__/cli.cpython-313.pyc b/src/ocr_pipeline/__pycache__/cli.cpython-313.pyc index 37d722c..e31b9fd 100644 Binary files a/src/ocr_pipeline/__pycache__/cli.cpython-313.pyc and b/src/ocr_pipeline/__pycache__/cli.cpython-313.pyc differ diff --git a/src/ocr_pipeline/citations/__pycache__/regex_extractor.cpython-313.pyc b/src/ocr_pipeline/citations/__pycache__/regex_extractor.cpython-313.pyc index 99cafaf..16be90a 100644 Binary files a/src/ocr_pipeline/citations/__pycache__/regex_extractor.cpython-313.pyc and b/src/ocr_pipeline/citations/__pycache__/regex_extractor.cpython-313.pyc differ diff --git a/src/ocr_pipeline/citations/regex_extractor.py b/src/ocr_pipeline/citations/regex_extractor.py index 24d0d8c..7d984e9 100644 --- a/src/ocr_pipeline/citations/regex_extractor.py +++ b/src/ocr_pipeline/citations/regex_extractor.py @@ -34,7 +34,13 @@ class RegexCitationExtractor: re.IGNORECASE, ) ISBN_PATTERN = re.compile( - r"\b(?:ISBN[-\s]*(?:13|10)?[-\s]*)?(?:97[89][-\s]*)?\d{1,5}[-\s]*\d{1,7}[-\s]*\d{1,7}[-\s]*[\dX]\b", + r"(?:" + # 13-digit: ISBN[-: ]*(13)?[-: ]* 978|979 [-: ]* group groups group check + r"(?P\bISBN[-:\s]*(?:13[-:\s]*)?97[89][-:\s]*\d{1,5}[-:\s]*\d{1,7}[-:\s]*\d{1,7}[-:\s]*[\dX]\b)" + r"|" + # 10-digit: ISBN[-: ]*(10)?[-: ]* group - group - check + r"(?P\bISBN[-:\s]*(?:10[-:\s]*)?\d{1,5}[-:\s]*\d{1,7}[-:\s]*\d{1,7}[-:\s]*[\dX]\b)" + r")", re.IGNORECASE, ) @@ -84,12 +90,15 @@ class RegexCitationExtractor: for match in cls.ISBN_PATTERN.finditer(text): ctx_start = max(0, match.start() - 80) ctx_end = min(len(text), match.end() + 80) - isbn = re.sub(r"[-\s]", "", match.group()) + raw = match.group() + # Drop the leading "ISBN" prefix and any separators to get the digits/check. + isbn_digits = re.sub(r"^(?:ISBN[-:\s]*(?:1[03])?[-:\s]*)", "", raw, flags=re.IGNORECASE) + isbn_digits = re.sub(r"[-:\s]", "", isbn_digits) citations.append( Citation( - raw=match.group(), + raw=raw, type="isbn", - identifier=f"ISBN:{isbn}", + identifier=f"ISBN:{isbn_digits}", confidence=0.7, context=text[ctx_start:ctx_end].strip(), ) diff --git a/src/ocr_pipeline/cli.py b/src/ocr_pipeline/cli.py index 8b642cd..d4653ff 100644 --- a/src/ocr_pipeline/cli.py +++ b/src/ocr_pipeline/cli.py @@ -10,6 +10,8 @@ from rich.table import Table from ocr_pipeline.config import find_config_file, settings from ocr_pipeline.demo import run_demo +from ocr_pipeline.doctor import exit_code_for, run_checks +from ocr_pipeline.doctor import render as render_doctor from ocr_pipeline.onboarding import ( default_config_location, tesseract_install_hint, @@ -272,6 +274,16 @@ def setup( console.print("[green]Setup complete.[/green]") +@app.command() +def doctor(): + """Check the install: engines, ML models, config, database, output dir.""" + results = run_checks() + render_doctor(results, console) + code = exit_code_for(results) + if code != 0: + raise typer.Exit(code=code) + + @app.command() def config(): """Show current configuration""" diff --git a/src/ocr_pipeline/doctor.py b/src/ocr_pipeline/doctor.py new file mode 100644 index 0000000..4458973 --- /dev/null +++ b/src/ocr_pipeline/doctor.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import importlib.util +import shutil +import sys +from dataclasses import dataclass +from typing import Any + +from ocr_pipeline.config import settings +from ocr_pipeline.onboarding import tesseract_install_hint +from ocr_pipeline.utils.logging import get_logger + +logger = get_logger(__name__) + + +@dataclass(frozen=True) +class CheckResult: + name: str + status: str # "ok" | "warn" | "fail" + detail: str + hint: str | None = None + + +def _has_module(name: str) -> bool: + return importlib.util.find_spec(name) is not None + + +def check_tesseract() -> CheckResult: + path = shutil.which("tesseract") + if path: + return CheckResult("Tesseract", "ok", f"found at {path}") + return CheckResult("Tesseract", "fail", "not on PATH", f"Install: {tesseract_install_hint()}") + + +def check_paddle() -> CheckResult: + if not _has_module("paddleocr"): + return CheckResult("PaddleOCR", "warn", "not installed", "uv sync --extra paddle") + try: + from paddleocr import PaddleOCR # type: ignore[import-not-found] + + PaddleOCR(use_angle_cls=True, lang="en", use_gpu=False, show_log=False) + return CheckResult("PaddleOCR", "ok", "initialized (en)") + except Exception as exc: + return CheckResult("PaddleOCR", "warn", f"init failed: {exc}") + + +def check_torch() -> CheckResult: + if not _has_module("torch"): + return CheckResult("PyTorch", "warn", "not installed", "uv sync --extra tables") + import torch # type: ignore[import-not-found] + + cuda = torch.cuda.is_available() + mps = getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available() + backend = "cuda" if cuda else "mps" if mps else "cpu" + return CheckResult("PyTorch", "ok", f"backend={backend}") + + +def check_tables() -> CheckResult: + if not _has_module("transformers") or not _has_module("torch"): + return CheckResult( + "Table detector", "warn", "missing torch/transformers", "uv sync --extra tables" + ) + return CheckResult("Table detector", "ok", "transformers + torch present") + + +def check_figures() -> CheckResult: + if not _has_module("layoutparser"): + return CheckResult( + "Figure detector", "warn", "layoutparser not installed", "uv sync --extra figures" + ) + try: + import layoutparser as lp # type: ignore[import-not-found] + + has_detectron = hasattr(lp, "Detectron2LayoutModel") + has_auto = hasattr(lp, "AutoLayoutModel") + if not (has_detectron or has_auto): + return CheckResult( + "Figure detector", + "warn", + "no layout model backend", + "Install Detectron2 for your platform", + ) + return CheckResult("Figure detector", "ok", "layoutparser + backend present") + except Exception as exc: + return CheckResult("Figure detector", "warn", f"import failed: {exc}") + + +def check_spacy() -> CheckResult: + if not _has_module("spacy"): + return CheckResult( + "scispaCy NER", "warn", "spacy not installed", "uv sync --extra scientific" + ) + import spacy # type: ignore[import-not-found] + + try: + spacy.load(settings.entities.model) + return CheckResult("scispaCy NER", "ok", f"loaded {settings.entities.model}") + except OSError: + return CheckResult( + "scispaCy NER", + "warn", + f"model {settings.entities.model!r} not downloaded", + "uv pip install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.4/en_core_sci_lg-0.5.4.tar.gz", + ) + + +def check_config() -> CheckResult: + from ocr_pipeline.config import find_config_file + + path = find_config_file() + if path is None: + return CheckResult("Config", "warn", "no config.yaml found", "Run: ocr-pipeline init") + return CheckResult("Config", "ok", f"loaded from {path}") + + +def check_db() -> CheckResult: + db_path = settings.watch_db_path + try: + db_path.parent.mkdir(parents=True, exist_ok=True) + with db_path.open("a"): + pass + return CheckResult("Database", "ok", f"writable at {db_path}") + except OSError as exc: + return CheckResult("Database", "fail", f"not writable: {exc}") + + +def check_output_dir() -> CheckResult: + out = settings.output_dir + try: + out.mkdir(parents=True, exist_ok=True) + return CheckResult("Output dir", "ok", str(out)) + except OSError as exc: + return CheckResult("Output dir", "fail", f"cannot create: {exc}") + + +def check_python() -> CheckResult: + return CheckResult( + "Python", + "ok", + f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", + ) + + +CHECKS = [ + check_python, + check_tesseract, + check_paddle, + check_torch, + check_tables, + check_figures, + check_spacy, + check_config, + check_db, + check_output_dir, +] + + +def run_checks() -> list[CheckResult]: + results: list[CheckResult] = [] + for check in CHECKS: + try: + results.append(check()) + except Exception as exc: # noqa: BLE001 - surfaced as a fail result, never propagated + results.append(CheckResult(check.__name__, "fail", f"unexpected error: {exc}")) + return results + + +def render(results: list[CheckResult], console: Any) -> None: + from rich.table import Table + + status_style = {"ok": "green", "warn": "yellow", "fail": "red"} + icon = {"ok": "[green]OK[/green]", "warn": "[yellow]WARN[/yellow]", "fail": "[red]FAIL[/red]"} + table = Table(title="ocr-pipeline health check") + table.add_column("Status") + table.add_column("Check", style="cyan") + table.add_column("Detail") + for r in results: + table.add_row(icon[r.status], r.name, r.detail) + console.print(table) + + failing = [r for r in results if r.status == "fail"] + warning = [r for r in results if r.status == "warn"] + if failing or warning: + console.print() + for r in failing + warning: + if r.hint: + console.print(f" [{status_style[r.status]}]{r.name}:[/] {r.hint}") + + +def exit_code_for(results: list[CheckResult]) -> int: + return 1 if any(r.status == "fail" for r in results) else 0 diff --git a/tests/test_citations.py b/tests/test_citations.py new file mode 100644 index 0000000..fd755f8 --- /dev/null +++ b/tests/test_citations.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import pytest + +from ocr_pipeline.citations.regex_extractor import Citation, RegexCitationExtractor + + +def _isbns(text: str) -> list[str]: + return [c.identifier for c in RegexCitationExtractor.extract(text) if c.type == "isbn"] + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("ISBN:978-0-306-40615-7", "ISBN:9780306406157"), + ("ISBN 978-0-306-40615-7", "ISBN:9780306406157"), + ("ISBN-13: 978-0-306-40615-7", "ISBN:9780306406157"), + ("ISBN:0-306-40615-2", "ISBN:0306406152"), + ("ISBN 0-306-40615-2", "ISBN:0306406152"), + ("ISBN-10: 0-306-40615-2", "ISBN:0306406152"), + ], +) +def test_recognises_real_isbns(text: str, expected: str) -> None: + assert expected in _isbns(text) + + +@pytest.mark.parametrize( + "text", + [ + "2024-01-15", + "Due in 17 days, 2024", + "https://www.jobbnorge.no/jobs/300588", + "https://euraxess.ec.europa.eu/jobs/432300", + "Anti-BRCA1 (1:1000), Anti-beta-actin (1:5000)", + "30 ug per lane; gel 10% SDS-PAGE", + "Lane 1: WT control", + "PMID: 12345678", + "DOI: 10.1038/nature12345", + ], +) +def test_no_isbn_false_positives(text: str) -> None: + assert _isbns(text) == [] + + +def test_isbn_stripped_to_digits() -> None: + identifiers = _isbns("ISBN: 978-0-306-40615-7") + assert identifiers == ["ISBN:9780306406157"] + + +def test_doi_still_works() -> None: + dois = [ + c for c in RegexCitationExtractor.extract("See DOI: 10.1038/nature12345") if c.type == "doi" + ] + assert len(dois) == 1 + assert dois[0].identifier == "10.1038/nature12345" + + +def test_pmid_still_works() -> None: + pmids = [c for c in RegexCitationExtractor.extract("PMID: 12345678") if c.type == "pmid"] + assert len(pmids) == 1 + assert pmids[0].identifier == "PMID:12345678" + + +def test_citation_dataclass_fields() -> None: + c = Citation(raw="x", type="doi", identifier="y", confidence=0.5) + assert c.context == "" diff --git a/tests/test_doctor.py b/tests/test_doctor.py new file mode 100644 index 0000000..e10d44f --- /dev/null +++ b/tests/test_doctor.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from typer.testing import CliRunner + +from ocr_pipeline.cli import app +from ocr_pipeline.config import settings +from ocr_pipeline.doctor import ( + check_config, + check_db, + check_output_dir, + check_paddle, + check_python, + check_spacy, + check_tables, + check_tesseract, + check_torch, + exit_code_for, + render, + run_checks, +) + +runner = CliRunner() + + +def test_check_python_ok() -> None: + r = check_python() + assert r.status == "ok" + assert "Python" in r.name + + +def test_check_tesseract_ok_when_on_path(monkeypatch) -> None: + monkeypatch.setattr("shutil.which", lambda _: "/usr/bin/tesseract") + r = check_tesseract() + assert r.status == "ok" + assert "/usr/bin/tesseract" in r.detail + + +def test_check_tesseract_fails_when_missing(monkeypatch) -> None: + monkeypatch.setattr("shutil.which", lambda _: None) + r = check_tesseract() + assert r.status == "fail" + assert r.hint is not None + + +def test_check_paddle_warns_when_missing(monkeypatch) -> None: + monkeypatch.setattr( + "ocr_pipeline.doctor._has_module", lambda name: False if name == "paddleocr" else True + ) + r = check_paddle() + assert r.status == "warn" + assert "paddle" in r.hint.lower() if r.hint else True + + +def test_check_torch_reports_backend(monkeypatch) -> None: + monkeypatch.setattr("ocr_pipeline.doctor._has_module", lambda name: True) + fake_torch = type( + "T", (), {"cuda": type("C", (), {"is_available": staticmethod(lambda: False)})()} + ) + monkeypatch.setattr("sys.modules", {"torch": fake_torch}) + r = check_torch() + assert r.status == "ok" + assert "backend=" in r.detail + + +def test_check_tables_warns_when_missing(monkeypatch) -> None: + monkeypatch.setattr( + "ocr_pipeline.doctor._has_module", + lambda name: False if name in ("torch", "transformers") else True, + ) + r = check_tables() + assert r.status == "warn" + + +def test_check_spacy_warns_when_model_missing(monkeypatch, tmp_path) -> None: + monkeypatch.setattr("ocr_pipeline.doctor._has_module", lambda name: True) + + class FakeSpacy: + @staticmethod + def load(_model): + raise OSError("model not found") + + monkeypatch.setitem(__import__("sys").modules, "spacy", FakeSpacy) + r = check_spacy() + assert r.status == "warn" + assert "en_core_sci_lg" in (r.hint or "") + + +def test_check_config_warns_when_no_file(monkeypatch, tmp_path) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("ocr_pipeline.config.user_config_dir", lambda: tmp_path / "missing") + r = check_config() + assert r.status == "warn" + assert "init" in (r.hint or "") + + +def test_check_db_writes_temp_file(tmp_path, monkeypatch) -> None: + db = tmp_path / "sub" / "test.db" + monkeypatch.setattr(settings.watch, "db_path", str(db)) + r = check_db() + assert r.status == "ok" + assert db.exists() + + +def test_check_output_dir_creates_path(tmp_path, monkeypatch) -> None: + out = tmp_path / "deep" / "output" + monkeypatch.setattr(settings.output, "base_directory", str(out)) + r = check_output_dir() + assert r.status == "ok" + assert out.is_dir() + + +def test_exit_code_for() -> None: + from ocr_pipeline.doctor import CheckResult + + assert exit_code_for([]) == 0 + assert exit_code_for([CheckResult("a", "ok", "x")]) == 0 + assert exit_code_for([CheckResult("a", "warn", "x")]) == 0 + assert exit_code_for([CheckResult("a", "fail", "x")]) == 1 + + +def test_run_checks_returns_all_categories() -> None: + results = run_checks() + names = {r.name for r in results} + assert {"Tesseract", "PaddleOCR", "Python", "Config"}.issubset(names) + + +def test_render_writes_table(capsys) -> None: + from rich.console import Console + + from ocr_pipeline.doctor import CheckResult + + console = Console(file=__import__("sys").stdout) + render( + [CheckResult("A", "ok", "fine"), CheckResult("B", "fail", "bad", hint="fix me")], console + ) + out = capsys.readouterr().out + assert "A" in out and "B" in out + + +def test_doctor_cli_runs(monkeypatch) -> None: + from ocr_pipeline.doctor import CheckResult + + monkeypatch.setattr("ocr_pipeline.cli.run_checks", lambda: [CheckResult("X", "ok", "ok")]) + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 0 + assert "X" in result.output + + +def test_doctor_cli_exit_code_on_fail(monkeypatch) -> None: + from ocr_pipeline.doctor import CheckResult + + monkeypatch.setattr("ocr_pipeline.cli.run_checks", lambda: [CheckResult("X", "fail", "bad")]) + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 1