refactor: solidify startup UX and engine-aware preprocessing

- Slim core deps: move ML stack to optional extras (paddle/tables/figures/scientific/full)
- Lazy settings proxy with config search paths (env var, cwd, user dir)
- New commands: init, demo, setup [basic|full], first-run guard on run/watch
- Engine-aware preprocessing: Paddle gets original image (fixes dark mode 0.83->0.95)
- Results table shows Skipped count; lazy run-dir creation
- kg_ocr marked experimental with extra, Docker defaults with OCR_PIPELINE_CONFIG
- 25/25 tests, ruff clean
This commit is contained in:
2026-07-19 22:14:54 +02:00
parent deab02610b
commit a0211155fa
59 changed files with 2426 additions and 629 deletions

View File

@@ -74,4 +74,6 @@ def clean_text(text: str) -> CleanResult:
text = re.sub(r"\n{3,}", "\n\n", text).strip()
operations.append("normalize_whitespace")
logger.debug("text_cleaned", original=original_length, cleaned=len(text), ops=operations)
return CleanResult(text=text, original_length=original_length, cleaned_length=len(text), operations=operations)
return CleanResult(
text=text, original_length=original_length, cleaned_length=len(text), operations=operations
)

View File

@@ -3,8 +3,6 @@ from __future__ import annotations
import re
from dataclasses import dataclass
import spacy
from ocr_pipeline.config import settings
from ocr_pipeline.utils.logging import get_logger
@@ -40,7 +38,17 @@ class ScientificEntityRecognizer:
if self._initialized:
return
try:
import spacy
except ImportError:
spacy = None # type: ignore[assignment]
if spacy is None:
logger.info(
"spacy_not_installed",
hint="Install with: uv sync --extra scientific (regex entities still run)",
)
self._initialized = True
return
try:
self._nlp = spacy.load(settings.entities.model)
self._nlp.max_length = 2_000_000
self._initialized = True
@@ -48,10 +56,10 @@ class ScientificEntityRecognizer:
logger.info("scispacy_model_loaded", model=settings.entities.model)
except Exception as e:
logger.warning("scispacy_load_failed", error=str(e), model=settings.entities.model)
self._fallback_init()
self._fallback_init(spacy)
self._initialized = True
def _fallback_init(self):
def _fallback_init(self, spacy):
try:
self._nlp = spacy.blank("en")
self._nlp.add_pipe("sentencizer")
@@ -90,7 +98,9 @@ class ScientificEntityRecognizer:
)
entities = (
self._merge_entities(entities, text) if settings.entities.merge_entities else entities
self._merge_entities(entities, text)
if settings.entities.merge_entities
else entities
)
entities = self._deduplicate_entities(entities)
@@ -153,7 +163,6 @@ class RegexEntityRecognizer:
],
"CHEMICAL": [
r"\b(?:DMSO|PBS|EDTA|Tris|HEPES|SDS|DTT|BME|NaCl|KCl|MgCl2|CaCl2|NaOH|HCl|H2SO4|HNO3|EtOH|MeOH|IPA|DMSO|DMF|DMA|THF|DCM|CHCl3|CH2Cl2|EtOAc|hexane|pentane|acetone|acetonitrile|water|H2O|buffer|media|serum|FBS|BSA|pen/strep|penicillin|streptomycin|trypsin|EDTA|collagenase|dispase|accutase)\b",
],
"CONCENTRATION": [
r"\b\d+(?:\.\d+)?\s*(?:[µumMpnc]?[Mm]|[µumMpnc]?[Mm]/[Ll]|[µumMpnc]?[Mm]\s*[Ll]?|[µumMpnc]?[gG]\s*/\s*[Ll]|[µumMpnc]?[gG]\s*/\s*[mM][lL]|[µumMpnc]?[gG]\s*/\s*[dL]|[µumMpnc]?[gG]\s*/\s*[mM][lL]|[µumMpnc]?[gG]\s*/\s*[dD][lL]|[µumMpnc]?[gG]\s*/\s*100\s*[mM][lL]|[µumMpnc]?[gG]\s*/\s*[kK][gG])\b",