refactor: solidify startup UX and engine-aware preprocessing
- Slim core deps: move ML stack to optional extras (paddle/tables/figures/scientific/full) - Lazy settings proxy with config search paths (env var, cwd, user dir) - New commands: init, demo, setup [basic|full], first-run guard on run/watch - Engine-aware preprocessing: Paddle gets original image (fixes dark mode 0.83->0.95) - Results table shows Skipped count; lazy run-dir creation - kg_ocr marked experimental with extra, Docker defaults with OCR_PIPELINE_CONFIG - 25/25 tests, ruff clean
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -74,4 +74,6 @@ def clean_text(text: str) -> CleanResult:
|
||||
text = re.sub(r"\n{3,}", "\n\n", text).strip()
|
||||
operations.append("normalize_whitespace")
|
||||
logger.debug("text_cleaned", original=original_length, cleaned=len(text), ops=operations)
|
||||
return CleanResult(text=text, original_length=original_length, cleaned_length=len(text), operations=operations)
|
||||
return CleanResult(
|
||||
text=text, original_length=original_length, cleaned_length=len(text), operations=operations
|
||||
)
|
||||
|
||||
@@ -3,8 +3,6 @@ from __future__ import annotations
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
import spacy
|
||||
|
||||
from ocr_pipeline.config import settings
|
||||
from ocr_pipeline.utils.logging import get_logger
|
||||
|
||||
@@ -40,7 +38,17 @@ class ScientificEntityRecognizer:
|
||||
if self._initialized:
|
||||
return
|
||||
try:
|
||||
|
||||
import spacy
|
||||
except ImportError:
|
||||
spacy = None # type: ignore[assignment]
|
||||
if spacy is None:
|
||||
logger.info(
|
||||
"spacy_not_installed",
|
||||
hint="Install with: uv sync --extra scientific (regex entities still run)",
|
||||
)
|
||||
self._initialized = True
|
||||
return
|
||||
try:
|
||||
self._nlp = spacy.load(settings.entities.model)
|
||||
self._nlp.max_length = 2_000_000
|
||||
self._initialized = True
|
||||
@@ -48,10 +56,10 @@ class ScientificEntityRecognizer:
|
||||
logger.info("scispacy_model_loaded", model=settings.entities.model)
|
||||
except Exception as e:
|
||||
logger.warning("scispacy_load_failed", error=str(e), model=settings.entities.model)
|
||||
self._fallback_init()
|
||||
self._fallback_init(spacy)
|
||||
self._initialized = True
|
||||
|
||||
def _fallback_init(self):
|
||||
def _fallback_init(self, spacy):
|
||||
try:
|
||||
self._nlp = spacy.blank("en")
|
||||
self._nlp.add_pipe("sentencizer")
|
||||
@@ -90,7 +98,9 @@ class ScientificEntityRecognizer:
|
||||
)
|
||||
|
||||
entities = (
|
||||
self._merge_entities(entities, text) if settings.entities.merge_entities else entities
|
||||
self._merge_entities(entities, text)
|
||||
if settings.entities.merge_entities
|
||||
else entities
|
||||
)
|
||||
entities = self._deduplicate_entities(entities)
|
||||
|
||||
@@ -153,7 +163,6 @@ class RegexEntityRecognizer:
|
||||
],
|
||||
"CHEMICAL": [
|
||||
r"\b(?:DMSO|PBS|EDTA|Tris|HEPES|SDS|DTT|BME|NaCl|KCl|MgCl2|CaCl2|NaOH|HCl|H2SO4|HNO3|EtOH|MeOH|IPA|DMSO|DMF|DMA|THF|DCM|CHCl3|CH2Cl2|EtOAc|hexane|pentane|acetone|acetonitrile|water|H2O|buffer|media|serum|FBS|BSA|pen/strep|penicillin|streptomycin|trypsin|EDTA|collagenase|dispase|accutase)\b",
|
||||
|
||||
],
|
||||
"CONCENTRATION": [
|
||||
r"\b\d+(?:\.\d+)?\s*(?:[µumMpnc]?[Mm]|[µumMpnc]?[Mm]/[Ll]|[µumMpnc]?[Mm]\s*[Ll]?|[µumMpnc]?[gG]\s*/\s*[Ll]|[µumMpnc]?[gG]\s*/\s*[mM][lL]|[µumMpnc]?[gG]\s*/\s*[dL]|[µumMpnc]?[gG]\s*/\s*[mM][lL]|[µumMpnc]?[gG]\s*/\s*[dD][lL]|[µumMpnc]?[gG]\s*/\s*100\s*[mM][lL]|[µumMpnc]?[gG]\s*/\s*[kK][gG])\b",
|
||||
|
||||
Reference in New Issue
Block a user