aman/kg-scr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve the OCR pipeline processing and outputs formatting

Browse Source
This commit is contained in:
Aman Nalakath
2026-07-19 19:54:53 +02:00
parent d375db47c3
commit 1cf1c6eeab
30 changed files with 3838 additions and 3900 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/ocr_pipeline/postprocess/__init__.py
View File

@@ -2,7 +2,7 @@ from __future__ import annotations
from .chunk import SemanticChunker, TextChunk, chunk_text
from .clean import CleanResult, clean_text
from .entities import Entity, SciSpacyNER, extract_entities, summarize_entities
from .entities import Entity, ScientificEntityRecognizer, extract_entities, summarize_entities
__all__ = [
"clean_text",
@@ -13,5 +13,5 @@ __all__ = [
"extract_entities",
"summarize_entities",
"Entity",
"SciSpacyNER",
"ScientificEntityRecognizer",
]

2
src/ocr_pipeline/postprocess/chunk.py
View File

@@ -3,7 +3,7 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_text_splitters import RecursiveCharacterTextSplitter
from ocr_pipeline.config import settings
from ocr_pipeline.utils.logging import get_logger

162
src/ocr_pipeline/postprocess/clean.py
View File

@@ -24,140 +24,54 @@ def fix_encoding(text: str) -> str:
return text
def normalize_unicode(text: str) -> str:
return unicodedata.normalize("NFC", text)
def fix_ligatures(text: str) -> str:
ligatures = {
"fi": "fi",
"fl": "fl",
"ff": "ff",
"ffi": "ffi",
"ffl": "ffl",
"ſt": "ft",
"st": "st",
"Ꜳ": "AA",
"ꜳ": "aa",
"Ꜵ": "AO",
"ꜵ": "ao",
"Ꜷ": "AU",
"ꜷ": "au",
"Ꜹ": "AV",
"ꜹ": "av",
"Ꜻ": "AV",
"ꜻ": "av",
"Ꜽ": "AY",
"ꜽ": "ay",
}
for lig, repl in ligatures.items():
text = text.replace(lig, repl)
return text
def remove_control_chars(text: str) -> str:
return "".join(ch for ch in text if unicodedata.category(ch)[0] != "C" or ch in "\n\t")
def fix_hyphenation(text: str) -> str:
return re.sub(r"(\w+)-\n(\w+)", r"\1\2", text)
def normalize_whitespace(text: str) -> str:
text = re.sub(r"[ \t]+", " ", text)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def remove_ocr_artifacts(text: str) -> str:
patterns = [
r"^\s*[\|┃║]\s*$",
r"^\s*[─━]\s*$",
r"^\s*[┌┐└┘├┤┬┴┼]\s*$",
r"(.)\1{10,}",
r"^\s*[•·▪▫◦‣⁃]\s*$",
]
lines = text.split("\n")
cleaned = []
for line in lines:
if any(re.match(p, line.strip()) for p in patterns):
continue
cleaned.append(line)
return "\n".join(cleaned)
def fix_scientific_notation(text: str) -> str:
text = re.sub(r"(\d)\s*[×xX]\s*10\s*[\^]?\s*([+-]?\d+)", r"\1×10^\2", text)
text = re.sub(r"(\d)\s*[×xX]\s*10\s*([+-]?\d+)", r"\1×10^\2", text)
return text
def normalize_units(text: str) -> str:
unit_fixes = {
r"(\d)\s*u[mM]\b": r"\1 µm",
r"(\d)\s*[µu]g\b": r"\1 µg",
r"(\d)\s*[mn]g\b": r"\1 mg",
r"(\d)\s*[kK]b\b": r"\1 kb",
r"(\d)\s*[mM]b\b": r"\1 Mb",
r"(\d)\s*[gG]b\b": r"\1 Gb",
r"(\d)\s*[pP]?[mM]\b": r"\1 pm",
r"(\d)\s*[nN][mM]\b": r"\1 nm",
r"(\d)\s*[cC][mM]\b": r"\1 cm",
r"(\d)\s*[mM][lL]\b": r"\1 mL",
r"(\d)\s*[uU][lL]\b": r"\1 µL",
r"(\d)\s*°\s*[cC]\b": r"\1 °C",
}
for pattern, repl in unit_fixes.items():
text = re.sub(pattern, repl, text)
return text
def fix_line_breaks(text: str) -> str:
text = re.sub(r"(?<!\n)\n(?!\n)", " ", text)
text = re.sub(r"\n{3,}", "\n\n", text)
number = r"(\d+(?:\.\d+)?)"
replacements = [
(rf"{number}\s*[uµ]M\b", r"\1 µM"),
(rf"{number}\s*[uµ]m\b", r"\1 µm"),
(rf"{number}\s*[uµ]g\b", r"\1 µg"),
(rf"{number}\s*ng\b", r"\1 ng"),
(rf"{number}\s*mg\b", r"\1 mg"),
(rf"{number}\s*[uµ]L\b", r"\1 µL"),
(rf"{number}\s*mL\b", r"\1 mL"),
(rf"{number}\s*pM\b", r"\1 pM"),
(rf"{number}\s*nM\b", r"\1 nM"),
(rf"{number}\s*mM\b", r"\1 mM"),
(rf"{number}\s*[kK]b\b", r"\1 kb"),
(rf"{number}\s*[mM]b\b", r"\1 Mb"),
(rf"{number}\s*[gG]b\b", r"\1 Gb"),
(rf"{number}\s*°\s*[cC]\b", r"\1 °C"),
]
for pattern, replacement in replacements:
text = re.sub(pattern, replacement, text)
return text
def clean_text(text: str) -> CleanResult:
original_len = len(text)
operations = []
text = fix_encoding(text)
operations.append("fix_encoding")
text = normalize_unicode(text)
original_length = len(text)
operations: list[str] = []
try:
text = fix_encoding(text)
finally:
operations.append("fix_encoding")
text = unicodedata.normalize("NFC", text)
operations.append("normalize_unicode")
text = fix_ligatures(text)
for source, target in {"fi": "fi", "fl": "fl", "ff": "ff", "ffi": "ffi", "ffl": "ffl"}.items():
text = text.replace(source, target)
operations.append("fix_ligatures")
text = remove_control_chars(text)
text = "".join(char for char in text if unicodedata.category(char)[0] != "C" or char in "\n\t")
operations.append("remove_control_chars")
text = fix_hyphenation(text)
text = re.sub(r"(\w+)-\n(\w+)", r"\1\2", text)
operations.append("fix_hyphenation")
text = normalize_whitespace(text)
operations.append("normalize_whitespace")
text = remove_ocr_artifacts(text)
text = re.sub(r"(?m)^\s*[|┃║─━┌┐└┘├┤┬┴┼]+\s*$", "", text)
text = re.sub(r"(.)\1{10,}", "", text)
operations.append("remove_ocr_artifacts")
text = fix_scientific_notation(text)
text = re.sub(r"(\d)\s*[×xX]\s*10\s*\^?\s*([+-]?\d+)", r"\1×10^\2", text)
operations.append("fix_scientific_notation")
text = normalize_units(text)
operations.append("normalize_units")
text = fix_line_breaks(text)
operations.append("fix_line_breaks")
cleaned_len = len(text)
logger.debug("text_cleaned", original=original_len, cleaned=cleaned_len, ops=operations)
return CleanResult(
text=text,
original_length=original_len,
cleaned_length=cleaned_len,
operations=operations,
)
text = re.sub(r"[ \t]+", " ", text)
text = re.sub(r"\n{3,}", "\n\n", text).strip()
operations.append("normalize_whitespace")
logger.debug("text_cleaned", original=original_length, cleaned=len(text), ops=operations)
return CleanResult(text=text, original_length=original_length, cleaned_length=len(text), operations=operations)

30
src/ocr_pipeline/postprocess/entities.py
View File

@@ -27,12 +27,14 @@ class EntityResult:
text: str
total_entities: int
entities_by_type: dict[str, int]
backend: str = "unknown"
class ScientificEntityRecognizer:
def __init__(self):
self._nlp = None
self._initialized = False
self.backend = "unavailable"
def _init(self):
if self._initialized:
@@ -42,6 +44,7 @@ class ScientificEntityRecognizer:
self._nlp = spacy.load(settings.entities.model)
self._nlp.max_length = 2_000_000
self._initialized = True
self.backend = "scispacy"
logger.info("scispacy_model_loaded", model=settings.entities.model)
except Exception as e:
logger.warning("scispacy_load_failed", error=str(e), model=settings.entities.model)
@@ -52,6 +55,7 @@ class ScientificEntityRecognizer:
try:
self._nlp = spacy.blank("en")
self._nlp.add_pipe("sentencizer")
self.backend = "blank_spacy"
logger.info("using_fallback_blank_model")
except Exception:
self._nlp = None
@@ -99,6 +103,7 @@ class ScientificEntityRecognizer:
text=text,
total_entities=len(entities),
entities_by_type=by_type,
backend=self.backend,
)
except Exception as e:
logger.warning("entity_extraction_failed", error=str(e))
@@ -141,15 +146,14 @@ class ScientificEntityRecognizer:
class RegexEntityRecognizer:
PATTERNS = {
"GENE": [
r"\b(?:[A-Z]{2,5}\d*[A-Z]?|[A-Z]{3,10})\b",
r"\b(?:BRCA[12]|TP53|EGFR|KRAS|MYC|PIK3CA|PTEN|APC|MLH1|MSH2|MSH6|PMS2|ATM|CHEK2|PALB2|RAD51|BARD1|BRIP1|CDH1|STK11|VHL|WT1|NF1|NF2|RB1|MEN1|RET|MET|ALK|ROS1|NTRK[123]|BRAF|NRAS|HRAS|IDH[12]|TERT|FGFR[1234]|KIT|PDGFRA|FLT3|JAK2|MPL|CALR|ASXL1|DNMT3A|TET2|SRSF2|U2AF1|SF3B1|ZRSR2|CBL|CSF3R|RUNX1|CEBPA|NPM1|FLT3|DNMT3A|IDH[12]|TET2|ASXL1|SRSF2|U2AF1|SF3B1|ZRSR2|CBL|CSF3R|RUNX1|CEBPA|NPM1)\b",
r"\b(?:BRCA[12]|TP53|EGFR|KRAS|MYC|PIK3CA|PTEN|APC|MLH1|MSH2|MSH6|PMS2|ATM|CHEK2|PALB2|RAD51|BARD1|BRIP1|CDH1|STK11|VHL|WT1|NF1|NF2|RB1|MEN1|RET|MET|ALK|ROS1|NTRK[123]|BRAF|NRAS|HRAS|IDH[12]|TERT|FGFR[1234]|KIT|PDGFRA|FLT3|JAK2|MPL|CALR|ASXL1|DNMT3A|TET2|SRSF2|U2AF1|SF3B1|ZRSR2|CBL|CSF3R|RUNX1|CEBPA|NPM1)\b",
],
"PROTEIN": [
r"\b(?:p53|p21|p16|CDK[1246]|cyclin\s+[ADEB]|Rb|E2F|MDM2|BAX|BCL[2XL]|CASP[389]|PARP|H2AX|ATM|ATR|CHK[12]|BRCA[12]|RAD51|FANCD2|FANCI|PCNA|RPA|MSH[26]|MLH1|PMS2|EXO1|MRE11|RAD50|NBN|CTIP|BRCA1|BRCA2|PALB2|RAD51C|RAD51D|BRIP1|BARD1)\b",
],
"CHEMICAL": [
r"\b(?:DMSO|PBS|EDTA|Tris|HEPES|SDS|DTT|BME|NaCl|KCl|MgCl2|CaCl2|NaOH|HCl|H2SO4|HNO3|EtOH|MeOH|IPA|DMSO|DMF|DMA|THF|DCM|CHCl3|CH2Cl2|EtOAc|hexane|pentane|acetone|acetonitrile|water|H2O|buffer|media|serum|FBS|BSA|pen/strep|penicillin|streptomycin|trypsin|EDTA|collagenase|dispase|accutase)\b",
r"\b(?:[A-Z][a-z]?\d*(?:\([^)]+\))?(?:\s*[+\-]\s*[A-Z][a-z]?\d*(?:\([^)]+\))?)*)\b",
],
"CONCENTRATION": [
r"\b\d+(?:\.\d+)?\s*(?:[µumMpnc]?[Mm]|[µumMpnc]?[Mm]/[Ll]|[µumMpnc]?[Mm]\s*[Ll]?|[µumMpnc]?[gG]\s*/\s*[Ll]|[µumMpnc]?[gG]\s*/\s*[mM][lL]|[µumMpnc]?[gG]\s*/\s*[dL]|[µumMpnc]?[gG]\s*/\s*[mM][lL]|[µumMpnc]?[gG]\s*/\s*[dD][lL]|[µumMpnc]?[gG]\s*/\s*100\s*[mM][lL]|[µumMpnc]?[gG]\s*/\s*[kK][gG])\b",
@@ -177,9 +181,14 @@ class RegexEntityRecognizer:
return entities
_scientific_recognizer = ScientificEntityRecognizer()
def extract_entities(text: str) -> EntityResult:
recognizer = ScientificEntityRecognizer()
result = recognizer.extract(text)
if not settings.entities.enabled:
return EntityResult(entities=[], text=text, total_entities=0, entities_by_type={})
result = _scientific_recognizer.extract(text)
regex_entities = RegexEntityRecognizer.extract(text)
all_entities = result.entities + regex_entities
@@ -194,6 +203,7 @@ def extract_entities(text: str) -> EntityResult:
text=text,
total_entities=len(all_entities),
entities_by_type=by_type,
backend=f"{result.backend}+regex" if result.backend != "unavailable" else "regex",
)
@@ -225,3 +235,13 @@ def format_entities_for_markdown(result: EntityResult) -> str:
lines.append(f"| {ents[0].text} | {label} | {len(ents)} | {'; '.join(contexts)} |")
return "\n".join(lines)
def summarize_entities(result: EntityResult) -> str:
if not result.entities:
return "No entities detected."
parts = [f"Total entities: {result.total_entities}"]
for label, count in sorted(result.entities_by_type.items()):
parts.append(f" {label}: {count}")
return "\n".join(parts)