Aman Nalakath
a0211155fa
- Slim core deps: move ML stack to optional extras (paddle/tables/figures/scientific/full) - Lazy settings proxy with config search paths (env var, cwd, user dir) - New commands: init, demo, setup [basic|full], first-run guard on run/watch - Engine-aware preprocessing: Paddle gets original image (fixes dark mode 0.83->0.95) - Results table shows Skipped count; lazy run-dir creation - kg_ocr marked experimental with extra, Docker defaults with OCR_PIPELINE_CONFIG - 25/25 tests, ruff clean
73 lines
2.3 KiB
Python
73 lines
2.3 KiB
Python
import pytest
|
||
|
||
from ocr_pipeline.config import settings
|
||
from ocr_pipeline.postprocess.chunk import TextChunk, chunk_text
|
||
from ocr_pipeline.postprocess.clean import clean_text
|
||
|
||
|
||
class TestTextCleaning:
|
||
def test_fix_encoding(self):
|
||
result = clean_text("café")
|
||
assert "café" in result.text or "cafe" in result.text
|
||
|
||
def test_normalize_whitespace(self):
|
||
result = clean_text("hello world\n\n\n\nfoo")
|
||
assert "hello world" in result.text
|
||
assert "foo" in result.text
|
||
|
||
def test_fix_hyphenation(self):
|
||
result = clean_text("hyphen-\nated")
|
||
assert "hyphenated" in result.text
|
||
|
||
def test_scientific_notation(self):
|
||
result = clean_text("1.5 x 10 ^ -3")
|
||
assert "×10^-3" in result.text or "x10^-3" in result.text
|
||
|
||
def test_normalize_units(self):
|
||
result = clean_text("50 uM and 100 ug/mL")
|
||
assert "µM" in result.text
|
||
assert "µg/mL" in result.text or "µg/mL" in result.text
|
||
|
||
def test_remove_ocr_artifacts(self):
|
||
result = clean_text("normal text\n|||\n───\nmore text")
|
||
assert "normal text" in result.text
|
||
assert "more text" in result.text
|
||
|
||
|
||
class TestChunking:
|
||
def test_basic_chunking(self):
|
||
text = "A" * 500 + "\n\n" + "B" * 500 + "\n\n" + "C" * 500
|
||
chunks = chunk_text(text)
|
||
assert len(chunks) >= 2
|
||
assert all(isinstance(c, TextChunk) for c in chunks)
|
||
|
||
def test_empty_text(self):
|
||
chunks = chunk_text("")
|
||
assert chunks == []
|
||
|
||
def test_single_chunk(self):
|
||
text = "Short text that fits in one chunk"
|
||
chunks = chunk_text(text)
|
||
assert len(chunks) == 1
|
||
|
||
def test_metadata_propagation(self):
|
||
text = "A" * 1500
|
||
meta = {"source": "test.png", "custom": "value"}
|
||
chunks = chunk_text(text, meta)
|
||
assert all(c.metadata.get("source") == "test.png" for c in chunks)
|
||
assert all(c.metadata.get("custom") == "value" for c in chunks)
|
||
|
||
|
||
class TestConfig:
|
||
def test_settings_load(self):
|
||
assert settings.input.paths
|
||
assert settings.ocr.engine in ("paddleocr", "tesseract", "auto")
|
||
assert settings.output.base_directory
|
||
|
||
def test_output_dir_property(self):
|
||
assert settings.output_dir.exists() or settings.output_dir.parent.exists()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
pytest.main([__file__, "-v"])
|