Aman
a32b7508c7
PaddleOCR with preprocessing, scispaCy NER, figure/table detection, citation extraction, and chunked Markdown output with frontmatter. Includes watch mode and notebook reprocessing.
73 lines
2.4 KiB
Python
73 lines
2.4 KiB
Python
import pytest
|
||
from pathlib import Path
|
||
|
||
from ocr_pipeline.postprocess.clean import clean_text, CleanResult
|
||
from ocr_pipeline.postprocess.chunk import chunk_text, TextChunk
|
||
from ocr_pipeline.config import settings
|
||
|
||
|
||
class TestTextCleaning:
|
||
def test_fix_encoding(self):
|
||
result = clean_text("café")
|
||
assert "café" in result.text or "cafe" in result.text
|
||
|
||
def test_normalize_whitespace(self):
|
||
result = clean_text("hello world\n\n\n\nfoo")
|
||
assert "hello world" in result.text
|
||
assert "foo" in result.text
|
||
|
||
def test_fix_hyphenation(self):
|
||
result = clean_text("hyphen-\nated")
|
||
assert "hyphenated" in result.text
|
||
|
||
def test_scientific_notation(self):
|
||
result = clean_text("1.5 x 10 ^ -3")
|
||
assert "×10^-3" in result.text or "x10^-3" in result.text
|
||
|
||
def test_normalize_units(self):
|
||
result = clean_text("50 uM and 100 ug/mL")
|
||
assert "µM" in result.text
|
||
assert "µg/mL" in result.text or "µg/mL" in result.text
|
||
|
||
def test_remove_ocr_artifacts(self):
|
||
result = clean_text("normal text\n|||\n───\nmore text")
|
||
assert "normal text" in result.text
|
||
assert "more text" in result.text
|
||
|
||
|
||
class TestChunking:
|
||
def test_basic_chunking(self):
|
||
text = "A" * 500 + "\n\n" + "B" * 500 + "\n\n" + "C" * 500
|
||
chunks = chunk_text(text)
|
||
assert len(chunks) >= 2
|
||
assert all(isinstance(c, TextChunk) for c in chunks)
|
||
|
||
def test_empty_text(self):
|
||
chunks = chunk_text("")
|
||
assert chunks == []
|
||
|
||
def test_single_chunk(self):
|
||
text = "Short text that fits in one chunk"
|
||
chunks = chunk_text(text)
|
||
assert len(chunks) == 1
|
||
|
||
def test_metadata_propagation(self):
|
||
text = "A" * 1500
|
||
meta = {"source": "test.png", "custom": "value"}
|
||
chunks = chunk_text(text, meta)
|
||
assert all(c.metadata.get("source") == "test.png" for c in chunks)
|
||
assert all(c.metadata.get("custom") == "value" for c in chunks)
|
||
|
||
|
||
class TestConfig:
|
||
def test_settings_load(self):
|
||
assert settings.input.paths
|
||
assert settings.ocr.engine in ("paddleocr", "tesseract", "auto")
|
||
assert settings.output.base_directory
|
||
|
||
def test_output_dir_property(self):
|
||
assert settings.output_dir.exists() or settings.output_dir.parent.exists()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
pytest.main([__file__, "-v"]) |