OCR-to-RAG pipeline for life science screenshots

PaddleOCR with preprocessing, scispaCy NER, figure/table detection,
citation extraction, and chunked Markdown output with frontmatter.
Includes watch mode and notebook reprocessing.

src/ocr_pipeline/watch/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .watcher import Watcher
+
+__all__ = ["Watcher"]

src/ocr_pipeline/watch/watcher.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from watchdog.events import FileSystemEventHandler
+from watchdog.observers import Observer
+
+from ocr_pipeline.config import settings
+from ocr_pipeline.utils.db import get_db
+from ocr_pipeline.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class ScreenshotHandler(FileSystemEventHandler):
+    def __init__(self, process_callback: Callable[[Path], None]):
+        self.process_callback = process_callback
+        self.pending: dict[str, float] = {}
+        self.lock = threading.Lock()
+        self.debounce = settings.watch.debounce_seconds
+
+    def _should_process(self, path: Path) -> bool:
+        if not path.is_file():
+            return False
+        if path.suffix.lower() not in (".png", ".jpg", ".jpeg", ".tiff", ".bmp"):
+            return False
+        for pattern in settings.watch.ignore_patterns:
+            if path.match(pattern):
+                return False
+        return True
+
+    def on_created(self, event):
+        if not event.is_directory:
+            path = Path(event.src_path)
+            if self._should_process(path):
+                with self.lock:
+                    self.pending[str(path)] = time.time()
+
+    def on_modified(self, event):
+        if not event.is_directory:
+            path = Path(event.src_path)
+            if self._should_process(path):
+                with self.lock:
+                    self.pending[str(path)] = time.time()
+
+    def check_pending(self) -> list[Path]:
+        now = time.time()
+        to_process = []
+
+        with self.lock:
+            for path_str, timestamp in list(self.pending.items()):
+                if now - timestamp >= self.debounce:
+                    path = Path(path_str)
+                    if path.exists():
+                        to_process.append(path)
+                    del self.pending[path_str]
+
+        return to_process
+
+
+class Watcher:
+    def __init__(self, process_callback: Callable[[Path], None]):
+        self.process_callback = process_callback
+        self.observer: Observer | None = None
+        self.handler: ScreenshotHandler | None = None
+        self.running = False
+        self.db = get_db()
+        self.run_id: int | None = None
+
+    def start(self) -> None:
+        if self.running:
+            return
+
+        self.run_id = self.db.start_run()
+        self.running = True
+
+        self.handler = ScreenshotHandler(self._process_file)
+        self.observer = Observer()
+
+        for base_path_str in settings.input.paths:
+            base_path = Path(base_path_str).expanduser()
+            if base_path.exists():
+                self.observer.schedule(
+                    self.handler, str(base_path), recursive=settings.input.recursive
+                )
+                logger.info("watching_directory", path=str(base_path))
+
+        self.observer.start()
+
+        self._watch_thread = threading.Thread(target=self._watch_loop, daemon=True)
+        self._watch_thread.start()
+
+        logger.info("watcher_started", run_id=self.run_id)
+
+    def _watch_loop(self) -> None:
+        poll_interval = settings.watch.poll_interval
+        processed = 0
+        failed = 0
+
+        while self.running:
+            time.sleep(poll_interval)
+
+            if self.handler:
+                for path in self.handler.check_pending():
+                    try:
+                        self._process_file(path)
+                        processed += 1
+                    except Exception as e:
+                        logger.error("watch_process_failed", path=str(path), error=str(e))
+                        failed += 1
+
+        self.db.complete_run(
+            time.strftime("%Y-%m-%d"),
+            self.run_id,
+            {"total": processed, "successful": processed, "failed": failed, "chunks": 0},
+        )
+        logger.info("watcher_stopped", run_id=self.run_id, processed=processed, failed=failed)
+
+    def _process_file(self, path: Path) -> None:
+        from ocr_pipeline.ocr.engine import compute_image_hash
+
+        file_hash = compute_image_hash(path)
+        if self.db.is_processed(file_hash, path):
+            logger.debug("file_already_processed", path=str(path))
+            return
+
+        logger.info("processing_new_file", path=str(path))
+        self.process_callback(path)
+        self.db.mark_processed(
+            file_hash=file_hash,
+            path=str(path),
+            timestamp=time.time(),
+            run_id=self.run_id,
+        )
+
+    def stop(self) -> None:
+        self.running = False
+        if self.observer:
+            self.observer.stop()
+            self.observer.join(timeout=5)
+        logger.info("watcher_stop_requested")