platform agnosticity
This commit is contained in:
54
config.yaml
54
config.yaml
@@ -1,7 +1,6 @@
|
||||
input:
|
||||
paths:
|
||||
- "~/Pictures"
|
||||
- "/mnt/storage3/aman/screenshots"
|
||||
patterns:
|
||||
- "SCR-*.png"
|
||||
- "*.jpg"
|
||||
@@ -11,8 +10,8 @@ input:
|
||||
recursive: true
|
||||
|
||||
ocr:
|
||||
engine: "paddleocr" # "paddleocr" | "tesseract" | "auto"
|
||||
languages: ["en"] # For tesseract: eng, lat, deu, fra, spa, ita, por, chi_sim, jpn, kor
|
||||
engine: "paddleocr"
|
||||
languages: ["en", "latin"]
|
||||
use_gpu: false
|
||||
use_angle_cls: true
|
||||
det_db_thresh: 0.3
|
||||
@@ -20,13 +19,6 @@ ocr:
|
||||
det_db_unclip_ratio: 1.5
|
||||
rec_batch_num: 6
|
||||
cpu_threads: 4
|
||||
preprocess:
|
||||
deskew: true
|
||||
denoise: true
|
||||
clahe: true
|
||||
adaptive_threshold: true
|
||||
remove_lines: true
|
||||
max_dimension: 4096
|
||||
|
||||
processing:
|
||||
workers: 4
|
||||
@@ -76,30 +68,32 @@ chunking:
|
||||
keep_separator: true
|
||||
|
||||
output:
|
||||
base_directory: "./data/ocr_output"
|
||||
# Uses platformdirs default: ~/.local/share/ocr-pipeline (Linux), ~/Library/Application Support/ocr-pipeline (macOS), %LOCALAPPDATA%\ocr-pipeline (Windows)
|
||||
format: "markdown"
|
||||
organize_by: "date_run" # "date_run" | "source_dir" | "flat"
|
||||
organize_by: "date_run"
|
||||
frontmatter:
|
||||
- "source_path"
|
||||
- "source_hash"
|
||||
- "timestamp"
|
||||
- "ocr_engine"
|
||||
- "ocr_confidence_mean"
|
||||
- "language"
|
||||
- "detected_entities"
|
||||
- "has_figures"
|
||||
- "has_tables"
|
||||
- "citations_found"
|
||||
- "chunk_index"
|
||||
- "total_chunks"
|
||||
fields:
|
||||
- "source_path"
|
||||
- "source_hash"
|
||||
- "timestamp"
|
||||
- "ocr_engine"
|
||||
- "ocr_confidence_mean"
|
||||
- "language"
|
||||
- "detected_entities"
|
||||
- "has_figures"
|
||||
- "has_tables"
|
||||
- "citations_found"
|
||||
- "chunk_index"
|
||||
- "total_chunks"
|
||||
include_raw_text: false
|
||||
|
||||
watch:
|
||||
enabled: true
|
||||
debounce_seconds: 5
|
||||
ignore_patterns:
|
||||
- ".DS_Store"
|
||||
- "*.tmp"
|
||||
- "*.partial"
|
||||
- "*.crdownload"
|
||||
db_path: "./data/processed_files.db"
|
||||
# Uses platform-aware defaults from config.py
|
||||
# ignore_patterns:
|
||||
# - ".DS_Store"
|
||||
# - "*.tmp"
|
||||
# - "*.partial"
|
||||
# - "*.crdownload"
|
||||
poll_interval: 1.0
|
||||
@@ -44,6 +44,7 @@ dependencies = [
|
||||
"python-slugify>=8.0.0",
|
||||
"xxhash>=3.4.0",
|
||||
"python-magic>=0.4.27",
|
||||
"platformdirs>=4.2.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
@@ -75,6 +76,7 @@ dev-dependencies = [
|
||||
"mypy>=1.8.0",
|
||||
"pre-commit>=3.6.0",
|
||||
]
|
||||
override-dependencies = ["lxml>=5.3.0"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
|
||||
@@ -1,14 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import platform
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import platformdirs
|
||||
import yaml
|
||||
from pydantic import Field, field_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
APP_NAME = "ocr-pipeline"
|
||||
APP_AUTHOR = "aman"
|
||||
|
||||
|
||||
def _get_platform_ignore_patterns() -> list[str]:
|
||||
"""Get platform-specific ignore patterns for file watching."""
|
||||
patterns = [
|
||||
"*.tmp",
|
||||
"*.partial",
|
||||
"*.crdownload",
|
||||
"*.part",
|
||||
"~$*", # MS Office temp files
|
||||
"*.swp",
|
||||
"*.swo", # Vim swap files
|
||||
]
|
||||
system = platform.system().lower()
|
||||
if system == "darwin":
|
||||
patterns.extend([".DS_Store", "._*", ".Spotlight-V100", ".Trashes", ".fseventsd"])
|
||||
elif system == "windows":
|
||||
patterns.extend(["Thumbs.db", "desktop.ini", "*.lnk", "$RECYCLE.BIN"])
|
||||
else:
|
||||
# Linux/Unix
|
||||
patterns.extend([".directory", "*.swp", "*.swo", ".git"])
|
||||
return patterns
|
||||
|
||||
|
||||
class InputConfig(BaseSettings):
|
||||
paths: list[str] = Field(default_factory=lambda: ["~/Pictures"])
|
||||
patterns: list[str] = Field(
|
||||
@@ -132,7 +160,9 @@ class FrontmatterConfig(BaseSettings):
|
||||
|
||||
|
||||
class OutputConfig(BaseSettings):
|
||||
base_directory: str = "./data/ocr_output"
|
||||
base_directory: str = Field(
|
||||
default_factory=lambda: str(platformdirs.user_data_dir(APP_NAME, APP_AUTHOR))
|
||||
)
|
||||
format: str = "markdown"
|
||||
organize_by: str = "date_run" # date_run | source_dir | flat
|
||||
frontmatter: FrontmatterConfig = Field(default_factory=FrontmatterConfig)
|
||||
@@ -141,15 +171,10 @@ class OutputConfig(BaseSettings):
|
||||
class WatchConfig(BaseSettings):
|
||||
enabled: bool = True
|
||||
debounce_seconds: int = 5
|
||||
ignore_patterns: list[str] = Field(
|
||||
default_factory=lambda: [
|
||||
".DS_Store",
|
||||
"*.tmp",
|
||||
"*.partial",
|
||||
"*.crdownload",
|
||||
]
|
||||
ignore_patterns: list[str] = Field(default_factory=_get_platform_ignore_patterns)
|
||||
db_path: str = Field(
|
||||
default_factory=lambda: str(Path(platformdirs.user_data_dir(APP_NAME, APP_AUTHOR)) / "processed_files.db")
|
||||
)
|
||||
db_path: str = "./data/processed_files.db"
|
||||
poll_interval: float = 1.0
|
||||
|
||||
|
||||
|
||||
@@ -1,16 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import platform
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import nbformat
|
||||
import platformdirs
|
||||
|
||||
from ocr_pipeline.config import settings
|
||||
from ocr_pipeline.utils.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def _get_pictures_dir() -> Path:
|
||||
"""Get platform-appropriate Pictures directory."""
|
||||
system = platform.system().lower()
|
||||
if system == "darwin":
|
||||
return Path.home() / "Pictures"
|
||||
elif system == "windows":
|
||||
# Windows: Pictures folder in user profile
|
||||
return Path.home() / "Pictures"
|
||||
else:
|
||||
# Linux/Unix: XDG Pictures dir or ~/Pictures
|
||||
pictures = Path.home() / "Pictures"
|
||||
if pictures.exists():
|
||||
return pictures
|
||||
# Fallback to XDG_PICTURES_DIR
|
||||
xdg = os.environ.get("XDG_PICTURES_DIR")
|
||||
if xdg:
|
||||
return Path(xdg)
|
||||
return pictures
|
||||
|
||||
|
||||
def parse_notebook_for_images(notebook_path: Path) -> list[Path]:
|
||||
image_paths = []
|
||||
try:
|
||||
@@ -38,7 +62,7 @@ def parse_notebook_for_images(notebook_path: Path) -> list[Path]:
|
||||
if part.startswith("SCR-") and any(
|
||||
part.endswith(ext) for ext in (".png", ".jpg", ".jpeg")
|
||||
):
|
||||
path = Path("/Users/Aman/Pictures") / part
|
||||
path = _get_pictures_dir() / part
|
||||
if path.exists():
|
||||
image_paths.append(path)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user