kg: graph build, traversal queries, neo4j export - kg_ocr.graph builds a networkx graph (docs, chunks, entities, citations, co-occurrence) from chunk markdown - analyzer for summaries, top entities/citations, anomaly checks - traversal: chunks_for_entity/citation, related_entities, expand_context - export: JSON round-trip, GraphML, batched MERGE into neo4j - new CLI: ocr-pipeline kg build|stats|query|export - lazy kg_ocr imports, networkx/neo4j behind extras - dropped dead watch.py shim, added KgConfig stub - trimmed README, updated TODO
Some checks failed
tests / core (macos-latest, py3.11) (push) Has been cancelled
tests / core (macos-latest, py3.12) (push) Has been cancelled
tests / core (macos-latest, py3.13) (push) Has been cancelled
tests / core (ubuntu-latest, py3.11) (push) Has been cancelled
tests / core (ubuntu-latest, py3.12) (push) Has been cancelled
tests / core (ubuntu-latest, py3.13) (push) Has been cancelled
tests / doctor CLI smoke test (push) Has been cancelled
Some checks failed
tests / core (macos-latest, py3.11) (push) Has been cancelled
tests / core (macos-latest, py3.12) (push) Has been cancelled
tests / core (macos-latest, py3.13) (push) Has been cancelled
tests / core (ubuntu-latest, py3.11) (push) Has been cancelled
tests / core (ubuntu-latest, py3.12) (push) Has been cancelled
tests / core (ubuntu-latest, py3.13) (push) Has been cancelled
tests / doctor CLI smoke test (push) Has been cancelled
This commit is contained in:
@@ -1 +1,31 @@
|
||||
# in future prefer a config, especially for graph traversal
|
||||
"""Config for kg_ocr: graph traversal defaults and Neo4j credentials.
|
||||
|
||||
Environment variables: NEO4J_URI, NEO4J_USER, NEO4J_PASSWORD.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KgConfig:
|
||||
"""Tunables for graph building/traversal plus Neo4j connection."""
|
||||
|
||||
# Traversal
|
||||
expand_limit: int = 5
|
||||
related_limit: int = 10
|
||||
|
||||
# Neo4j
|
||||
neo4j_uri: str = "bolt://localhost:7687"
|
||||
neo4j_user: str = "neo4j"
|
||||
neo4j_password: str = field(default="", repr=False)
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> KgConfig:
|
||||
return cls(
|
||||
neo4j_uri=os.environ.get("NEO4J_URI", cls.neo4j_uri),
|
||||
neo4j_user=os.environ.get("NEO4J_USER", cls.neo4j_user),
|
||||
neo4j_password=os.environ.get("NEO4J_PASSWORD", ""),
|
||||
)
|
||||
|
||||
@@ -1,18 +1,27 @@
|
||||
from txtai.embeddings import Embeddings
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
||||
|
||||
|
||||
def create_and_index(
|
||||
data: list[str], model: str = "sentence-transformers/all-MiniLM-L6-v2"
|
||||
) -> Embeddings:
|
||||
"""Create and index embeddings from text."""
|
||||
embeddings = Embeddings({
|
||||
"path": model,
|
||||
"content": True,
|
||||
"hybrid": True,
|
||||
"scoring": "bm25",
|
||||
})
|
||||
def create_and_index(data: list[str], model: str = DEFAULT_MODEL) -> Any:
|
||||
"""Create and index embeddings from text.
|
||||
|
||||
Requires txtai (uv sync --extra kg). Returns a txtai Embeddings instance.
|
||||
"""
|
||||
try:
|
||||
from txtai.embeddings import Embeddings
|
||||
except ImportError as exc:
|
||||
raise ImportError("create_and_index needs txtai: uv sync --extra kg") from exc
|
||||
|
||||
embeddings = Embeddings(
|
||||
{
|
||||
"path": model,
|
||||
"content": True,
|
||||
"hybrid": True,
|
||||
"scoring": "bm25",
|
||||
}
|
||||
)
|
||||
embeddings.index(data)
|
||||
return embeddings
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user