Tablemind - Developer Guide

pip install docling sentence-transformers qdrant-client fastapi uvicorn

from rag_ingestion import RAGIngestor
from rag_library import RAGLibrary

# Initialize
ingestor = RAGIngestor(collection_name="my_docs")
rag = RAGLibrary(ingestor=ingestor)

# Ingest a document
result = ingestor.ingest_document("paper.pdf")
print(f"Ingested {result['chunks_indexed']} chunks")

# Query
response = rag.query("What's the F1 score in Table 3?",
                    search_mode="semantic",
                    table_priority=True)
print(response['answer'])

# Reindex all documents
result = ingestor.reindex_collection("./docs")
print(f"Reindexed {result['indexed_count']} documents")

from docling_parser import DoclingParser

parser = DoclingParser()
parsed = parser.parse("document.pdf")

# Access parsed data
print(f"Sections: {list(parsed.sections.keys())}")
print(f"Tables: {len(parsed.tables)}")
print(f"Figures: {len(parsed.figures)}")

# Get table as markdown
table_md = parsed.tables[0]['markdown']

from rag_ingestion import RAGIngestor

ingestor = RAGIngestor(
    collection_name="my_documents",
    embedding_model="nomic-ai/nomic-embed-text-v1.5",
    qdrant_path="./qdrant_db"
)

# Ingest single file
result = ingestor.ingest_document("paper.pdf")
# Returns: {"status": "success", "doc_id": "sha256_hash", "chunks_indexed": 42}

# Batch ingest directory
results = ingestor.ingest_directory("./docs", pattern="*.pdf")

# Delete document
ingestor.delete_document(doc_id="sha256_hash")

from rag_library import RAGLibrary

rag = RAGLibrary(ingestor=ingestor)

# Standard RAG query (chunk-based, fast)
response = rag.query(
    query="What is the accuracy of Model A?",
    search_mode="semantic",  # or "keyword", "hybrid"
    table_priority=True,
    agentic_references=True
)

# Full document review (for broad questions)
response = rag.query(
    query="Does the paper's narrative flow logically?",
    query_mode="full_review"
)

# Auto mode (LLM chooses appropriate mode)
response = rag.query(
    query="Compare all approaches in the paper",
    query_mode="auto"
)

# Access results
print(response['answer'])
for source in response['sources']:
    print(f"- {source['file_name']} ({source['chunk_type']})")

# Set environment variables
export LLM_PROVIDER=gemini
export LLM_MODEL=gemini-2.5-pro
export PDF_DIRECTORY=./docs

# Start server
python web_app.py
# Server runs on http://localhost:5005

export LLM_PROVIDER=gemini
export LLM_MODEL=gemini-2.5-pro
export PDF_DIRECTORY=./docs

python web_app.py
# Server runs on http://localhost:5005

# Qdrant Configuration
QDRANT_COLLECTION_NAME=my_documents
QDRANT_PATH=./qdrant_db

# Model Configuration
EMBEDDING_MODEL=nomic-ai/nomic-embed-text-v1.5
RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2

# LLM Configuration
LLM_PROVIDER=gemini  # anthropic, openai, ollama
LLM_MODEL=gemini-2.5-pro
LLM_TEMPERATURE=0.3
LLM_MAX_TOKENS=8192

# API Keys
# Note: These keys are loaded from .env and passed to the LLM service
ANTHROPIC_API_KEY=your-api-key
OPENAI_API_KEY=your-api-key

# Chunking Configuration
MAX_TOKENS=4000
MERGE_PEERS=true
INCLUDE_STRUCTURE_CONTEXT=true

# Retrieval
RETRIEVE_N=100
RERANK_TOP_K=6
QUERY_EXPANSION=false

# Documents
PDF_DIRECTORY=./docs

from rag_library import RAGConfig, RAGSystem

# .env values are used by default
config = RAGConfig()

# Or override specific values
config = RAGConfig(
    collection_name="custom_collection",
    temperature=0.3
    # Other values come from .env
)

from rag_ingestion import RAGIngestor

ingestor = RAGIngestor(collection_name="docs")

watcher = ingestor.create_file_watcher(
    directory="./docs",
    pattern="*.pdf",
    check_interval=10.0,
    debounce_interval=2.0,
    callback=lambda event_type, data: print(f"{event_type}: {data}"),
    autostart=True
)

# Watcher runs in background, auto-ingests new/modified files
watcher.stop()

from docling_parser import DoclingParser
from rag_library import RAGLibrary

parser = DoclingParser()
parsed = parser.parse("paper.pdf")

rag = RAGLibrary(ingestor=None)  # No ingestor needed
response = rag.query_with_full_context(
    query="Summarize the methodology",
    parsed_document=parsed,
    selected_tables=[0, 2]
)

# Run the CLI directly (uses agentic auto-detection by default)
python rag_library.py "What are the main findings?"

# Or as a module
python -m rag_library "What are the main findings?"

# Query with custom retrieval options
python rag_library.py "Compare table 3" --retrieve-n 50 --top-k 10

# Enable verbose output (shows agentic classification)
python rag_library.py "What datasets were used?" -v

# Force specific query mode
python rag_library.py "What are the findings?" --query-mode specific

# Force full-document review mode
python rag_library.py "Summarize the results" --query-mode full_review

# Table-only query with agentic fetching
python rag_library.py "What are the performance metrics?" --tables-only --prioritize-tables

# Query with all options
python rag_library.py "Analyze the results" \
  --retrieve-n 100 \
  --top-k 20 \
  --agentic \
  --prioritize-tables \
  --verbose

# Run the example script
python ragexample.py

# Run specific examples
python ragexample.py --parse          # Parse PDF example
python ragexample.py --ingest         # Ingest documents example
python ragexample.py --query           # Query documents example
python ragexample.py "Your question"  # Custom query

# For custom query with options, edit ragexample.py or use rag_library.py directly:
python rag_library.py "Your question" --prioritize-tables -v

from rag_ingestion import ingest_pdfs
from rag_library import query_rag

# Step 1: Ingest all PDFs from a directory
print("Ingesting documents...")
results = ingest_pdfs(
    directory="./pdfs",
    collection_name="research_papers",
    pattern="*.pdf"
)

for r in results:
    print(f"  {r['file_path']}: {r['status']} ({r['chunks_indexed']} chunks)")

# Step 2: Query the documents
print("\nQuerying documents...")
result = query_rag(
    "What are the main findings across all papers?",
    collection_name="research_papers",
    db_path="./qdrant_db",
    retrieve_n=50,
    rerank_top_k=10,
    agentic=True
)

print(f"\nAnswer:\n{result['answer']}")

# Step 3: Show sources
print(f"\nSources ({len(result['sources'])}):")
for i, s in enumerate(result['sources'], 1):
    marker = "📊" if s.get('is_table') else "📄"
    print(f"  {i}. {marker} {s.get('file_name')} - {s.get('heading')}")

from pathlib import Path
from rag_ingestion import RAGIngestor, VectorDBConfig
from rag_library import query_rag

# Step 1: Set up ingestor with multi-format support
db_config = VectorDBConfig(
    collection_name="multi_format_docs",
    path="./qdrant_db"
)
ingestor = RAGIngestor(db_config=db_config)

# Step 2: Define directory with mixed formats
docs_dir = Path("./documents")

# Step 3: Ingest all supported formats
print("Ingesting multi-format documents...")

# Supported extensions: .pdf, .md, .markdown, .html, .htm, .docx, .txt
supported_extensions = [".pdf", ".md", ".html", ".docx", ".txt"]

results = []
for ext in supported_extensions:
    for file_path in docs_dir.glob(f"*{ext}"):
        try:
            result = ingestor.ingest_file(file_path)
            results.append(result)
            print(f"  ✓ {file_path.name}: {result['status']} ({result['chunks_indexed']} chunks, {result['num_tables']} tables)")
        except Exception as e:
            print(f"  ✗ {file_path.name}: {e}")

# Step 4: Query across all formats
print("\nQuerying multi-format collection...")
result = query_rag(
    "What are the main findings across all documents?",
    collection_name="multi_format_docs",
    db_path="./qdrant_db",
    retrieve_n=30,
    rerank_top_k=10,
    agentic=True,
    verbose=True
)

print(f"\nAnswer:\n{result['answer']}")

# Step 5: Show sources with format indicators
print(f"\nSources ({len(result['sources'])}):")
for i, s in enumerate(result['sources'], 1):
    # Determine icon based on content type
    if s.get('is_table'):
        icon = "📊"
    elif s.get('is_figure'):
        icon = "🖼️"
    else:
        icon = "📄"

    # Get file extension for format indicator
    file_name = s.get('file_name', "Unknown")
    ext = Path(file_name).suffix.upper() if '.' in file_name else "TXT"

    print(f"  {i}. {icon} [{ext}] {file_name} - {s.get('heading', 'N/A')}")

# Put documents (PDF, MD, HTML, DOCX, TXT) in ./documents folder
# Then run:

# Ingest all supported formats
python ragexample.py --ingest

# Query with agentic AI (auto-detects optimal strategy)
python ragexample.py "What are the main findings?"

# The agent evaluates query intent and selects appropriate mode