Tablemind

from rag_ingestion import RAGIngestor

ingestor = RAGIngestor()

def on_file_change(event_type, event_data):
    """Callback for file change events"""
    if event_type == "added":
        print(f"➕ New: {event_data['file_path']}")
        print(f"   Chunks: {event_data['chunks_indexed']}")
    elif event_type == "modified":
        print(f"🔄 Modified: {event_data['file_path']}")
    elif event_type == "deleted":
        print(f"🗑️  Deleted: {event_data['file_path']}")

watcher = ingestor.watch(
    "./documents",
    callback=on_file_change
)

# Or use context manager for automatic cleanup
with ingestor.watch("./documents"):
    import time
    time.sleep(300)  # Watch for 5 minutes

# Ingest only new documents (default)
python -m rag_ingestion ./pdfs --pattern "*.pdf"

# Force re-index all documents
python -m rag_ingestion ./pdfs --pattern "*.pdf" --force

# Watch directory for changes (automatic reindexing)
python -m rag_ingestion ./pdfs --watch

# Watch with custom intervals
python -m rag_ingestion ./pdfs --watch --check-interval 10 --debounce 2

# Watch with verbose output
python -m rag_ingestion ./pdfs --watch --verbose

# Use custom collection
python -m rag_ingestion ./pdfs --collection my_docs --watch

from docling_parser import parse_document

# Format is auto-detected from file extension
pdf_doc = parse_document("paper.pdf")
md_doc = parse_document("README.md")
html_doc = parse_document("report.html")
docx_doc = parse_document("document.docx")
txt_doc = parse_document("notes.txt")

# All return ParsedDocument with same structure
for doc in [pdf_doc, md_doc, html_doc, docx_doc, txt_doc]:
    print(f"{doc.file_name}: {doc.metadata.get('format')}")

from docling_parser import (
    parse_pdf, parse_markdown, parse_html,
    parse_docx, parse_text_file
)

# Use format-specific functions for clarity
pdf_doc = parse_pdf("paper.pdf")
md_doc = parse_markdown("README.md")
html_doc = parse_html("report.html")
docx_doc = parse_docx("document.docx")
txt_doc = parse_text_file("notes.txt")

from docling_parser import parse_documents

# Parse all supported formats in a directory
docs = parse_documents("./docs", recursive=True)

for doc in docs:
    print(f"{doc.file_name} ({doc.metadata.get('format')}): {len(doc.tables)} tables")

🔍 Question: What are the main findings from the ablation study?

📚 Step 1: Retrieving top 30 chunks...
   Retrieved 30 chunks

🧠 Step 1.5: Agentic analysis...
   📊 Referenced but missing: Table 3
   ✅ Added 1 agenticly-fetched tables/figures
   ⬆️  Boosted 1 existing tables/figures

🎯 Step 2: Reranking to top 10 most relevant...
   🔄 Re-inserted agentic-boosted table 3 (excluded by reranker)

📝 Step 3: Generating answer...

from rag_library import query_rag

# ✅ GOOD: Explicitly enable prioritize_tables
result = query_rag(
    question="What are the main findings from the ablation study?",
    prioritize_tables=True,  # Critical for table queries!
    retrieve_n=30,
    rerank_top_k=10,
    agentic=True,
    verbose=True
)

# ❌ BAD: Without prioritize_tables, tables may be missed
result = query_rag(
    question="What are the main findings from the ablation study?",
    # Missing prioritize_tables parameter!
    retrieve_n=30,
    rerank_top_k=10
)

# Query with table prioritization
tablemind "What are the main findings from the ablation study?" --prioritize-tables -v

# Table-only query (excludes non-table chunks)
tablemind "Compare the performance metrics" --tables-only --prioritize-tables

# High retrieval for comprehensive table search
tablemind "Show me Table 3" --prioritize-tables --retrieve-n 50 --top-k 10

from rag_library import RAGSystem, RAGConfig

config = RAGConfig()
rag = RAGSystem(config)

# Only search table chunks
result = rag.query(
    question="Compare all the performance metrics across tables",
    include_tables_only=True,
    prioritize_tables=True,
    retrieve_n=50,
    rerank_top_k=10
)

from rag_library import query_rag

# Query that triggers full-document review (agentic detection)
result = query_rag(
    question="How can I improve this paper's flow and clarity?",
    verbose=True
)

# Agent reasoning is included in response metadata
print(f"Query Mode: {result['query_mode']}")
print(f"Agent Reasoning: {result['agent_reasoning']}")

# Output:
# 🤖 Agent evaluating query type...
#    ✓ Decision: full_review
#    📝 Reasoning: Query asks for document-level assessment of flow and clarity
#
# 📄 Full-Document Review Mode
#    Question: How can I improve this paper's flow and clarity?
#
# 📚 Step 1: Retrieving all document chunks...
#    Retrieved 247 chunks from document
#
# 📋 Step 2: Organizing content by sections...
#    Found 8 sections
#
# 🔍 Step 3: Analyzing each section...
#
# 🤖 Step 4: Generating comprehensive review...
#
# Query Mode: full_review
# Agent Reasoning: Query asks for document-level assessment of flow and clarity

from rag_library import RAGSystem, RAGConfig

rag = RAGSystem()

# Force full-document review mode
result = rag.query(
    question="What are the key findings?",
    query_mode="full_review",  # Override auto-detection
    verbose=True
)

# Force specific mode (standard RAG)
result = rag.query(
    question="Review the structure",
    query_mode="specific",  # Use standard RAG instead
    verbose=True
)

# Auto-detect query mode (default)
python -m rag_library "How can I improve the writing?" -v

# Force specific mode
python -m rag_library "What are the findings?" --query-mode specific -v

# Force full-document review
python -m rag_library "Summarize the results" --query-mode full_review -v

from rag_library import query_rag

# Default semantic search
result = query_rag(
    "What are the main findings?",
    search_mode="semantic"  # or omit for default
)

# Also works with RAGSystem
from rag_library import RAGSystem, RAGConfig
config = RAGConfig(search_mode="semantic")
rag = RAGSystem(config)
result = rag.query("Summarize the methodology")

from rag_library import query_rag

# Exact keyword matching
result = query_rag(
    "Model XGBoost-7B parameters",
    search_mode="keyword",
    verbose=True
)

# Best for specific terms, part numbers, names
result = query_rag(
    "tensor parallelism",
    search_mode="keyword"
)

from rag_library import RAGSystem, RAGConfig

# Hybrid: 50% semantic, 50% keyword (default)
config = RAGConfig(
    search_mode="hybrid",
    hybrid_alpha=0.5
)
rag = RAGSystem(config)
result = rag.query("What is the accuracy of Model X?")

# Hybrid: 70% semantic, 30% keyword
config = RAGConfig(
    search_mode="hybrid",
    hybrid_alpha=0.7
)
rag = RAGSystem(config)
result = rag.query("Hyperparameter settings for bert-base")

# hybrid_alpha values:
# 0.0 = pure keyword (BM25)
# 0.5 = balanced (recommended default)
# 1.0 = pure semantic (vector)

from rag_ingestion import reindex_documents, RAGIngestor, VectorDBConfig

# Convenience function - reindex all documents in a directory
result = reindex_documents("./docs")
print(f"Reindexed {result['indexed_count']} documents")

# Or use RAGIngestor directly for more control
ingestor = RAGIngestor(db_config=VectorDBConfig(collection_name="my_docs"))
result = ingestor.reindex_collection("./docs")

# With progress callback
def on_progress(message, current, total):
    print(f"[{current}/{total}] {message}")

result = reindex_documents("./docs", progress_callback=on_progress)

# Semantic search (default)
tablemind "What are the main findings?"

# Keyword search (BM25)
tablemind "Model XGBoost-7B" --search-mode keyword

# Hybrid search with default balance (0.5)
tablemind "What is the accuracy?" --search-mode hybrid

# Hybrid search with custom balance
tablemind "Model parameters" --search-mode hybrid --hybrid-alpha 0.7

# Keyword search with table prioritization
tablemind "performance metrics" --search-mode keyword --prioritize-tables -v

# Search mode: semantic, keyword, or hybrid
SEARCH_MODE=hybrid

# Hybrid balance (0.0=keyword, 1.0=semantic, 0.5=balanced)
HYBRID_ALPHA=0.5

# Collection and database
QDRANT_COLLECTION_NAME=my_documents
QDRANT_PATH=./qdrant_db

# Qdrant Configuration
QDRANT_COLLECTION_NAME=my_documents
QDRANT_PATH=./qdrant_db

# Model Configuration
EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2

# LLM Configuration
LLM_PROVIDER=anthropic
LLM_MODEL=claude-sonnet-4-20250514
LLM_TEMPERATURE=0.7
LLM_MAX_TOKENS=2000

# API Keys
# Note: These keys are loaded from .env and passed to the LLM service
ANTHROPIC_API_KEY=your-api-key
OPENAI_API_KEY=your-api-key

# Chunking Configuration
MAX_TOKENS=4000
MERGE_PEERS=true
INCLUDE_STRUCTURE_CONTEXT=true

from rag_library import RAGConfig, RAGSystem

# .env values are used by default
config = RAGConfig()

# Or override specific values
config = RAGConfig(
    collection_name="custom_collection",
    temperature=0.3
    # Other values come from .env
)

# Run the CLI directly (uses agentic auto-detection by default)
python rag_library.py "What are the main findings?"

# Or as a module
python -m rag_library "What are the main findings?"

# Query with custom retrieval options
python rag_library.py "Compare table 3" --retrieve-n 50 --top-k 10

# Enable verbose output (shows agentic classification)
python rag_library.py "What datasets were used?" -v

# Force specific query mode
python rag_library.py "What are the findings?" --query-mode specific

# Force full-document review mode
python rag_library.py "Summarize the results" --query-mode full_review

# Table-only query with agentic fetching
python rag_library.py "What are the performance metrics?" --tables-only --prioritize-tables

# Query with all options
python rag_library.py "Analyze the results" \
  --retrieve-n 100 \
  --top-k 20 \
  --agentic \
  --prioritize-tables \
  --verbose

# Run the example script
python ragexample.py

# Run specific examples
python ragexample.py --parse          # Parse PDF example
python ragexample.py --ingest         # Ingest documents example
python ragexample.py --query           # Query documents example
python ragexample.py "Your question"  # Custom query

# For custom query with options, edit ragexample.py or use rag_library.py directly:
python rag_library.py "Your question" --prioritize-tables -v

# Start the web server
python web_app.py

# The interface will be available at:
# http://localhost:5005

# === Chat Endpoints ===

# Create a new conversation
POST /api/conversations
Response: {"conversation_id": "uuid"}

# Send a chat message
POST /api/chat
Body: {
  "conversation_id": "uuid",
  "message": "Your question",
  "search_mode": "semantic",       # semantic|keyword|hybrid
  "prioritize_tables": true,
  "agentic_mode": true,
  "include_table_content": true,  # NEW: show tables in response
  "stream": true
}

# Get conversation history
GET /api/conversations/{conversation_id}

# === Document Management Endpoints ===

# Upload a document
POST /api/documents/upload
Content-Type: multipart/form-data
Response: {"task_id": "uuid", "filename": "doc.pdf", "status": "pending"}

# Get ingestion status
GET /api/documents/status/{task_id}
Response: {"status": "completed", "chunks_indexed": 45, ...}

# List all documents
GET /api/documents
Response: {"documents": [{"name": "doc.pdf", "path": "...", ...}]}

# Delete a document
DELETE /api/documents/{path}

# Reindex all documents (clears and rebuilds vector database)
POST /api/documents/reindex
Response: {
  "status": "started",
  "message": "Reindexing started in background",
  "note": "Use /api/documents/status/ to track progress"
}

# === File Watcher Endpoints ===

# Get watcher status (includes last event)
GET /api/watcher
Response: {
  "enabled": true,
  "folder": "./docs",
  "pattern": "*",  # Watches all files, filters by extension
  "last_event": {  # Most recent file change event
    "type": "added",
    "data": {"file_path": "doc.pdf", "chunks_indexed": 45},
    "timestamp": "2026-01-03T12:34:56"
  }
}

# Start file watcher
POST /api/watcher/start
Body: {"check_interval": 10.0, "debounce_interval": 2.0}
Response: {"status": "started", "enabled": true, "folder": "./docs"}

# Stop file watcher
POST /api/watcher/stop

# Note: File watcher supports PDF, MD, HTML, DOCX, TXT
# Unsupported extensions are automatically skipped

# === Configuration ===

# Get current configuration
GET /api/config
Response: {
  "embedding_model": "...",
  "llm_provider": "anthropic",
  "llm_model": "claude-sonnet-4-5",
  "collection_name": "my_documents",
  "available_search_modes": ["semantic", "keyword", "hybrid"]
}

from rag_ingestion import ingest_pdfs
from rag_library import query_rag

# Step 1: Ingest all PDFs from a directory
print("Ingesting documents...")
results = ingest_pdfs(
    directory="./pdfs",
    collection_name="research_papers",
    pattern="*.pdf"
)

for r in results:
    print(f"  {r['file_path']}: {r['status']} ({r['chunks_indexed']} chunks)")

# Step 2: Query the documents
print("\nQuerying documents...")
result = query_rag(
    "What are the main findings across all papers?",
    collection_name="research_papers",
    db_path="./qdrant_db",
    retrieve_n=50,
    rerank_top_k=10,
    agentic=True
)

print(f"\nAnswer:\n{result['answer']}")

# Step 3: Show sources
print(f"\nSources ({len(result['sources'])}):")
for i, s in enumerate(result['sources'], 1):
    marker = "📊" if s.get('is_table') else "📄"
    print(f"  {i}. {marker} {s.get('file_name')} - {s.get('heading')})

from pathlib import Path
from rag_ingestion import RAGIngestor, VectorDBConfig
from rag_library import query_rag

# Step 1: Set up ingestor with multi-format support
db_config = VectorDBConfig(
    collection_name="multi_format_docs",
    path="./qdrant_db"
)
ingestor = RAGIngestor(db_config=db_config)

# Step 2: Define directory with mixed formats
docs_dir = Path("./documents")

# Step 3: Ingest all supported formats
print("Ingesting multi-format documents...")

# Supported extensions: .pdf, .md, .markdown, .html, .htm, .docx, .txt
supported_extensions = [".pdf", ".md", ".html", ".docx", ".txt"]

results = []
for ext in supported_extensions:
    for file_path in docs_dir.glob(f"*{ext}"):
        try:
            result = ingestor.ingest_file(file_path)
            results.append(result)
            print(f"  ✓ {file_path.name}: {result['status']} ({result['chunks_indexed']} chunks, {result['num_tables']} tables)")
        except Exception as e:
            print(f"  ✗ {file_path.name}: {e}")

# Step 4: Query across all formats
print("\nQuerying multi-format collection...")
result = query_rag(
    "What are the main findings across all documents?",
    collection_name="multi_format_docs",
    db_path="./qdrant_db",
    retrieve_n=30,
    rerank_top_k=10,
    agentic=True,
    verbose=True
)

print(f"\nAnswer:\n{result['answer']}")

# Step 5: Show sources with format indicators
print(f"\nSources ({len(result['sources'])}):")
for i, s in enumerate(result['sources'], 1):
    # Determine icon based on content type
    if s.get('is_table'):
        icon = "📊"
    elif s.get('is_figure'):
        icon = "🖼️"
    else:
        icon = "📄"

    # Get file extension for format indicator
    file_name = s.get('file_name', "Unknown")
    ext = Path(file_name).suffix.upper() if '.' in file_name else "TXT"

    print(f"  {i}. {icon} [{ext}] {file_name} - {s.get('heading', 'N/A')})

# Put documents (PDF, MD, HTML, DOCX, TXT) in ./documents folder
# Then run:

# Ingest all supported formats
python ragexample.py --ingest

# Query with agentic AI (auto-detects optimal strategy)
python ragexample.py "What are the main findings?"

# The agent evaluates query intent and selects appropriate mode