QMD：本地文档语义搜索完全指南

# QMD 语义搜索流程
def semantic_search(query, top_k=5):
    # 1. 使用 Ollama 将查询文本转换为向量
    query_embedding = ollama.embeddings(
        model="nomic-embed-text",
        prompt=query
    )
    
    # 2. 在向量数据库中搜索最相似的文档片段
    results = vector_db.search(
        query_embedding,
        n=top_k
    )
    
    # 3. 返回最相关的文档
    return results

# BM25 算法核心思想
def bm25_score(doc, query, k1=1.5, b=0.75):
    """
    - k1: 词频饱和参数（控制词频影响力）
    - b: 文档长度归一化参数
    """
    scores = []
    for term in query:
        tf = doc.term_freq(term)           # 词频
        df = corpus.doc_freq(term)        # 文档频率
        idf = log((N - df + 0.5) / (df + 0.5))  # 逆文档频率
        
        # BM25 公式
        score = idf * (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * doc.len / avg_doc_len))
        scores.append(score)
    
    return sum(scores)

def hybrid_search(query, alpha=0.5):
    """
    alpha: 语义搜索权重 (1-alpha 是 BM25 权重)
    """
    # 获取语义搜索结果
    semantic_results = semantic_search(query)
    
    # 获取 BM25 结果
    bm25_results = bm25_search(query)
    
    # RRFR 融合
    fused_results = []
    for doc in corpus:
        rrf_score = 0
        for rank, result in enumerate(semantic_results[:k]):
            if result.doc == doc:
                rrf_score += 1 / (k + rank + 1)
        
        for rank, result in enumerate(bm25_results[:k]):
            if result.doc == doc:
                rrf_score += alpha / (k + rank + 1)
        
        fused_results.append((doc, rrf_score))
    
    return sorted(fused_results, key=lambda x: x[1], reverse=True)

# macOS/Linux 安装 Ollama
curl -fsSL https://ollama.com/install.sh | sh

# Windows: 下载安装包 https://ollama.com/download

# 启动 Ollama 服务
ollama serve

# 下载嵌入模型（推荐）
ollama pull nomic-embed-text

# 验证
ollama list

# 创建索引（会扫描所有支持的文件）
qmd index ~/documents

# 指定目录
qmd index /path/to/papers

# 递归扫描子目录
qmd index ~/documents --recursive

# 查看索引统计
qmd stats

# 语义搜索（默认）
qmd search "What is transformer architecture?"

# 关键词搜索
qmd search "transformer attention" --method bm25

# 混合搜索
qmd search "neural network optimization" --method hybrid

# 显示更多结果
qmd search "machine learning" --top 10

# 简短输出
qmd search "python async"

# 输出示例
---
Score: 0.89
File: /docs/python-guide.md
Line: 42
Snippet: "Python's async/await syntax provides..."
---

Score: 0.85
File: /docs/advanced-python.md
Line: 128
Snippet: "Asynchronous programming in Python uses..."

from qmd import QMD

# 初始化
qmd = QMD(
    data_dir="~/.local/share/qmd",  # 数据目录
    embed_model="nomic-embed-text"    # 嵌入模型
)

# 添加文档
qmd.add("~/documents/python-guide.md")
qmd.add("~/documents/notes/")

# 搜索
results = qmd.search("async programming in Python")
for result in results:
    print(f"{result.file}:{result.line} - {result.snippet}")

from qmd import SearchMethod

# 配置搜索参数
results = qmd.search(
    query="machine learning optimization",
    method=SearchMethod.HYBRID,  # 混合搜索
    alpha=0.7,                    # 语义权重
    top_k=10,                     # 返回结果数
    min_score=0.5                 # 最低分数阈值
)

# 只搜索特定格式
results = qmd.search(
    "Python best practices",
    extensions=[".md", ".txt"]    # 只搜索 Markdown 和文本
)

# 排除特定目录
results = qmd.search(
    "API design",
    exclude_dirs=[".git", "node_modules", "__pycache__"]
)

{
  "mcpServers": {
    "qmd": {
      "command": "qmd",
      "args": ["mcp", "--port", "8080"],
      "env": {
        "QMD_DATA_DIR": "~/.local/share/qmd"
      }
    }
  }
}

[general]
# 数据目录
data_dir = "~/.local/share/qmd"

# 索引目录
index_dir = "~/.local/share/qmd/index"

# 日志级别
log_level = "INFO"

[search]
# 默认搜索方法 (semantic, bm25, hybrid)
default_method = "hybrid"

# 默认返回结果数
default_top_k = 5

# 语义搜索权重（用于混合搜索）
semantic_weight = 0.7

[embed]
# Ollama 服务地址
ollama_base_url = "http://localhost:11434"

# 嵌入模型
model = "nomic-embed-text"

# 批处理大小
batch_size = 32

[index]
# 并行线程数
num_workers = 4

# 文件大小限制（MB）
max_file_size = 50

# 排除的目录
exclude_dirs = [".git", "node_modules", "__pycache__", ".venv"]