Vector Databases and Hybrid Search: pgvector, Qdrant and RRF

[ User Query ] 
      |
      +-------------------------------------------+
      |                                           |
      v                                           v
[ Dense Encoder ]                           [ Sparse Tokenizer ]
(e.g., text-embedding-3-large)             (BM25 / SPLADE Analyzer)
      |                                           |
      v (Vector: 1536d float32)                   v (Sparse: term -> weight)
+-----------------------------+             +-----------------------------+
| Vector DB: Dense Index      |             | Search Engine: Inverted Idx |
| HNSW / Cosine Similarity    |             | Okapi BM25 Scoring          |
+-----------------------------+             +-----------------------------+
      | (Ranked List 1: Top-K)                    | (Ranked List 2: Top-K)
      | [DocA (1), DocB (2), ...]                 | [DocC (1), DocA (2), ...]
      +-------------------------------------------+
                                  |
                                  v
                +------------------------------------+
                | Reciprocal Rank Fusion (RRF)       |
                | Score(d) = sum( 1 / (k + rank_i) ) |
                +------------------------------------+
                                  |
                                  v
                [ Unified Ranked Candidates (Top-N) ]
                                  |
                                  v
                [ Payload / Metadata Filter Stage    ]

RRF_Score(d ∈ D) = Σ [ 1 / (k + r_m(d)) ]

-- 1. Подключение расширений
CREATE EXTENSION IF NOT EXISTS vector;
CREATE EXTENSION IF NOT EXISTS pg_trgm;

-- 2. Создание таблицы документов с плотным вектором и полнотекстовым полем
CREATE TABLE documents (
    id BIGSERIAL PRIMARY KEY,
    title TEXT NOT NULL,
    content TEXT NOT NULL,
    metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
    -- Вектор OpenAI text-embedding-3-small (1536 измерений)
    embedding vector(1536) NOT NULL,
    -- Полнотекстовый поисковый вектор (генерируемая колонка)
    tsv_content tsvector GENERATED ALWAYS AS (
        to_tsvector('russian', coalesce(title, '') || ' ' || coalesce(content, ''))
    ) STORED
);

-- 3. Создание HNSW индекса для косинусного расстояния (vector_cosine_ops)
-- m: максимальное количество соединений на узел (16 - оптимум)
-- ef_construction: размер динамического списка при построении графа (64-128)
CREATE INDEX idx_documents_embedding_hnsw 
ON documents USING hnsw (embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 64);

-- 4. Создание GIN индекса для полнотекстового лексического поиска
CREATE INDEX idx_documents_tsv ON documents USING gin(tsv_content);

-- 5. Индекс по метаданным для фильтрации (Single-Tenant / Multi-Tenant)
CREATE INDEX idx_documents_metadata ON documents USING gin(metadata);

WITH 
-- Параметры поиска: $1 - вектор запроса, $2 - текст запроса, $3 - k константа RRF (60), $4 - tenant_id
dense_search AS (
    SELECT 
        id,
        ROW_NUMBER() OVER (ORDER BY embedding <=> $1::vector) AS dense_rank
    FROM documents
    WHERE metadata->>'tenant_id' = $4
    ORDER BY embedding <=> $1::vector
    LIMIT 50
),
sparse_search AS (
    SELECT 
        id,
        ROW_NUMBER() OVER (ORDER BY ts_rank_cd(tsv_content, plainto_tsquery('russian', $2)) DESC) AS sparse_rank
    FROM documents
    WHERE 
        tsv_content @@ plainto_tsquery('russian', $2)
        AND metadata->>'tenant_id' = $4
    ORDER BY ts_rank_cd(tsv_content, plainto_tsquery('russian', $2)) DESC
    LIMIT 50
)
SELECT 
    d.id,
    d.title,
    d.content,
    d.metadata,
    COALESCE(1.0 / ($3 + ds.dense_rank), 0.0) + 
    COALESCE(1.0 / ($3 + ss.sparse_rank), 0.0) AS rrf_score
FROM dense_search ds
FULL OUTER JOIN sparse_search ss ON ds.id = ss.id
JOIN documents d ON d.id = COALESCE(ds.id, ss.id)
ORDER BY rrf_score DESC
LIMIT 10;

import { QdrantClient } from "@qdrant/js-client-rest";
import { z } from "zod";

// Zod-схемы валидации контрактов
export const SearchResultSchema = z.object({
  id: z.string(),
  score: z.number(),
  payload: z.object({
    documentId: z.string(),
    content: z.string(),
    tenantId: z.string(),
    category: z.string(),
  }),
});

export type SearchResult = z.infer<typeof SearchResultSchema>;

export interface HybridSearchOptions {
  limit: number;
  rrfKConstant?: number;
  tenantId: string;
}

export class HybridSearchService {
  private qdrant: QdrantClient;
  private readonly collectionName = "enterprise_knowledge_base";

  constructor(qdrantUrl: string, apiKey?: string) {
    this.qdrant = new QdrantClient({
      url: qdrantUrl,
      apiKey: apiKey,
    });
  }

  /**
   * Слияние двух отсортированных списков идентификаторов методом RRF.
   */
  private applyRRF(
    denseResults: Array<{ id: string; payload: Record<string, unknown> }>,
    sparseResults: Array<{ id: string; payload: Record<string, unknown> }>,
    kConstant: number = 60,
    limit: number = 10
  ): SearchResult[] {
    const scores = new Map<string, { score: number; payload: Record<string, unknown> }>();

    // 1. Учет рангов плотного поиска
    denseResults.forEach((item, index) => {
      const rank = index + 1;
      const rrfContribution = 1 / (kConstant + rank);
      const existing = scores.get(item.id);
      if (existing) {
        existing.score += rrfContribution;
      } else {
        scores.set(item.id, { score: rrfContribution, payload: item.payload });
      }
    });

    // 2. Учет рангов разреженного поиска
    sparseResults.forEach((item, index) => {
      const rank = index + 1;
      const rrfContribution = 1 / (kConstant + rank);
      const existing = scores.get(item.id);
      if (existing) {
        existing.score += rrfContribution;
      } else {
        scores.set(item.id, { score: rrfContribution, payload: item.payload });
      }
    });

    // 3. Сортировка по итоговому RRF Score и приведение к типам
    const sorted = Array.from(scores.entries())
      .sort((a, b) => b[1].score - a[1].score)
      .slice(0, limit)
      .map(([id, data]) => ({
        id,
        score: Number(data.score.toFixed(6)),
        payload: data.payload,
      }));

    return z.array(SearchResultSchema).parse(sorted);
  }

  /**
   * Выполнение гибридного запроса к Qdrant с пре-фильтрацией по метаданным
   */
  public async search(
    denseVector: number[],
    sparseIndices: number[],
    sparseValues: number[],
    options: HybridSearchOptions
  ): Promise<SearchResult[]> {
    const filter = {
      must: [
        {
          key: "tenantId",
          match: { value: options.tenantId },
        },
      ],
    };

    // Параллельный запрос к Dense и Sparse векторам в Qdrant
    const [denseResponse, sparseResponse] = await Promise.all([
      this.qdrant.search(this.collectionName, {
        vector: {
          name: "dense-text",
          vector: denseVector,
        },
        filter,
        limit: 50,
        with_payload: true,
      }),
      this.qdrant.search(this.collectionName, {
        vector: {
          name: "sparse-bm25",
          vector: {
            indices: sparseIndices,
            values: sparseValues,
          },
        },
        filter,
        limit: 50,
        with_payload: true,
      }),
    ]);

    const formattedDense = denseResponse.map((hit) => ({
      id: String(hit.id),
      payload: (hit.payload ?? {}) as Record<string, unknown>,
    }));

    const formattedSparse = sparseResponse.map((hit) => ({
      id: String(hit.id),
      payload: (hit.payload ?? {}) as Record<string, unknown>,
    }));

    return this.applyRRF(
      formattedDense,
      formattedSparse,
      options.rrfKConstant ?? 60,
      options.limit
    );
  }
}