Advanced Chunking, Cross-Encoder Reranking and RAGAS Metrics

[ Raw Document ]
       |
       v
[ Semantic Chunking Engine ] 
(Sentence Split -> Sliding Cosine Distance -> Breakpoint Detection by Percentile)
       |
       v
[ High-Quality Coherent Chunks ] ---> [ Vector Index (Bi-Encoder) ]
                                                    |
[ User Query ] -------------------------------------+
                                                    |
                                                    v (Stage 1: High Recall)
                                         [ Top-50 Candidates ]
                                                    |
                                                    v
                                         [ Cross-Encoder Reranker ]
                                         (Joint Self-Attention: [CLS] q [SEP] d)
                                                    |
                                                    v (Stage 2: High Precision)
                                         [ Top-5 Re-ranked Chunks ]
                                                    |
                                                    v
                                         [ LLM Generation Context ]
                                                    |
                                                    v
                                         [ Continuous RAGAS Eval ]
                                         (Faithfulness / Context Precision)

import { z } from "zod";

export interface SemanticChunkerConfig {
  minChunkTokens: number;
  maxChunkTokens: number;
  breakpointPercentileThreshold: number; // e.g. 90
}

export interface ChunkOutput {
  index: number;
  content: string;
  tokenEstimate: number;
}

export class SemanticChunker {
  private config: SemanticChunkerConfig;

  constructor(config: Partial<SemanticChunkerConfig> = {}) {
    this.config = {
      minChunkTokens: config.minChunkTokens ?? 80,
      maxChunkTokens: config.maxChunkTokens ?? 600,
      breakpointPercentileThreshold: config.breakpointPercentileThreshold ?? 90,
    };
  }

  /**
   * Вычисление косинусного сходства между двумя нормализованными векторами
   */
  private cosineDistance(vecA: number[], vecB: number[]): number {
    let dotProduct = 0;
    let normA = 0;
    let normB = 0;

    for (let i = 0; i < vecA.length; i++) {
      dotProduct += vecA[i] * vecB[i];
      normA += vecA[i] * vecA[i];
      normB += vecB[i] * vecB[i];
    }

    if (normA === 0 || normB === 0) return 1.0;
    const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
    return 1.0 - similarity; // Дистанция: 0 (идентичны) .. 2 (противоположны)
  }

  /**
   * Расчет значения перцентиля в числовом массиве
   */
  private calculatePercentile(values: number[], percentile: number): number {
    if (values.length === 0) return 0;
    const sorted = [...values].sort((a, b) => a - b);
    const index = (percentile / 100) * (sorted.length - 1);
    const lower = Math.floor(index);
    const upper = Math.ceil(index);
    const weight = index - lower;
    return sorted[lower] * (1 - weight) + sorted[upper] * weight;
  }

  /**
   * Разделение сырого текста на семантические чанки
   * @param text Исходный документ
   * @param embedFn Функция пакетного построения эмбеддингов
   */
  public async chunkText(
    text: string,
    embedFn: (sentences: string[]) => Promise<number[][]>
  ): Promise<ChunkOutput[]> {
    // 1. Разбивка на предложения с сохранением пунктуации
    const rawSentences = text
      .split(/(?<=[.?!])\s+(?=[А-ЯA-Z0-9])/g)
      .map((s) => s.trim())
      .filter((s) => s.length > 0);

    if (rawSentences.length <= 1) {
      return [{ index: 0, content: text, tokenEstimate: Math.ceil(text.length / 4) }];
    }

    // 2. Получение векторов предложений
    const embeddings = await embedFn(rawSentences);

    // 3. Расчет скользящих расстояний между соседними предложениями
    const distances: number[] = [];
    for (let i = 0; i < embeddings.length - 1; i++) {
      const dist = this.cosineDistance(embeddings[i], embeddings[i + 1]);
      distances.push(dist);
    }

    // 4. Определение динамического порога точки среза
    const threshold = this.calculatePercentile(
      distances,
      this.config.breakpointPercentileThreshold
    );

    // 5. Формирование чанков с учетом ограничений по размеру
    const chunks: ChunkOutput[] = [];
    let currentSentences: string[] = [rawSentences[0]];
    let currentTokens = Math.ceil(rawSentences[0].length / 4);

    for (let i = 0; i < distances.length; i++) {
      const nextSentence = rawSentences[i + 1];
      const nextSentenceTokens = Math.ceil(nextSentence.length / 4);
      const isDistanceExceeded = distances[i] >= threshold;
      const willExceedMaxTokens =
        currentTokens + nextSentenceTokens > this.config.maxChunkTokens;
      const satisfiesMinTokens = currentTokens >= this.config.minChunkTokens;

      if ((isDistanceExceeded && satisfiesMinTokens) || willExceedMaxTokens) {
        chunks.push({
          index: chunks.length,
          content: currentSentences.join(" "),
          tokenEstimate: currentTokens,
        });
        currentSentences = [nextSentence];
        currentTokens = nextSentenceTokens;
      } else {
        currentSentences.push(nextSentence);
        currentTokens += nextSentenceTokens;
      }
    }

    if (currentSentences.length > 0) {
      chunks.push({
        index: chunks.length,
        content: currentSentences.join(" "),
        tokenEstimate: currentTokens,
      });
    }

    return chunks;
  }
}

import { z } from "zod";

const RerankResponseSchema = z.object({
  id: z.string(),
  results: z.array(
    z.object({
      index: z.number(),
      relevance_score: z.number(),
    })
  ),
});

export interface RerankedCandidate {
  originalIndex: number;
  content: string;
  relevanceScore: number;
  metadata: Record<string, unknown>;
}

export class CohereReranker {
  private apiKey: string;
  private model: string;

  constructor(apiKey: string, model: string = "rerank-v3.5") {
    this.apiKey = apiKey;
    this.model = model;
  }

  /**
   * Точный реранкинг кандидатов с отсечением по скору
   */
  public async rerank(
    query: string,
    candidates: Array<{ content: string; metadata: Record<string, unknown> }>,
    topN: number = 5,
    minRelevanceScore: number = 0.4
  ): Promise<RerankedCandidate[]> {
    if (candidates.length === 0) return [];

    const response = await fetch("https://api.cohere.com/v1/rerank", {
      method: "POST",
      headers: {
        Authorization: `Bearer ${this.apiKey}`,
        "Content-Type": "application/json",
      },
      body: JSON.stringify({
        model: this.model,
        query: query,
        documents: candidates.map((c) => c.content),
        top_n: topN,
        return_documents: false,
      }),
    });

    if (!response.ok) {
      const errorPayload = await response.text();
      throw new Error(`Cohere Rerank API error: ${response.status} - ${errorPayload}`);
    }

    const data = RerankResponseSchema.parse(await response.json());

    return data.results
      .filter((res) => res.relevance_score >= minRelevanceScore)
      .map((res) => ({
        originalIndex: res.index,
        content: candidates[res.index].content,
        relevanceScore: Number(res.relevance_score.toFixed(4)),
        metadata: candidates[res.index].metadata,
      }));
  }
}

import OpenAI from "openai";
import { z } from "zod";

const FaithfulnessVerificationSchema = z.object({
  statements: z.array(
    z.object({
      statement: z.string(),
      isSupportedByContext: z.boolean(),
      reason: z.string(),
    })
  ),
});

export class RagasEvaluator {
  private openai: OpenAI;
  private evalModel: string;

  constructor(apiKey: string, evalModel: string = "gpt-4o-mini") {
    this.openai = new OpenAI({ apiKey });
    this.evalModel = evalModel;
  }

  /**
   * Оценка метрики Faithfulness: доля подтвержденных фактов в ответе
   */
  public async computeFaithfulness(
    contextChunks: string[],
    generatedAnswer: string
  ): Promise<{ score: number; failedStatements: string[] }> {
    const prompt = `
Проанализируй сгенерированный ответ и сопоставь каждое содержащееся в нем утверждение с предоставленным контекстом.

КОНТЕКСТ:
${contextChunks.join("\n---\n")}

ОТВЕТ:
${generatedAnswer}

Выдели атомарные утверждения из ответа и для каждого определи, подтверждается ли оно контекстом строго и без домыслов.
`;

    const completion = await this.openai.chat.completions.create({
      model: this.evalModel,
      temperature: 0.0,
      messages: [{ role: "user", content: prompt }],
      response_format: { type: "json_object" },
    });

    const parsed = FaithfulnessVerificationSchema.parse(
      JSON.parse(completion.choices[0]?.message.content ?? "{}")
    );

    if (parsed.statements.length === 0) return { score: 1.0, failedStatements: [] };

    const supportedCount = parsed.statements.filter((s) => s.isSupportedByContext).length;
    const failed = parsed.statements
      .filter((s) => !s.isSupportedByContext)
      .map((s) => s.statement);

    return {
      score: Number((supportedCount / parsed.statements.length).toFixed(4)),
      failedStatements: failed,
    };
  }
}