import { EmbeddingProvider, ProviderClassification } from '../types'; // Deterministic embedding provider used in CI tests for the indexer and the // search layer. Same text always produces the same vector, and texts that share // characters produce more-similar vectors than texts that don't — so // similaritySearch() returns sensible orderings. // // This is NOT a real embedding model. It can't capture semantics that span // multiple words or handle synonyms. It exists purely so we can exercise the // indexing + search pipeline end-to-end in CI without bundling onnxruntime-node // or a 100MB model file. const DEFAULT_DIMENSION = 32; const hashString = (s: string): number => { // Simple FNV-1a 32-bit hash, used to map each character bigram to a vector // index. Deterministic and fast. let h = 0x811c9dc5; for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 0x01000193); } return h >>> 0; }; const l2Norm = (v: number[]): number => { let sum = 0; for (const x of v) sum += x * x; return Math.sqrt(sum); }; const normalise = (v: number[]): number[] => { const norm = l2Norm(v); if (norm === 0) return v; return v.map(x => x / norm); }; const addUnitX = (v: number[]) => { return v.map((x, i) => (i === 0 ? x + 1 : x)); }; const embedOne = (text: string, dimension: number): number[] => { const vec = new Array(dimension).fill(0); const lower = text.toLowerCase(); // Character bigrams give a richer signal than unigrams (which would // collapse "cat" and "act" together) without exploding the keyspace. for (let i = 0; i < lower.length - 1; i++) { const bigram = lower.slice(i, i + 2); const bucket = hashString(bigram) % dimension; vec[bucket] += 1; } // Always include a constant signal so empty or one-character strings // still produce a non-zero vector (otherwise normalise() returns zeros // and similarity is undefined). vec[0] += 1; // Add a constant post-normalization to increase the similarity scores to more // closely match the production model (which distributes scores roughly sin [0.7, 1]) return normalise( addUnitX(normalise(vec)), ); }; export interface TestEmbeddingProviderOptions { id?: string; modelId?: string; dimension?: number; } export default class TestEmbeddingProvider implements EmbeddingProvider { public id: string; public modelId: string; public dimension: number; public classification: ProviderClassification = 'local'; public constructor(options: TestEmbeddingProviderOptions = {}) { this.id = options.id ?? 'test'; this.modelId = options.modelId ?? 'test-model-v1'; this.dimension = options.dimension ?? DEFAULT_DIMENSION; } public async embed(texts: string[]): Promise { return texts.map(t => embedOne(t, this.dimension)); } }