mirror of
https://github.com/ruvnet/ruflo.git
synced 2026-09-14 14:01:28 +08:00
ruv
57a4a1b735
Opt-in cross-encoder rerank pass over the hybrid+BM25 top-K. Uses
Xenova/ms-marco-MiniLM-L-6-v2 (int8, ~30MB) via direct AutoTokenizer +
AutoModelForSequenceClassification (xenova v2 pipeline can't do pair
encoding reliably). Lazy-loaded, one-shot load policy, graceful
degradation when model unavailable.
Architecture: hybrid top-K*6 → cross-encoder scores all pairs →
combined = 0.5*normalise(hybrid) + 0.5*normalise(ce) → top-K. MMR
bypassed when reranking is on (cross-encoder's calibrated score is
more precise than MMR's diversity heuristic at this stage).
Linear combination is essential — ablation showed:
- Hybrid alone: 8/10 top-1, 8/10 top-3, MRR 0.800
- CE alone: 6/10 top-1, 10/10 top-3, MRR 0.733
- Combined 0.5/0.5: 9/10 top-1, 10/10 top-3, MRR 0.933 ← default
Default is OFF (latency 25x hybrid: 1.0s vs 39ms at N=385).
Callers opt in via {rerank: true}. Weights tunable via
{hybridWeight, ceWeight}; grid-search shows 0.3-0.5 hybrid all hit
the same plateau.
Cumulative since cosine baseline (3.10.17 → 3.10.20):
| Metric | Cosine | 3.10.18 | 3.10.19 | 3.10.20 |
|------------------|-------:|--------:|--------:|--------:|
| Top-1 hit rate | 0% | 50% | 80% | 90% |
| Top-3 hit rate | 0% | 70% | 80% | 100% |
| MRR@3 | 0.000 | 0.583 | 0.800 | 0.933 |
| Top-1 diversity | 100% | 80% | 100% | 100% |
| Latency (rerank) | 29ms | 41ms | 39ms | 984ms |
5 new tests cover the degradation contract (no network needed).
Co-Authored-By: RuFlo <ruv@ruv.net>
68 lines
2.6 KiB
TypeScript
68 lines
2.6 KiB
TypeScript
// Cross-encoder reranker — graceful-degradation tests (ADR-080).
|
|
//
|
|
// We don't run the actual cross-encoder here (network + 30MB model download
|
|
// would make the test suite flaky). Instead we verify the fallback contract:
|
|
// when the model can't be loaded the API must still return a well-shaped
|
|
// answer (input order preserved, score=0) so callers never break.
|
|
|
|
import { describe, it, expect, beforeEach } from 'vitest';
|
|
import {
|
|
crossEncoderRerank,
|
|
getCrossEncoder,
|
|
getCrossEncoderStatus,
|
|
resetCrossEncoder,
|
|
} from '../src/memory/cross-encoder-rerank.js';
|
|
|
|
describe('crossEncoderRerank — graceful degradation contract', () => {
|
|
beforeEach(() => {
|
|
resetCrossEncoder();
|
|
});
|
|
|
|
it('returns input order with score=0 when model name is invalid', async () => {
|
|
// Force the loader to fail by passing a guaranteed-bad model name.
|
|
const ce = await getCrossEncoder('does-not-exist/no-such-model-anywhere');
|
|
expect(ce).toBeNull();
|
|
|
|
// Even though the model failed, rerank() must still return a usable
|
|
// ranking — caller falls back to hybrid order.
|
|
const out = await crossEncoderRerank('query', ['doc1', 'doc2', 'doc3']);
|
|
expect(out).toHaveLength(3);
|
|
expect(out.map((r) => r.index)).toEqual([0, 1, 2]);
|
|
expect(out.every((r) => r.score === 0)).toBe(true);
|
|
});
|
|
|
|
it('respects topK when degraded', async () => {
|
|
resetCrossEncoder();
|
|
// Trigger a failure first
|
|
await getCrossEncoder('does-not-exist/no-such-model-anywhere');
|
|
const out = await crossEncoderRerank('query', ['a', 'b', 'c', 'd', 'e'], 2);
|
|
expect(out).toHaveLength(2);
|
|
});
|
|
|
|
it('handles empty doc list', async () => {
|
|
const out = await crossEncoderRerank('query', []);
|
|
expect(out).toEqual([]);
|
|
});
|
|
|
|
it('getCrossEncoderStatus reports loaded=false after failed load', async () => {
|
|
resetCrossEncoder();
|
|
await getCrossEncoder('does-not-exist/no-such-model-anywhere');
|
|
const status = getCrossEncoderStatus();
|
|
expect(status.attempted).toBe(true);
|
|
expect(status.loaded).toBe(false);
|
|
expect(status.error).toBeTruthy();
|
|
});
|
|
|
|
it('does not retry after a failure (one-shot load policy)', async () => {
|
|
resetCrossEncoder();
|
|
const first = await getCrossEncoder('does-not-exist/no-such-model-anywhere');
|
|
expect(first).toBeNull();
|
|
// Subsequent call should NOT re-attempt — returns null immediately.
|
|
const tStart = Date.now();
|
|
const second = await getCrossEncoder('any-other-name');
|
|
const elapsed = Date.now() - tStart;
|
|
expect(second).toBeNull();
|
|
expect(elapsed).toBeLessThan(50); // No retry; instant return.
|
|
});
|
|
});
|