mirror of
https://github.com/vectorize-io/hindsight.git
synced 2026-09-14 19:31:49 +08:00
Nicolò Boschi
613a699e9f
Round-robin interleave fusion for consolidation dedup recall (guarantees the semantic-#1 'twin' a slot so the LLM updates instead of duplicating), unified 'reranking' strategy param (cross_encoder/rrf/interleave), case-sensitive exact-dup guard, obs-dedup tool + benchmark wired into the perf dashboard (English dataset). Near-dup observation rate 4% -> 0% on the English hermes transcript (1/10 and 1/4), coverage 89% -> 94%, no false merges.
33 lines
1.1 KiB
Bash
Executable File
33 lines
1.1 KiB
Bash
Executable File
#!/bin/bash
|
|
set -e
|
|
|
|
# Observation Duplication Benchmark Runner
|
|
# Measures the duplicate-observation rate consolidation produces over a dataset.
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
# Source .env if it exists
|
|
if [ -f "$REPO_ROOT/.env" ]; then
|
|
source "$REPO_ROOT/.env"
|
|
echo "Loaded environment from .env"
|
|
fi
|
|
|
|
# Enable observations (required for consolidation)
|
|
export HINDSIGHT_API_ENABLE_OBSERVATIONS=true
|
|
# Note: the benchmark uses SyncTaskBackend (inline/serial task execution) and disables
|
|
# auto-consolidation per-bank, so its explicit drain loop is the sole consolidator with no
|
|
# background worker racing it. See benchmarks/obs/obs_benchmark.py for the reasoning.
|
|
|
|
echo "Running observation duplication benchmark with configuration:"
|
|
echo " HINDSIGHT_API_LLM_PROVIDER=${HINDSIGHT_API_LLM_PROVIDER:-not set}"
|
|
echo " HINDSIGHT_API_LLM_MODEL=${HINDSIGHT_API_LLM_MODEL:-not set}"
|
|
echo ""
|
|
|
|
# Run benchmark
|
|
cd "$REPO_ROOT"
|
|
uv run python -m benchmarks.obs.obs_benchmark
|
|
|
|
echo ""
|
|
echo "Benchmark complete! Check benchmarks/results/ for detailed results."
|