mirror of
https://github.com/getsentry/sentry-for-ai.git
synced 2026-09-14 14:28:22 +08:00
5402298cfc
Add vitest-evals based eval system for testing SDK install skills against realistic fixture projects with real Claude API calls. Includes sandboxed runner with file tools (no shell/network), LLM judge for soft assertions, auto-discovered scenarios, and CI workflow gated on eval/skill path changes or run-evals label. Initial scenario: sentry-ruby-sdk rails-basic (Rails 8, Puma, SQLite). Assertions validated against real telemetry using a file-writing transport. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
20 lines
528 B
TypeScript
20 lines
528 B
TypeScript
import { defineConfig } from "vitest/config";
|
|
|
|
const jsonOutputFile = process.env["VITEST_EVALS_JSON"];
|
|
|
|
export default defineConfig({
|
|
test: {
|
|
include: ["src/**/*.eval.ts"],
|
|
exclude: ["**/node_modules/**", "**/dist/**"],
|
|
setupFiles: ["./src/setup.ts"],
|
|
includeTaskLocation: true,
|
|
reporters: [
|
|
["vitest-evals/reporter", { toolDetails: false }],
|
|
...(jsonOutputFile ? ([["json"]] as const) : []),
|
|
],
|
|
outputFile: {
|
|
...(jsonOutputFile ? { json: jsonOutputFile } : {}),
|
|
},
|
|
},
|
|
});
|