mirror of
https://github.com/getsentry/sentry-for-ai.git
synced 2026-09-14 14:28:22 +08:00
Johannes Daxböck
5402298cfc
Add vitest-evals based eval system for testing SDK install skills against realistic fixture projects with real Claude API calls. Includes sandboxed runner with file tools (no shell/network), LLM judge for soft assertions, auto-discovered scenarios, and CI workflow gated on eval/skill path changes or run-evals label. Initial scenario: sentry-ruby-sdk rails-basic (Rails 8, Puma, SQLite). Assertions validated against real telemetry using a file-writing transport. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
22 lines
504 B
JSON
22 lines
504 B
JSON
{
|
|
"name": "sentry-for-ai-evals",
|
|
"private": true,
|
|
"type": "module",
|
|
"scripts": {
|
|
"evals": "vitest run --config vitest.config.ts",
|
|
"evals:verbose": "EVAL_VERBOSE=true vitest run --config vitest.config.ts",
|
|
"typecheck": "tsc --noEmit"
|
|
},
|
|
"dependencies": {
|
|
"@anthropic-ai/sdk": "^0.98.0",
|
|
"vitest-evals": "0.11.0",
|
|
"dotenv": "^17.4.2",
|
|
"zod": "^4.4.3"
|
|
},
|
|
"devDependencies": {
|
|
"@types/node": "^25.9.1",
|
|
"typescript": "^6.0.3",
|
|
"vitest": "^4.1.7"
|
|
}
|
|
}
|