mirror of
https://github.com/UditAkhourii/adhd.git
synced 2026-09-14 16:00:16 +08:00
Udit Raj
75616bf73c
- bench/run-evals.ts — ADHD vs single-shot baseline across problem set,
LLM-as-judge scoring on breadth/novelty/trap-detection/actionability/
builder-usefulness with A/B order randomized per problem
- bench/problems.json — 6 open-ended engineering problems
- bench/judge.ts — separate critic pass with skeptical-staff-eng prompt
- EVALS.md — populated: ADHD 5W / 1L / 0T vs baseline
breadth +4.17, novelty +5.17, trap_detection +7.67,
actionability +3.00, builder_usefulness +0.83 (mean Δ across 6 problems)
- .github/workflows/ci.yml — typecheck + build on Node 18/20/22
- .github/workflows/evals.yml — manual-trigger eval run, artifacts uploaded,
EVALS.md committed back to main
- tsconfig.bench.json — typecheck covers bench/ too
- CONTRIBUTING.md — frame-authoring guide, eval-problem guide, loop invariants
- README badges (CI, Evals, npm, license, Node), Evals section
33 lines
1.4 KiB
JSON
33 lines
1.4 KiB
JSON
[
|
|
{
|
|
"id": "lru-100ms",
|
|
"category": "systems",
|
|
"problem": "Design a thread-safe LRU cache that survives process restart without losing more than the last 100ms of writes."
|
|
},
|
|
{
|
|
"id": "llm-hang-cli",
|
|
"category": "ux/reliability",
|
|
"problem": "We have a CLI that calls an LLM and the LLM sometimes hangs for 90 seconds before responding. Design the right retry/timeout/UX strategy."
|
|
},
|
|
{
|
|
"id": "rate-limit-leader",
|
|
"category": "distsys",
|
|
"problem": "Design a rate limiter that stays correct across a leader election. Existing leader had counters in memory; new leader takes over with no warning."
|
|
},
|
|
{
|
|
"id": "fuzzy-bug",
|
|
"category": "debugging",
|
|
"problem": "0.1% of API requests time out intermittently. No stack trace, no obvious pattern, no recent deploy. How should we investigate? Generate hypothesis classes, not specific fixes."
|
|
},
|
|
{
|
|
"id": "monolith-split",
|
|
"category": "refactor",
|
|
"problem": "We have a 200k-line Rails monolith. The team wants to split it. Generate strategies for how to decompose it — by domain, data, team, churn, or otherwise."
|
|
},
|
|
{
|
|
"id": "naming-feature-flag",
|
|
"category": "naming",
|
|
"problem": "Generate names for a feature-flag service that supports gradual rollout, kill-switches, and per-tenant overrides. The name should signal control and reversibility."
|
|
}
|
|
]
|