mirror of
https://github.com/zernie/vigiles.git
synced 2026-09-14 20:53:57 +08:00
zernie
94af23a5b7
Closes #173, #174, #175, #176 and #170. #174 — decideHook read exit 2, `decision` and `permissionDecision` but not the harness's halt-the-turn field, so `assertBlocksDisasters` reported a guard that stopped every disaster as blocking none. The field is read from the PORT, not hard-coded: `continue` is documented for Claude Code and unverified for Codex. The issue's adjacent suggestion was deliberately NOT implemented — adding the halt to `hook-block-ineffective` as a block-attempt signal would fire on hooks that WORK, since a halt is not event-scoped. It reads as a suppressor instead, fixing a real false alarm. A mutation implementing the original suggestion literally is pinned by a test that fails on it. #173 — compile printed its errors, exited 1, and wrote a hash-valid CLAUDE.md anyway, so `lint` went green over refs it had just called dead. Nothing is written on a failed compile now. #176 — eight small defects, including `audit --out` outside the repo appending unreachable `../../..` entries to .gitignore (a documented read-only report editing a tracked file), an unknown command exiting 1 against a documented 2, and STABILITY.md promising the removed `scan` verb. #175 + #176.2 — three missing capabilities, each opt-in and additive: `{surface}` in testGlobs, `--single`, and `rulesDir` so `.claude/rules/**` stops being invisible. #170 — one enumeration of export forms written against the grammar, both naming checks driven off it, and `check-internal-tag.mjs` asking the TypeScript compiler instead of a regex. It immediately found the issue's open case, 13 tagged types the regex had never seen, and a live gap in the ESLint rule. BREAKING CHANGE: `HookRunResult` gains a required `haltsTurn`, so code that CONSTRUCTS one (a test fake) must set it. Reading a result is unaffected. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
9.8 KiB
9.8 KiB
API Report File for "vigiles-eval"
Do not edit this file. It is a report generated by API Extractor.
// @public
export interface AgentRunArgs {
// (undocumented)
readonly cwd: string;
readonly env?: Record<string, string>;
// (undocumented)
readonly hasSettings: boolean;
// (undocumented)
readonly model: string;
// (undocumented)
readonly pluginDir: string | undefined;
readonly replaceEnv?: boolean;
// (undocumented)
readonly task: string;
// (undocumented)
readonly timeoutMs: number;
// (undocumented)
readonly tools: readonly string[];
}
// @public
export type AgentRunner = (args: AgentRunArgs) => Promise<RunOut>;
// @public (undocumented)
export interface ArmReport {
readonly metrics: Record<string, number>;
// (undocumented)
readonly runs: number;
readonly stats: Record<string, MetricStat>;
readonly usage: ArmUsage;
}
// @public
export interface ArmsCheckReport {
// (undocumented)
readonly arms: Record<string, CheckReport>;
}
// @public
export interface ArmsMeasureSpec {
// (undocumented)
readonly allowedTools?: readonly string[];
readonly arms: Record<string, EvalArm>;
// (undocumented)
readonly checks: readonly Check<RunContext>[];
// (undocumented)
readonly fixture?: Record<string, string>;
// (undocumented)
readonly model?: string;
// (undocumented)
readonly spacingSec?: number;
readonly stubSkillBodies?: boolean;
// (undocumented)
readonly task: string;
// (undocumented)
readonly timeoutMs?: number;
// (undocumented)
readonly trials?: number;
}
// @public
export interface ArmUsage {
// (undocumented)
readonly meanCostUsd: number;
// (undocumented)
readonly meanDurationMs: number;
// (undocumented)
readonly totalCacheCreationTokens: number;
// (undocumented)
readonly totalCacheReadTokens: number;
// (undocumented)
readonly totalCostUsd: number;
// (undocumented)
readonly totalInputTokens: number;
// (undocumented)
readonly totalOutputTokens: number;
}
// @public
export interface Check<T> {
eval(target: T): CheckResult;
readonly kind: string;
toJSON(): CheckJSON;
}
// @public
export interface CheckRate {
// (undocumented)
readonly check: CheckJSON;
readonly n: number;
readonly passK: number;
readonly rate: number;
readonly se: number;
}
// @public (undocumented)
export interface CheckReport {
// (undocumented)
readonly n: number;
// (undocumented)
readonly perCheck: readonly CheckRate[];
readonly usage: ArmUsage;
}
// @public
export interface CheckResult {
readonly message: string;
readonly pass: boolean;
readonly score: number;
}
// @public
export interface EvalArm {
readonly files?: Record<string, string>;
readonly interceptTools?: readonly ToolIntercept[];
readonly model?: string;
readonly plugin?: string;
readonly pluginDir?: string;
readonly settings?: unknown;
}
// @public
export interface EvalDriver {
readonly experimental?: string;
readonly harness?: string;
// (undocumented)
readonly parse: ModelOutputParser;
// (undocumented)
readonly runError?: (out: RunOut) => string | null;
// (undocumented)
readonly runner: AgentRunner;
}
// @public (undocumented)
export interface EvalReport {
readonly aborted: boolean;
// (undocumented)
readonly arms: Record<string, ArmReport>;
// (undocumented)
readonly name: string;
readonly totalCostUsd: number;
// (undocumented)
readonly trials: number;
}
// @public (undocumented)
export interface EvalSpec<M extends Metrics> {
readonly allowedTools?: readonly string[];
readonly arms: Record<string, EvalArm>;
readonly cache?: CacheMode;
readonly cacheDir?: string;
readonly concurrency?: number;
readonly ephemeralEnv?: boolean;
readonly fixture?: Record<string, string>;
readonly lock?: EvalLockOptions;
readonly maxCostUsd?: number;
readonly measure: (ctx: RunContext) => M;
readonly model?: string;
// (undocumented)
readonly name?: string;
readonly rateLimitRetries?: number;
readonly retryBackoffMs?: number;
readonly spacingSec?: number;
readonly stubs?: readonly ToolStub[];
readonly task: string;
readonly timeoutMs?: number;
readonly trials?: number;
}
// @public
export interface EvalUsage {
readonly cacheCreationTokens: number;
readonly cacheReadTokens: number;
readonly costUsd: number;
readonly durationMs: number;
readonly inputTokens: number;
// (undocumented)
readonly outputTokens: number;
}
// @public
export type JudgeFn = (opts: {
output: string;
rubric: string;
threshold?: number;
}) => {
score: number;
pass: boolean;
reason?: string;
};
// @public
export interface MeasureSpec {
readonly allowedTools?: readonly string[];
readonly checks: readonly Check<RunContext>[];
readonly fixture?: Record<string, string>;
readonly interceptTools?: readonly ToolIntercept[];
readonly model?: string;
readonly plugin?: string;
readonly pluginDir?: string;
readonly settings?: unknown;
readonly spacingSec?: number;
readonly stubSkillBodies?: boolean;
readonly task: string;
readonly timeoutMs?: number;
readonly trials?: number;
}
// @public (undocumented)
export type Metrics = Record<string, number | boolean>;
// @public
export interface MetricStat {
readonly mean: number;
readonly n: number;
readonly passK: number;
readonly se: number;
readonly std: number;
}
// @public (undocumented)
export type ModelOutputParser = (out: RunOut) => ParsedModelRun;
// @public
export const paid_claudeEvalDriver: EvalDriver;
// @public
export function paid_judge(opts: JudgeOptions): JudgeResult;
// @public
export function paid_judged(rubric: string, opts?: {
min?: number;
judge?: JudgeFn;
}): Check<Trace>;
// @public
export function paid_measure(spec: MeasureSpec): Promise<CheckReport>;
// @public
export function paid_measureArms(spec: ArmsMeasureSpec): Promise<ArmsCheckReport>;
// @public
export function paid_measureTriggerRate(spec: TriggerRateSpec, opts?: {
evalDriver?: EvalDriver;
}): Promise<TriggerRateReport>;
// @public (undocumented)
export function paid_runEval<M extends Metrics>(spec: EvalSpec<M>): Promise<EvalReport>;
// @public
export interface ParsedModelRun {
// (undocumented)
readonly hooks: ReturnType<typeof parseHooks>;
// (undocumented)
readonly output: string;
// (undocumented)
readonly subagents: ReturnType<typeof parseSubagents>;
// (undocumented)
readonly toolCalls: ReturnType<typeof parseToolCalls>;
// (undocumented)
readonly turns: number;
// (undocumented)
readonly usage: EvalUsage;
}
// @public (undocumented)
export interface PromptDiversityIssue {
// (undocumented)
readonly kind: "too-few" | "too-similar";
// (undocumented)
readonly message: string;
}
// @public
export interface PromptTriggerStat {
// (undocumented)
readonly fired: number;
// (undocumented)
readonly prompt: string;
readonly rate: number;
// (undocumented)
readonly trials: number;
}
// @public
export interface RunContext extends Trace {
// (undocumented)
readonly cwd: string;
// (undocumented)
readonly exitCode: number;
sh(command: string): string;
// (undocumented)
readonly stdout: string;
readonly toolCalls: readonly ToolCall[];
readonly turns: number;
readonly usage: EvalUsage;
}
// @public
export interface RunOut {
// (undocumented)
code: number;
stderr?: string;
// (undocumented)
stdout: string;
}
// @public
export interface SelectionTrialResult {
// (undocumented)
readonly errored: boolean;
// (undocumented)
readonly fired: readonly string[];
}
// @public
export interface Trace {
file(path: string): string | null;
readonly hooks: readonly HookFire[];
readonly modelRequests: readonly ModelRequest[];
readonly output: string;
readonly subagents?: readonly SubagentTrace[];
readonly toolCalls: readonly ToolCall[];
readonly turns: number;
}
// @public (undocumented)
export interface TriggerRateReport {
readonly competitors: number;
readonly errored?: number;
readonly experimental?: string;
readonly falsePositiveRate?: number;
readonly n: number;
readonly namespace?: string;
readonly perIrrelevant?: readonly PromptTriggerStat[];
// (undocumented)
readonly perPrompt: readonly PromptTriggerStat[];
readonly precision?: number;
readonly rate: number;
readonly usage: ArmUsage;
}
// @public
export interface TriggerRateSpec {
readonly allowedTools?: readonly string[];
readonly concurrency?: number;
readonly fired: (trace: Trace) => boolean;
readonly fixture?: Record<string, string>;
readonly installSet?: readonly string[];
readonly irrelevantPrompts?: readonly string[];
readonly lock?: EvalLockOptions;
readonly minDistance?: number;
readonly minModel?: string;
readonly minPrompts?: number;
readonly model?: string;
readonly name?: string;
readonly pluginDir?: string;
readonly prompts: readonly string[];
readonly skillsDir?: string;
readonly spacingSec?: number;
readonly stubSkillBodies?: boolean;
readonly timeoutMs?: number;
readonly trials?: number;
}
// (No @packageDocumentation comment for this package)