mirror of
https://github.com/workos/skills.git
synced 2026-09-14 20:07:08 +08:00
Nick Nisi
dc9dfb093c
* fix(workos): include required --org flag in portal generate-link examples The "Not in the CLI" table in workos-management.md listed `workos portal generate-link --intent=sso` and `workos portal generate-link --intent=dsync` as the correct way to reach the Admin Portal for connection creation. Both omit the required --org flag and fail before opening the Portal. This is the same hallucination shape the PR is trying to prevent agents from producing — caught in review. Add --org=<org_id> to both rows, matching the Quick Reference entry at the top of the file. * test(workos): tighten no-CLI assertion in connection-create evals The first assertion in evals 8 and 9 was titled "States connection creation is NOT in the CLI" but its needles included Admin Portal and WorkOS Dashboard. An answer that only said "Use the Admin Portal" without ever stating CLI support is absent passed the assertion. Since the second assertion in each eval already checks for the Admin Portal/Dashboard destination, the first was both redundant and weakened. Remove destination terms from the first assertion so it genuinely checks for the explicit no-CLI signal. * test(workos): improve risky eval coverage Add targeted recipes for SSO, AuthKit, and RBAC cases that showed negative or noisy eval deltas. Tighten brittle flow-step expectations where the previous wording rewarded incidental prose ordering instead of the intended behavior. Add multi-case eval filtering and an eval:risk helper so the latest triage cases can be rerun with more samples and saved outputs. * Fix explicit triage handling in risk reruns * fix: format:check script incorrectly using prettier * chore: format files for oxfmt check
109 lines
2.3 KiB
TypeScript
109 lines
2.3 KiB
TypeScript
import { describe, expect, it } from 'vitest';
|
|
import { mkdtempSync, rmSync, writeFileSync } from 'fs';
|
|
import { tmpdir } from 'os';
|
|
import { join } from 'path';
|
|
import { loadCases, mean, stddev } from '../eval/runner.ts';
|
|
|
|
describe('mean', () => {
|
|
it('returns 0 for empty array', () => {
|
|
expect(mean([])).toBe(0);
|
|
});
|
|
|
|
it('returns the single value for length-1 array', () => {
|
|
expect(mean([42])).toBe(42);
|
|
});
|
|
|
|
it('computes arithmetic mean', () => {
|
|
expect(mean([95, 97, 93])).toBe(95);
|
|
});
|
|
|
|
it('handles negative values', () => {
|
|
expect(mean([-10, 10])).toBe(0);
|
|
});
|
|
|
|
it('handles decimal values', () => {
|
|
expect(mean([1, 2])).toBe(1.5);
|
|
});
|
|
});
|
|
|
|
describe('stddev', () => {
|
|
it('returns 0 for empty array', () => {
|
|
expect(stddev([])).toBe(0);
|
|
});
|
|
|
|
it('returns 0 for single value', () => {
|
|
expect(stddev([42])).toBe(0);
|
|
});
|
|
|
|
it('returns 0 for identical values', () => {
|
|
expect(stddev([5, 5, 5])).toBe(0);
|
|
});
|
|
|
|
it('computes population stddev', () => {
|
|
// mean=95, deviations: 0, 2, -2, variance=8/3, stddev=sqrt(8/3)≈1.633
|
|
expect(stddev([95, 97, 93])).toBeCloseTo(1.633, 2);
|
|
});
|
|
|
|
it('handles symmetric extremes', () => {
|
|
expect(stddev([100, 0])).toBe(50);
|
|
});
|
|
|
|
it('handles two identical values', () => {
|
|
expect(stddev([80, 80])).toBe(0);
|
|
});
|
|
});
|
|
|
|
describe('loadCases', () => {
|
|
it('filters by multiple case IDs', () => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'workos-eval-cases-'));
|
|
try {
|
|
writeFileSync(
|
|
join(dir, 'cases.yaml'),
|
|
`- id: first
|
|
product: sso
|
|
skill: workos-sso
|
|
skillType: generated
|
|
prompt: one
|
|
expected:
|
|
methods: []
|
|
envVars: []
|
|
imports: []
|
|
params: []
|
|
flowSteps: []
|
|
antiPatterns: []
|
|
- id: second
|
|
product: sso
|
|
skill: workos-sso
|
|
skillType: generated
|
|
prompt: two
|
|
expected:
|
|
methods: []
|
|
envVars: []
|
|
imports: []
|
|
params: []
|
|
flowSteps: []
|
|
antiPatterns: []
|
|
- id: third
|
|
product: rbac
|
|
skill: workos-rbac
|
|
skillType: generated
|
|
prompt: three
|
|
expected:
|
|
methods: []
|
|
envVars: []
|
|
imports: []
|
|
params: []
|
|
flowSteps: []
|
|
antiPatterns: []
|
|
`,
|
|
);
|
|
|
|
const cases = loadCases(dir, { caseIds: ['first', 'third'] });
|
|
|
|
expect(cases.map((c) => c.id)).toEqual(['first', 'third']);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
});
|