garrytan__gstack/test/skill-fixture.test.ts

/**
 * Unit + pin tests for test/helpers/skill-fixture.ts (free tier, no EVALS).
 *
 * Two layers:
 *   1. Semantics against a synthetic SKILL.md: frontmatter always included,
 *      sections concatenated in caller order, missing section throws with the
 *      section name, fenced `## ` template headings do not split sections,
 *      body extraction drops exactly the shared preamble block, head
 *      extraction truncates the body.
 *   2. Pins against the REAL generated SKILL.md files: every exported section
 *      list extracts cleanly from the skill it targets, and the body/head
 *      helpers work for every skill the E2E fixtures feed through them. A
 *      heading rename in gen-skill-docs fails HERE (free, <1s) instead of
 *      mid-flight in a paid E2E run.
 */

import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import {
  extractSkillSections,
  extractSkillBody,
  extractSkillHead,
  REVIEW_E2E_SECTIONS,
  REVIEW_ARMY_E2E_SECTIONS,
  RETRO_E2E_SECTIONS,
  CODEX_REVIEW_E2E_SECTIONS,
} from './helpers/skill-fixture';
import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES, selectTests } from './helpers/touchfiles';

const ROOT = path.resolve(import.meta.dir, '..');

// ─── Synthetic fixture ──────────────────────────────────────────────────────

const SYNTHETIC_SKILL = `---
name: fixture-test
description: synthetic skill for skill-fixture unit tests
---
Intro line before any section.

## When to invoke this skill
Invoke text.

## Preamble (run first)
preamble junk that fixtures must drop

## AskUserQuestion Format
more shared-preamble junk

## Plan Status Footer
footer junk, last shared-preamble section

## Step 1 — Do the thing
step one body
\`\`\`markdown
## Embedded Template Heading
template content inside a fence
\`\`\`
step one continues after the fence

## Step 2 — Other
step two body
`;

let tmpDir: string;
let skillDir: string;

beforeAll(() => {
  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-fixture-test-'));
  skillDir = path.join(tmpDir, 'fixture-test');
  fs.mkdirSync(skillDir, { recursive: true });
  fs.writeFileSync(path.join(skillDir, 'SKILL.md'), SYNTHETIC_SKILL);
});

afterAll(() => {
  try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
});

describe('extractSkillSections (synthetic)', () => {
  test('always includes frontmatter and concatenates sections in caller order', () => {
    const out = extractSkillSections(skillDir, ['Step 2 — Other', 'Step 1 — Do the thing']);
    expect(out.startsWith('---\nname: fixture-test')).toBe(true);
    expect(out).toContain('## Step 1 — Do the thing');
    expect(out).toContain('## Step 2 — Other');
    // Caller order preserved: Step 2 requested first, so it appears first.
    expect(out.indexOf('## Step 2 — Other')).toBeLessThan(out.indexOf('## Step 1 — Do the thing'));
    // Unrequested sections are dropped.
    expect(out).not.toContain('preamble junk');
    expect(out).not.toContain('Intro line before any section');
  });

  test('fenced ## headings do not terminate a section', () => {
    const out = extractSkillSections(skillDir, ['Step 1 — Do the thing']);
    expect(out).toContain('template content inside a fence');
    expect(out).toContain('step one continues after the fence');
    expect(out).not.toContain('step two body');
  });

  test('missing section throws with the section name and the file path', () => {
    expect(() => extractSkillSections(skillDir, ['Step 99 — Renamed'])).toThrow(/Step 99 — Renamed/);
    expect(() => extractSkillSections(skillDir, ['Step 99 — Renamed'])).toThrow(/SKILL\.md/);
  });

  test('a fenced heading is not findable as a section', () => {
    expect(() => extractSkillSections(skillDir, ['Embedded Template Heading'])).toThrow(/not found/);
  });
});

describe('extractSkillBody (synthetic)', () => {
  test('keeps frontmatter + intro + full body, drops the shared preamble block', () => {
    const out = extractSkillBody(skillDir);
    expect(out.startsWith('---\nname: fixture-test')).toBe(true);
    expect(out).toContain('Intro line before any section');
    expect(out).toContain('## When to invoke this skill');
    expect(out).toContain('## Step 1 — Do the thing');
    expect(out).toContain('template content inside a fence');
    expect(out).toContain('## Step 2 — Other');
    expect(out).not.toContain('preamble junk');
    expect(out).not.toContain('shared-preamble junk');
    expect(out).not.toContain('footer junk');
  });

  test('throws when the preamble markers are missing', () => {
    const bare = path.join(tmpDir, 'bare');
    fs.mkdirSync(bare, { recursive: true });
    fs.writeFileSync(path.join(bare, 'SKILL.md'), '---\nname: bare\n---\n## Only Section\nbody\n');
    expect(() => extractSkillBody(bare)).toThrow(/Preamble \(run first\)/);
  });

  test('both exact preamble headings preserve the complete intro/scope gate and tail', () => {
    for (const heading of ['Preamble (run first)', 'Preamble (after scope gate)']) {
      const file = path.join(tmpDir, 'scoped-body.md');
      const scope = '## Scope gate\nAnnounce the actual selected plan before any tools.\n\n';
      const input = SYNTHETIC_SKILL.replace('## Preamble (run first)', scope + `## ${heading}`);
      fs.writeFileSync(file, input);
      const out = extractSkillBody(file);
      const expected = extractSkillBody(skillDir).replace('Invoke text.\n\n', 'Invoke text.\n\n' + scope);
      expect(out).toBe(expected);
      expect(out).toContain(scope.trimEnd());
      expect(out).not.toContain(`## ${heading}`);
    }
  });

  test('unknown, missing, duplicated and reversed boundary sections fail loudly', () => {
    const cases: Array<[string, string, RegExp]> = [
      ['unknown start', SYNTHETIC_SKILL.replace('Preamble (run first)', 'Preamble (after setup)'), /Preamble/],
      ['suffixed scoped start', SYNTHETIC_SKILL.replace('Preamble (run first)', 'Preamble (after scope gate) extra'), /Preamble/],
      ['suffixed old start', SYNTHETIC_SKILL.replace('Preamble (run first)', 'Preamble (run first) extra'), /Preamble/],
      ['missing footer', SYNTHETIC_SKILL.replace('## Plan Status Footer', '## Renamed Footer'), /Plan Status Footer/],
      ['suffixed footer', SYNTHETIC_SKILL.replace('## Plan Status Footer', '## Plan Status Footer extra'), /Plan Status Footer/],
      ['both starts', SYNTHETIC_SKILL.replace('## AskUserQuestion Format', '## Preamble (after scope gate)'), /ambiguous/],
      ['duplicate start', SYNTHETIC_SKILL.replace('## AskUserQuestion Format', '## Preamble (run first)'), /ambiguous/],
      ['duplicate footer', SYNTHETIC_SKILL.replace('## Step 2 — Other', '## Plan Status Footer'), /ambiguous/],
      ['footer before start', SYNTHETIC_SKILL.replace('## Preamble (run first)', '## Plan Status Footer').replace('## Plan Status Footer\nfooter junk', '## Preamble (after scope gate)\nfooter junk'), /precedes/],
      ['missing tail', SYNTHETIC_SKILL.slice(0, SYNTHETIC_SKILL.indexOf('## Step 1 — Do the thing')), /no content after/],
    ];
    for (const [name, input, error] of cases) {
      const file = path.join(tmpDir, 'invalid-boundaries.md'); fs.writeFileSync(file, input);
      expect(() => extractSkillBody(file), name).toThrow(error);
      expect(() => extractSkillBody(file), name).toThrow(/invalid-boundaries\.md/);
    }
  });

  test('quoted and fenced boundary examples do not become live section markers', () => {
    for (const [open, close] of [['```markdown\n', '```\n'], ['~~~markdown\n', '~~~\n']]) {
      const file = path.join(tmpDir, 'fenced-boundaries.md');
      const examples = `${open}## Preamble (after scope gate)\n## Plan Status Footer\n${close}> ## Preamble (after scope gate)\n\n`;
      fs.writeFileSync(file, SYNTHETIC_SKILL.replace('Intro line before any section.', examples + 'Intro line before any section.'));
      expect(extractSkillBody(file)).toContain(examples.trimEnd());
      fs.writeFileSync(file, SYNTHETIC_SKILL.replace('## Preamble (run first)\npreamble junk', `${open}## Preamble (after scope gate)\n${close}preamble junk`));
      expect(() => extractSkillBody(file)).toThrow(/Preamble/);
    }
  });

  test('other section extraction keeps its existing prefix matching', () => {
    const file = path.join(tmpDir, 'section-prefix.md');
    fs.writeFileSync(file, SYNTHETIC_SKILL.replace('Preamble (run first)', 'Preamble (after scope gate)'));
    expect(extractSkillSections(file, ['Preamble (after'])).toContain('preamble junk');
    expect(extractSkillSections(file, ['Step 1'])).toContain('step one continues after the fence');
  });

  test('the existing global source dependency selects every affected workflow fixture', () => {
    expect(GLOBAL_TOUCHFILES).toContain('test/helpers/skill-fixture.ts');
    const selection = selectTests(['test/helpers/skill-fixture.ts'], E2E_TOUCHFILES, GLOBAL_TOUCHFILES);
    expect(selection.selected.sort()).toEqual(Object.keys(E2E_TOUCHFILES).sort());
  });
});

describe('extractSkillHead (synthetic)', () => {
  test('keeps frontmatter + first N body lines only', () => {
    const out = extractSkillHead(skillDir, 3);
    expect(out.startsWith('---\nname: fixture-test')).toBe(true);
    expect(out).toContain('Intro line before any section');
    expect(out).toContain('## When to invoke this skill');
    expect(out).not.toContain('## Step 1 — Do the thing');
    expect(out).toContain('body truncated by test/helpers/skill-fixture.ts');
  });
});

describe('error polarity', () => {
  test('missing SKILL.md throws (never writes an empty fixture)', () => {
    expect(() => extractSkillSections(path.join(tmpDir, 'nope'), ['x'])).toThrow(/no SKILL\.md/);
  });

  test('file without frontmatter throws', () => {
    const nofm = path.join(tmpDir, 'nofm');
    fs.mkdirSync(nofm, { recursive: true });
    fs.writeFileSync(path.join(nofm, 'SKILL.md'), '# no frontmatter\n## Section\n');
    expect(() => extractSkillHead(nofm)).toThrow(/frontmatter/);
  });
});

// ─── Pins against the real generated SKILL.md files ─────────────────────────
// These turn "someone renamed a section in gen-skill-docs" into a FREE test
// failure instead of a paid E2E setup throw.

describe('real-skill pins: section lists used by E2E fixtures', () => {
  test('REVIEW_E2E_SECTIONS extracts from review/SKILL.md', () => {
    const out = extractSkillSections(path.join(ROOT, 'review'), REVIEW_E2E_SECTIONS);
    expect(out).toContain('## Step 4: Critical pass (core review)');
    expect(out).toContain('## Important Rules');
    // Drops the shared preamble and the untested workflow tail.
    expect(out).not.toContain('## Telemetry (run last)');
    expect(out).not.toContain('## Step 5: Fix-First Review');
    // Meaningfully smaller than the source.
    const full = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
    expect(out.length).toBeLessThan(full.length * 0.5);
  });

  test('REVIEW_ARMY_E2E_SECTIONS extracts from review/SKILL.md + carved sections', () => {
    // /review is carved (T9): the Step 4.5 dispatch body lives in
    // sections/review-army.md and the Plan Completion Audit in
    // sections/plan-completion.md — the skeleton keeps only STOP-Read pointers.
    // The E2E fixture (test/skill-e2e-review-army.test.ts) extracts the
    // skeleton H2s minus Step 4.5, then appends both section files; this pin
    // mirrors that exact construction so a section rename or an empty carve
    // still fails FREE before a paid E2E setup throw.
    const skeletonSections = REVIEW_ARMY_E2E_SECTIONS.filter(
      (s) => !s.startsWith('Step 4.5'),
    );
    const out = extractSkillSections(path.join(ROOT, 'review'), skeletonSections);
    expect(out).toContain('## Step 1.5: Scope Drift Detection');
    expect(out).not.toContain('## Telemetry (run last)');

    const planCompletion = fs.readFileSync(
      path.join(ROOT, 'review', 'sections', 'plan-completion.md'), 'utf-8');
    expect(planCompletion).toContain('PLAN COMPLETION AUDIT');

    const army = fs.readFileSync(
      path.join(ROOT, 'review', 'sections', 'review-army.md'), 'utf-8');
    expect(army).toContain('## Step 4.5: Review Army — Specialist Dispatch');
    expect(army).toContain('quality_score');
    expect(army).toContain('MULTI-SPECIALIST CONFIRMED');
  });

  test('RETRO_E2E_SECTIONS skeleton extracts from retro/SKILL.md + carved section', () => {
    // Carved (retro wave): the '## Engineering Retro: [date range]' report
    // format lives in retro/sections/report-format.md; the E2E fixture builds
    // skeleton sections + the section file (see skill-e2e-retro.test.ts).
    const skeletonSections = RETRO_E2E_SECTIONS.filter(
      (s) => s !== 'Engineering Retro: [date range]',
    );
    const out = extractSkillSections(path.join(ROOT, 'retro'), skeletonSections);
    // Steps 0.5-14 live under Prior Learnings / Capture Learnings.
    expect(out).toContain('### Step 1: Gather');
    expect(out).toContain('### Step 14: Write the Narrative');
    expect(out).not.toContain('## Global Retrospective Mode');
    expect(out).not.toContain('## Telemetry (run last)');

    const reportFormat = fs.readFileSync(
      path.join(ROOT, 'retro', 'sections', 'report-format.md'), 'utf-8');
    expect(reportFormat).toContain('## Engineering Retro: [date range]');
    expect(reportFormat).toContain('### Team Breakdown');
  });

  test('CODEX_REVIEW_E2E_SECTIONS extracts from the Codex host variant when present', () => {
    const codexReview = path.join(ROOT, '.agents', 'skills', 'gstack-review');
    if (!fs.existsSync(path.join(codexReview, 'SKILL.md'))) return; // gitignored artifact, absent in fresh checkouts
    const out = extractSkillSections(codexReview, CODEX_REVIEW_E2E_SECTIONS);
    expect(out).toContain('## Step 4: Critical pass (core review)');
    expect(out).not.toContain('## Telemetry (run last)');
  });
});

describe('real-skill pins: body/head extraction used by E2E fixtures', () => {
  // scrape/skillify/context-*: skill-e2e-skillify + skill-e2e-context-skills.
  // review/plan-eng-review/ship: skill-e2e-coverage-audit + skill-e2e-triage.
  const BODY_EXTRACTED_SKILLS = [
    'scrape', 'skillify', 'context-save', 'context-restore',
    'review', 'plan-eng-review', 'ship',
  ];

  for (const skill of BODY_EXTRACTED_SKILLS) {
    test(`extractSkillBody(${skill}) drops the shared preamble, keeps the flow`, () => {
      const out = extractSkillBody(path.join(ROOT, skill));
      expect(out).not.toContain('## Preamble (run first)');
      expect(out).not.toContain('## Preamble (after scope gate)');
      expect(out).not.toContain('## Telemetry (run last)');
      const full = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
      expect(out.length).toBeLessThan(full.length * 0.75);
      expect(out.length).toBeGreaterThan(500);
    });
  }

  test('body extraction keeps the sections the skillify/context E2E tests assert on', () => {
    expect(extractSkillBody(path.join(ROOT, 'skillify'))).toContain('## Step 1 — Provenance guard (D1)');
    expect(extractSkillBody(path.join(ROOT, 'scrape'))).toContain('## Step 2 — Refuse mutating intents');
    expect(extractSkillBody(path.join(ROOT, 'context-save'))).toContain('## List flow');
    expect(extractSkillBody(path.join(ROOT, 'context-restore'))).toContain('## If no saved contexts exist');
  });

  test('the scoped Eng render retains its original scope gate before the workflow', () => {
    const file = path.join(ROOT, 'plan-eng-review', 'SKILL.md');
    const full = fs.readFileSync(file, 'utf-8');
    const scopeStart = full.indexOf('## Scope gate');
    const preamble = full.indexOf('## Preamble (after scope gate)');
    expect(scopeStart).toBeGreaterThan(0); expect(preamble).toBeGreaterThan(scopeStart);
    expect(extractSkillBody(file)).toContain(full.slice(scopeStart, preamble).trimEnd());
  });

  // The union of skills installed by the routing + opus-47 discovery fixtures.
  const HEAD_EXTRACTED_SKILLS = [
    '', 'qa', 'qa-only', 'ship', 'review', 'plan-ceo-review', 'plan-eng-review',
    'plan-design-review', 'design-review', 'design-consultation', 'retro',
    'document-release', 'investigate', 'office-hours', 'browse',
    'setup-browser-cookies', 'gstack-upgrade', 'humanizer',
  ];

  test('extractSkillHead works for every discovery-fixture skill', () => {
    for (const skill of HEAD_EXTRACTED_SKILLS) {
      const src = path.join(ROOT, skill, 'SKILL.md');
      if (!fs.existsSync(src)) continue; // mirrors the fixtures' existsSync guard
      const out = extractSkillHead(src);
      expect(out.startsWith('---\n')).toBe(true);
      expect(out).toContain('description:');
      // Frontmatter length varies (allowed-tools + triggers); the invariant
      // is "frontmatter + 30 body lines + marker", never the full body.
      const fullLines = fs.readFileSync(src, 'utf-8').split('\n').length;
      expect(out.split('\n').length).toBeLessThan(Math.min(150, fullLines));
    }
  });
});