callstack__agent-device/scripts/__tests__/xctest-run-summary.test.ts

// A whole-bundle XCTest lane's reporter is also its liveness check, and both halves only
// ever execute on a macOS runner — so they are proven here instead.

import { describe, expect, test } from 'vitest';
import {
  livenessFailure,
  MAX_FAILURE_TEXT,
  MAX_LISTED_FAILURES,
  renderSummary,
  type ResultSummary,
} from '../xctest-run-summary.ts';

const TITLE = 'iOS runner full XCTest suite';

function failures(count: number, text = 'assertion failed') {
  return Array.from({ length: count }, (_, index) => ({
    testName: `testCase${index}()`,
    failureText: text,
  }));
}

describe('the liveness check', () => {
  test('a run that executed no tests fails, because xcodebuild calls that success', () => {
    // The lane's whole reason to assert: a build without the unit-test compile flag, an
    // empty test plan, or a `-skip-testing:` entry that swallowed the suite all exit 0.
    const failure = livenessFailure({ result: 'Passed', totalTestCount: 0 }, 153);
    expect(failure).toContain('executed no tests');
    expect(failure).toContain('AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS');
    expect(failure).toContain('-skip-testing');
  });

  test('a summary missing the count entirely is treated as no tests, not as unknown', () => {
    expect(livenessFailure({}, 153)).not.toBeNull();
  });

  test('a run that executed exactly the source-derived reach passes, red or green', () => {
    expect(
      livenessFailure({ totalTestCount: 153, result: 'Failed', failedTests: 9 }, 153),
    ).toBeNull();
    expect(livenessFailure({ totalTestCount: 1, result: 'Passed' }, 1)).toBeNull();
  });

  test('a run that executed fewer or more tests than the source reaches fails', () => {
    // The quieter failure "at least one ran" never sees: a `#if` guard that compiles a file
    // of tests out of the lane's platform, or a flag that matched more than it names.
    const fewer = livenessFailure({ totalTestCount: 120, result: 'Passed' }, 133);
    expect(fewer).toContain('executed 120 test(s) but the source says it reaches 133');
    expect(fewer).toContain('check:xctest-selection');
    expect(livenessFailure({ totalTestCount: 134, result: 'Passed' }, 133)).not.toBeNull();
  });

  test('a lane the source says reaches nothing can never be credible', () => {
    expect(livenessFailure({ totalTestCount: 0 }, 0)).toContain('derives no reachable tests');
    expect(livenessFailure({ totalTestCount: 5 }, 0)).toContain('derives no reachable tests');
  });
});

describe('the job summary', () => {
  const green: ResultSummary = {
    result: 'Passed',
    totalTestCount: 153,
    passedTests: 153,
    failedTests: 0,
    skippedTests: 0,
    expectedFailures: 0,
    startTime: 1000,
    finishTime: 1450,
  };

  test('leads with the headline a reader needs', () => {
    const rendered = renderSummary(green, TITLE, 153);
    expect(rendered).toContain('- result: **Passed**');
    expect(rendered).toContain(
      '- executed: **153** of the 153 the source reaches on this lane (passed 153, failed 0, skipped 0',
    );
    expect(rendered).toContain('- duration: 450s');
    expect(rendered).not.toContain('#### Failures');
  });

  test('reports an unknown duration rather than NaN when the times are absent', () => {
    expect(renderSummary({ totalTestCount: 1 }, TITLE, 1)).toContain('- duration: unknown');
  });

  test('lists failures and names each one', () => {
    const rendered = renderSummary(
      { ...green, result: 'Failed', testFailures: failures(3) },
      TITLE,
      153,
    );
    expect(rendered).toContain('#### Failures');
    expect(rendered).toContain('`testCase0()`');
    expect(rendered).toContain('assertion failed');
  });

  test('caps the list, and says how many it dropped', () => {
    // The night this lane matters most is the night the failure list is longest, and the
    // job summary is capped at 1 MiB — losing the headline to the tail would be the worst
    // possible trade.
    const rendered = renderSummary(
      { ...green, result: 'Failed', testFailures: failures(MAX_LISTED_FAILURES + 7) },
      TITLE,
      153,
    );
    const listed = rendered.split('\n').filter((line) => line.startsWith('- `test')).length;
    expect(listed).toBe(MAX_LISTED_FAILURES);
    expect(rendered).toContain('…and 7 more');
  });

  test('truncates one enormous failure message instead of letting it dominate', () => {
    const rendered = renderSummary(
      { ...green, result: 'Failed', testFailures: failures(1, 'x'.repeat(5000)) },
      TITLE,
      153,
    );
    expect(rendered).toContain('x'.repeat(MAX_FAILURE_TEXT));
    expect(rendered).not.toContain('x'.repeat(MAX_FAILURE_TEXT + 1));
  });

  test('flattens newlines so a multi-line stack cannot forge markdown structure', () => {
    const rendered = renderSummary(
      {
        ...green,
        result: 'Failed',
        testFailures: [{ testName: 'testX()', failureText: 'line one\n#### Injected\nline two' }],
      },
      TITLE,
      153,
    );
    // Markdown headings only bind at the start of a line, so flattening is what disarms
    // the injection: the text survives verbatim, but it can no longer open a section.
    expect(rendered.split('\n').filter((line) => line.startsWith('#'))).toEqual([
      `### ${TITLE}`,
      '#### Failures',
    ]);
    expect(rendered).toContain('line one #### Injected line two');
  });

  test('falls back to the identifier when a failure carries no test name', () => {
    const rendered = renderSummary(
      {
        ...green,
        testFailures: [{ testIdentifierString: 'RunnerTests/testY()', failureText: 'boom' }],
      },
      TITLE,
      153,
    );
    expect(rendered).toContain('`RunnerTests/testY()`');
  });
});