firecrawl__anydoc/node/test.mjs

// Smoke test: the bindings load and every entry point round-trips a fixture.
import assert from 'node:assert/strict'
import { execFile } from 'node:child_process'
import { mkdtemp, readFile, rm } from 'node:fs/promises'
import { createServer } from 'node:http'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { test } from 'node:test'
import { promisify } from 'node:util'

import {
  formatFromBytes,
  formatFromExtension,
  formatFromPath,
  toDocument,
  toMarkdown,
  toMarkdownBytes,
} from './anydoc.js'

const fixture = (name) => fileURLToPath(new URL(`../tests/fixtures/${name}`, import.meta.url))

const OUTLINE = fixture('docx/handmade-outline.docx')
const RICH = fixture('docx/handmade-rich.docx')
const CSV = fixture('csv/sheet.csv')
const ENCRYPTED = fixture('malformed/encrypted--errors.odt')
const MIXED = fixture('pdf/handmade-mixed.pdf')

test('toMarkdown detects the format from the file content', async () => {
  const markdown = await toMarkdown(OUTLINE)
  assert.match(markdown, /^# /m)
})

test('toMarkdownBytes converts in memory', async () => {
  const markdown = await toMarkdownBytes(await readFile(RICH), 'docx')
  assert.match(markdown, /\| Quarter \| Widgets \|/)
})

test('toMarkdownBytes detects the format when none is named', async () => {
  const markdown = await toMarkdownBytes(await readFile(RICH))
  assert.match(markdown, /\| Quarter \| Widgets \|/)
  // CSV carries no signature, so it has to be named.
  await assert.rejects(toMarkdownBytes(await readFile(CSV)), /unrecognized file content/)
  assert.match(await toMarkdownBytes(await readFile(CSV), 'csv'), /\| --- \|/)
})

test('toDocument exposes the document model', async () => {
  const document = await toDocument(await readFile(OUTLINE), 'docx')
  const heading = document.blocks.find((block) => block.kind === 'heading')
  assert.ok(heading.level >= 1 && heading.level <= 6)
  assert.equal(typeof heading.content[0].text, 'string')
  assert.equal(heading.content[0].kind, 'text')
  assert.equal(typeof heading.content[0].style.bold, 'boolean')
})

test('toDocument carries embedded assets as buffers', async () => {
  const document = await toDocument(await readFile(RICH), 'docx')
  const image = document.assets.find((asset) => asset.mediaType === 'image/png')
  assert.ok(Buffer.isBuffer(image.data))
  assert.ok(image.data.length > 0)
  assert.equal(image.id, document.assets.indexOf(image))
})

test('format detection reads content, extension, and path', async () => {
  assert.equal(formatFromBytes(await readFile(RICH)), 'docx')
  // CSV carries no signature: only the extension names it.
  assert.equal(formatFromBytes(await readFile(CSV)), null)
  assert.equal(formatFromExtension('.pptm'), 'pptx')
  assert.equal(formatFromExtension('xls'), 'xlsx')
  assert.equal(formatFromPath('/tmp/report.odt'), 'odt')
  assert.equal(formatFromPath('/tmp/report.unknown'), null)
})

// `code` is what callers branch on, so every kind of failure is pinned here.
test('conversion errors reject with a coded Error', async () => {
  const rejects = (promise, code, message) =>
    assert.rejects(promise, (error) => {
      assert.equal(error.code, code)
      assert.match(error.message, message)
      return true
    })

  await rejects(toMarkdownBytes(Buffer.from('not a document'), 'docx'), 'malformed', /malformed/)
  await rejects(toMarkdownBytes(await readFile(CSV)), 'unsupported', /unrecognized file content/)
  await rejects(toMarkdownBytes(await readFile(ENCRYPTED), 'odt'), 'encrypted', /encrypted/)
  await rejects(toDocument(await readFile(ENCRYPTED), 'odt'), 'encrypted', /encrypted/)
  await rejects(toMarkdown(fixture('docx/no-such-file.docx')), 'io', /io error/)
  await rejects(toMarkdown(MIXED), 'needsOcr', /page 2 of 2 needs OCR/)
})

test('a pdf with scanned pages rejects naming them instead of dropping them', async () => {
  await assert.rejects(toMarkdown(MIXED), (error) => {
    assert.deepEqual([error.pages, error.pageCount], [[2], 2])
    return true
  })
})

test('the package exports everything the native binding does', async () => {
  const [wrapper, native] = await Promise.all([import('./anydoc.js'), import('./index.js')])
  assert.deepEqual(Object.keys(wrapper).sort(), Object.keys(native).sort())
})

// A stand-in for api.firecrawl.dev that answers every request with `reply`
// and records each hit as [path, whether a PDF came with it]. Runs `run`
// keyless, whatever the environment.
async function withHostedStub(reply, run) {
  const hits = []
  const server = createServer((request, response) => {
    const chunks = []
    request.on('data', (chunk) => chunks.push(chunk))
    request.on('end', () => {
      hits.push([request.url, Buffer.concat(chunks).includes('%PDF-')])
      response.writeHead(reply.status, { 'content-type': 'application/json' })
      response.end(JSON.stringify(reply.body))
    })
  })
  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
  const url = `http://127.0.0.1:${server.address().port}`
  const saved = { FIRECRAWL_API_URL: process.env.FIRECRAWL_API_URL, FIRECRAWL_API_KEY: process.env.FIRECRAWL_API_KEY }
  process.env.FIRECRAWL_API_URL = url
  delete process.env.FIRECRAWL_API_KEY
  try {
    await run(hits, url)
  } finally {
    for (const [name, value] of Object.entries(saved)) {
      if (value === undefined) delete process.env[name]
      else process.env[name] = value
    }
    server.close()
  }
}

const HOSTED = { status: 200, body: { success: true, data: { markdown: '# Read by the hosted parser\n' } } }

test("ocr: 'hosted' sends a pdf with scanned pages to Firecrawl Parse, and nothing else", async () => {
  await withHostedStub(HOSTED, async (hits) => {
    assert.equal(await toMarkdown(MIXED, { ocr: 'hosted' }), HOSTED.body.data.markdown)
    assert.deepEqual(hits, [['/v2/parse', true]])
    assert.match(await toMarkdown(OUTLINE, { ocr: 'hosted' }), /^# /m)
    assert.deepEqual(hits, [['/v2/parse', true]])
  })
})

test('the keyless limit says to set an api key', async () => {
  const limited = { status: 429, body: { success: false, error: 'Rate limit exceeded' } }
  await withHostedStub(limited, async () => {
    await assert.rejects(toMarkdown(MIXED, { ocr: 'hosted' }), (error) => {
      assert.equal(error.code, 'hosted')
      assert.match(error.message, /set FIRECRAWL_API_KEY/)
      return true
    })
  })
})

const CLI = fileURLToPath(new URL('./cli.js', import.meta.url))

// execFile rejects on a non-zero exit; the error carries code/stdout/stderr.
const runCli = (args, options) =>
  promisify(execFile)(process.execPath, [CLI, ...args], options).catch((error) => error)

test('cli converts a file to stdout', async () => {
  const { code, stdout, stderr } = await runCli([OUTLINE])
  assert.equal(code, undefined)
  assert.match(stdout, /^# /m)
  assert.equal(stderr, '')
})

test('cli writes to --output instead of stdout', async () => {
  const dir = await mkdtemp(join(tmpdir(), 'anydoc-cli-'))
  try {
    const out = join(dir, 'outline.md')
    const { code, stdout } = await runCli([OUTLINE, '-o', out])
    assert.equal(code, undefined)
    assert.equal(stdout, '')
    assert.match(await readFile(out, 'utf8'), /^# /m)
  } finally {
    await rm(dir, { recursive: true, force: true })
  }
})

test('cli reads stdin with an explicit format', async () => {
  const child = promisify(execFile)(process.execPath, [CLI, '-', '--format', 'csv'])
  child.child.stdin.end(await readFile(CSV))
  const { stdout } = await child
  assert.match(stdout, /\| --- \|/)
})

test('cli exits 1 when the document cannot be converted', async () => {
  const { code, stderr } = await runCli(['no-such-file.docx'])
  assert.equal(code, 1)
  assert.match(stderr, /^anydoc: /)
})

test('cli exits 3 when pages need OCR', async () => {
  const { code, stderr } = await runCli([MIXED])
  assert.equal(code, 3)
  assert.match(stderr, /^anydoc: page 2 of 2 needs OCR/)
})

test('cli --ocr hosted converts a pdf with scanned pages through Firecrawl Parse', async () => {
  await withHostedStub(HOSTED, async (hits, url) => {
    const { code, stdout } = await runCli([MIXED, '--ocr', 'hosted', '--api-url', url])
    assert.equal(code, undefined)
    assert.equal(stdout, HOSTED.body.data.markdown)
  })
})

test('cli exits 2 on usage errors', async () => {
  for (const args of [[], ['--frmat', 'csv', CSV], ['--format', 'nope', CSV], [OUTLINE, RICH], ['--ocr', 'cloud', MIXED]]) {
    const { code, stderr } = await runCli(args)
    assert.equal(code, 2)
    assert.match(stderr, /^anydoc: /)
  }
})

test('cli help and version go to stdout and exit 0', async () => {
  const help = await runCli(['--help'])
  assert.equal(help.code, undefined)
  assert.match(help.stdout, /Exit codes:/)
  const version = await runCli(['--version'])
  assert.match(version.stdout.trim(), /^\d+\.\d+\.\d+/)
})