software-mansion__argent/scripts/extract-tools.mjs

#!/usr/bin/env node
/**
 * Extracts tool id + description from all ToolDefinition objects in
 * packages/tool-server/src/tools/**\/*.ts and outputs MCP tools/list JSON
 * suitable for `spidershield scan --tools-json`.
 *
 * Must stay dependency-free (node: builtins only): tool-description-quality.yml
 * runs it on a bare checkout without `npm ci`. extract-tools.test.mjs reparses
 * the same tree with the real TypeScript parser and requires an exact match, so
 * a lexing gap on a shape that reaches the tree fails CI.
 *
 * Usage:
 *   node scripts/extract-tools.mjs > tools.json
 */

import { readFileSync, readdirSync, statSync } from "node:fs";
import { join, extname } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { dirname } from "node:path";
import { argv as processArgv } from "node:process";

const __filename = fileURLToPath(import.meta.url);
const __dir = dirname(__filename);
const toolsRoot = join(__dir, "..", "packages", "tool-server", "src", "tools");

// Single-character escapes with a non-identity meaning; unescapeJsString
// handles the rest (\xNN, \uNNNN, \u{...}, line continuations, and identity
// escapes, where the backslash simply drops).
const UNESCAPE_MAP = { n: "\n", r: "\r", t: "\t", v: "\v", b: "\b", f: "\f", 0: "\0" };

// Sticky so findOwnDescriptionValue can anchor it per candidate position
// instead of allocating a substring per character.
const DESCRIPTION_KEY = /description:\s*/y;

// Sticky, anchored by findIdLiteralsInCode at a known code-context position;
// the backreference forces a matching closing delimiter. A template id is
// static only without interpolation, so the caller rejects a captured `${`.
const ID_LITERAL = /id:\s*(["'`])([^"'`]+)\1/y;

// Characters after which a `/` starts a regex literal rather than division.
// `+`, `-` and `!` are absent on purpose: isRegexPosition disambiguates their
// postfix forms (`++`, `--`, non-null `!`) before consulting this set, so
// entries here could never be reached.
const REGEX_PREV_CHARS = new Set("(,=:[&|?{};*%<>~^");

// Keywords a regex literal can directly follow even though they end in an
// identifier character (`return /.../` occurs in helpers that share a file
// with tool definitions).
const REGEX_PREV_KEYWORDS = new Set([
  "return",
  "typeof",
  "case",
  "in",
  "of",
  "delete",
  "void",
  "throw",
  "new",
  "do",
  "else",
  "yield",
  "await",
  "instanceof",
]);

function walk(dir) {
  const entries = [];
  for (const name of readdirSync(dir)) {
    const full = join(dir, name);
    const stat = statSync(full);
    if (stat.isDirectory()) {
      entries.push(...walk(full));
    } else if (extname(name) === ".ts" && !name.endsWith(".d.ts")) {
      entries.push(full);
    }
  }
  return entries;
}

// Lexical skip helpers. Every scanner routes candidate characters through the
// single `skipNonCode` dispatcher below, so their notion of "code context"
// cannot drift apart.

/** @returns {number} index just past the newline ending a `//` comment */
function skipLineComment(src, i) {
  const nl = src.indexOf("\n", i + 2);
  return nl === -1 ? src.length : nl + 1;
}

/** @returns {number} index just past the `*` + `/` ending a block comment */
function skipBlockComment(src, i) {
  const end = src.indexOf("*/", i + 2);
  return end === -1 ? src.length : end + 2;
}

/** src[i] is `'` or `"`. @returns {number} index just past the closing delimiter */
function skipStringLiteral(src, i) {
  const delim = src[i];
  for (let j = i + 1; j < src.length; j++) {
    if (src[j] === "\\") j++;
    else if (src[j] === delim) return j + 1;
  }
  return src.length;
}

/**
 * src[i] is a backtick. `${...}` interpolations are skipped too: their strings,
 * nested templates, comments and regexes must be tracked, or a nested
 * template's backtick "closes" the outer one and everything after is mis-lexed.
 *
 * @returns {number} index just past the closing backtick
 */
function skipTemplateLiteral(src, i) {
  for (let j = i + 1; j < src.length; j++) {
    const ch = src[j];
    if (ch === "\\") j++;
    else if (ch === "$" && src[j + 1] === "{") j = skipInterpolation(src, j + 1) - 1;
    else if (ch === "`") return j + 1;
  }
  return src.length;
}

/**
 * Skips a comment, string/template literal or regex literal starting at src[i]
 * whole, returning the scan state just past it; null when src[i] is code.
 *
 * @param {string} src
 * @param {number} i   index of the candidate character
 * @param {string | null} prevSig  last significant code char, or null
 * @param {number} prevSigIdx      index of prevSig in src (-1 when null)
 * @returns {{ i: number, prevSig: string | null, prevSigIdx: number } | null}
 *   the returned `i` sits ON the last skipped character, ready for the
 *   caller's loop increment; comments leave prevSig untouched (invisible to
 *   code context), a skipped literal becomes the new prevSig.
 */
function skipNonCode(src, i, prevSig, prevSigIdx) {
  const ch = src[i];
  if (ch === "/" && src[i + 1] === "/") {
    return { i: skipLineComment(src, i) - 1, prevSig, prevSigIdx };
  }
  if (ch === "/" && src[i + 1] === "*") {
    return { i: skipBlockComment(src, i) - 1, prevSig, prevSigIdx };
  }
  if (ch === "/") {
    const end = skipRegexLiteral(src, i, prevSig, prevSigIdx);
    // a value just ended; a following `/` is division
    if (end > i) return { i: end - 1, prevSig: ")", prevSigIdx: end - 1 };
    return null;
  }
  if (ch === '"' || ch === "'") {
    const end = skipStringLiteral(src, i) - 1;
    return { i: end, prevSig: ch, prevSigIdx: end };
  }
  if (ch === "`") {
    const end = skipTemplateLiteral(src, i) - 1;
    return { i: end, prevSig: "`", prevSigIdx: end };
  }
  return null;
}

/** src[i] is the `{` of a `${`. @returns {number} index just past the matching `}` */
function skipInterpolation(src, i) {
  let depth = 1;
  let prevSig = null;
  let prevSigIdx = -1;
  for (let j = i + 1; j < src.length; j++) {
    const ch = src[j];
    const skipped = skipNonCode(src, j, prevSig, prevSigIdx);
    if (skipped) {
      ({ i: j, prevSig, prevSigIdx } = skipped);
      continue;
    }
    if (ch === "{") {
      depth++;
    } else if (ch === "}") {
      if (--depth === 0) return j + 1;
    }
    if (!/\s/.test(ch)) {
      prevSig = ch;
      prevSigIdx = j;
    }
  }
  return src.length;
}

/**
 * True when a `/` may start a regex literal here: after an operator/separator,
 * after a regex-permitting keyword, or at the start of input. After an
 * identifier, number, closing paren/bracket or string, `/` is division.
 *
 * @param {string} src
 * @param {string | null} prevSig  last significant code char, or null
 * @param {number} prevSigIdx      index of prevSig in src (-1 when null)
 */
function isRegexPosition(src, prevSig, prevSigIdx) {
  if (prevSig === null) return true;
  if (prevSig === "+" || prevSig === "-") {
    // Postfix `++`/`--` ends a value, so a following `/` is division; a lone
    // `+`/`-` is a binary/unary operator, after which `/` starts a regex.
    return src[prevSigIdx - 1] !== prevSig;
  }
  if (prevSig === "!") {
    // `!` is either a prefix logical-NOT (`!/re/.test(x)`) or a TS postfix
    // non-null assertion (`x! / y`). Both leave the context the same as at the
    // position just before the `!`, so recurse on the char preceding it.
    // (`!=`/`!==` never reach here: their `=` is the last significant char.)
    let before = prevSigIdx - 1;
    while (before >= 0 && /\s/.test(src[before])) before--;
    return isRegexPosition(src, before < 0 ? null : src[before], before);
  }
  if (REGEX_PREV_CHARS.has(prevSig)) return true;
  if (/[A-Za-z0-9_$]/.test(prevSig)) {
    let start = prevSigIdx;
    while (start > 0 && /[A-Za-z0-9_$]/.test(src[start - 1])) start--;
    if (!REGEX_PREV_KEYWORDS.has(src.slice(start, prevSigIdx + 1))) return false;
    // A keyword-NAMED property access (`counts.in`, `obj?.new`) is a value,
    // not a keyword: the `/` after it is division.
    let before = start - 1;
    while (before >= 0 && /\s/.test(src[before])) before--;
    return src[before] !== ".";
  }
  return false;
}

/**
 * Returns the index just past a regex literal's closing `/` and flags, or i
 * when the `/` is division instead. `[...]` classes (where `/` does not
 * terminate), `\` escapes and the single-line rule are handled.
 *
 * @param {string} src
 * @param {number} i        index of the opening `/`
 * @param {string | null} prevSig  last significant code char before i, or null
 * @param {number} prevSigIdx      index of prevSig in src (-1 when null)
 * @returns {number}
 */
function skipRegexLiteral(src, i, prevSig, prevSigIdx) {
  if (!isRegexPosition(src, prevSig, prevSigIdx)) return i;
  let inClass = false;
  for (let j = i + 1; j < src.length; j++) {
    const ch = src[j];
    if (ch === "\\") {
      j++;
    } else if (ch === "\n") {
      return i; // regex literals are single-line; this `/` was not one
    } else if (inClass) {
      if (ch === "]") inClass = false;
    } else if (ch === "[") {
      inClass = true;
    } else if (ch === "/") {
      let end = j + 1;
      while (end < src.length && /[a-zA-Z]/.test(src[end])) end++; // flags
      return end;
    }
  }
  return i;
}

/**
 * Render a captured string/template-literal body as its runtime (cooked) text.
 * An out-of-range \u{...} is left as source text - such a file would not
 * compile anyway.
 *
 * @param {string} raw
 * @returns {string}
 */
function unescapeJsString(raw) {
  return raw.replace(
    /\\(?:u\{([0-9A-Fa-f]+)\}|u([0-9A-Fa-f]{4})|x([0-9A-Fa-f]{2})|(\r\n|[\s\S]))/g,
    (whole, uBrace, u4, x2, single) => {
      if (uBrace !== undefined) {
        const cp = parseInt(uBrace, 16);
        return cp <= 0x10ffff ? String.fromCodePoint(cp) : whole;
      }
      if (u4 !== undefined) return String.fromCharCode(parseInt(u4, 16));
      if (x2 !== undefined) return String.fromCharCode(parseInt(x2, 16));
      // Line continuation: backslash + any LineTerminatorSequence vanishes.
      if (single === "\r\n" || /^[\n\r\u2028\u2029]$/.test(single)) return "";
      return UNESCAPE_MAP[single] ?? single;
    }
  );
}

/**
 * From the source just after a tool's `id: "..."`, return the text of that
 * tool's own `description:` value (from its opening delimiter), or null.
 *
 * Scans forward tracking object-literal brace depth, skipping comments and
 * string/template/regex literals whole so nothing inside them can shift the
 * depth or match as a `description:` token:
 *   - the first `description:` at depth 0 is the sibling one; a balanced nested
 *     object between the two (e.g. `interaction: { ... }`) does not hide it, and
 *   - a `}` dropping the depth below 0 first means the `id:` is a key nested in
 *     another object (e.g. `defaultPayload: { id: "example" }`), not a tool.
 *
 * That scoping is what stops a nested `id:` key from being emitted as a spurious
 * tool or displacing the real one in the downstream `spidershield` security scan.
 *
 * @param {string} afterId  source text starting just after the `id: "..."` match
 * @returns {string | null}
 */
function findOwnDescriptionValue(afterId) {
  let depth = 0;
  let prevSig = null; // last significant code char (regex-vs-division context)
  let prevSigIdx = -1;
  for (let i = 0; i < afterId.length; i++) {
    const ch = afterId[i];
    const skipped = skipNonCode(afterId, i, prevSig, prevSigIdx);
    if (skipped) {
      ({ i, prevSig, prevSigIdx } = skipped);
      continue;
    }
    if (ch === "{") {
      depth++;
    } else if (ch === "}") {
      if (--depth < 0) return null; // id's own object closed - not a tool
    } else if (
      depth === 0 &&
      ch === "d" &&
      // word boundary before the token so `...Xdescription:` doesn't match
      !/[A-Za-z0-9_$]/.test(afterId[i - 1] ?? "")
    ) {
      DESCRIPTION_KEY.lastIndex = i;
      if (DESCRIPTION_KEY.test(afterId)) return afterId.slice(DESCRIPTION_KEY.lastIndex);
    }
    if (!/\s/.test(ch)) {
      prevSig = ch;
      prevSigIdx = i;
    }
  }
  return null;
}

/**
 * Find every static `id:` string/template literal that occurs in CODE - never
 * one inside a string, template literal, comment, or regex literal.
 *
 * A plain global regex also matched `id:` inside comments and description text;
 * such a candidate borrows the enclosing object's real `description:` and, via
 * first-wins dedup, can displace the real tool in the security scan. Lexing to
 * code context first removes that class, and skips keys like `$id:` too.
 *
 * A template-literal id counts only without `${...}` interpolation; an
 * interpolated id is dynamic and out of scope for this static extractor.
 *
 * @param {string} src
 * @returns {{ id: string, end: number }[]}  end = index just past the matched literal
 */
function findIdLiteralsInCode(src) {
  const out = [];
  let prevSig = null; // last significant code char (regex-vs-division context)
  let prevSigIdx = -1;
  for (let i = 0; i < src.length; i++) {
    const ch = src[i];
    const skipped = skipNonCode(src, i, prevSig, prevSigIdx);
    if (skipped) {
      ({ i, prevSig, prevSigIdx } = skipped);
      continue;
    }
    // Word boundary before the token so `grid:`, `$id:`, `androidId:` etc. are
    // not mistaken for a tool id.
    if (ch === "i" && !/[A-Za-z0-9_$]/.test(src[i - 1] ?? "")) {
      ID_LITERAL.lastIndex = i;
      const m = ID_LITERAL.exec(src);
      if (m) {
        // An interpolated template id is dynamic, not a static tool id.
        if (!(m[1] === "`" && m[2].includes("${"))) {
          // Cook the id: "esc\u002Dtool" names "esc-tool" at runtime, and the
          // scan must see the runtime name.
          out.push({ id: unescapeJsString(m[2]), end: ID_LITERAL.lastIndex });
        }
        i = ID_LITERAL.lastIndex - 1; // resume just past the value (loop's i++ advances)
        prevSig = m[1]; // the id literal's closing delimiter
        prevSigIdx = i;
        continue;
      }
    }
    if (!/\s/.test(ch)) {
      prevSig = ch;
      prevSigIdx = i;
    }
  }
  return out;
}

/**
 * True when nothing extends a captured description literal: after whitespace
 * and comments the property must end (`,` or `}`). Any other suffix (`+`
 * concatenation, a method call, `as const`, ...) means the rendered text
 * differs from the captured literal, so the caller must warn-skip.
 *
 * @param {string} rest  source text starting just past the literal's closing delimiter
 * @returns {boolean}
 */
function literalIsWholeValue(rest) {
  for (let i = 0; i < rest.length; i++) {
    const ch = rest[i];
    if (/\s/.test(ch)) continue;
    if (ch === "/" && rest[i + 1] === "/") {
      const nl = rest.indexOf("\n", i + 2);
      if (nl === -1) return false; // line comment to EOF - property never ends
      i = nl;
      continue;
    }
    if (ch === "/" && rest[i + 1] === "*") {
      const end = rest.indexOf("*/", i + 2);
      if (end === -1) return false; // unterminated block comment
      i = end + 1;
      continue;
    }
    return ch === "," || ch === "}";
  }
  return false; // EOF right after the literal - not a well-formed property
}

/**
 * Extract `{ name, description }` for every string-literal `id:` tool definition
 * in a single source string. Pure (no I/O) so the parsing rules can be
 * unit-tested against crafted fixtures.
 *
 * @param {string} src       source text
 * @param {string} filePath  label used in warnings
 * @returns {{name: string, description: string}[]}
 */
export function extractToolsFromSource(src, filePath = "<source>") {
  const tools = [];

  for (const { id, end } of findIdLiteralsInCode(src)) {
    // Null means this `id:` had no sibling `description:` at its own object
    // level - a nested object key, not a tool (see findOwnDescriptionValue).
    const value = findOwnDescriptionValue(src.slice(end));

    let description = null;
    if (value !== null) {
      const delim = value[0];
      // `\\[\s\S]` (not `\\.`) lets an escaped line terminator - a line
      // continuation - stay part of one legal literal.
      const valueMatch =
        delim === "`"
          ? value.match(/^`((?:\\[\s\S]|[^`\\])*)`/)
          : delim === '"'
            ? value.match(/^"((?:[^"\\]|\\[\s\S])*)"/)
            : delim === "'"
              ? value.match(/^'((?:[^'\\]|\\[\s\S])*)'/)
              : null;
      if (valueMatch) {
        // Leaving description null here falls through to the warn-and-skip
        // below rather than emitting wrong text into the security scan. The
        // interpolation probe replaces escape pairs with a placeholder (never
        // deletes them) so `$` + escape + `{` cannot glue into a fake `${`, and
        // an escaped `\${` or `$\{` stays the text it renders as.
        const rest = value.slice(valueMatch[0].length);
        const hasInterpolation =
          delim === "`" && /\$\{/.test(valueMatch[1].replace(/\\[\s\S]/g, " "));
        if (literalIsWholeValue(rest) && !hasInterpolation) {
          // Template literals cook `\r\n` / `\r` to `\n` per spec (relevant on a
          // CRLF checkout) before escape semantics apply.
          const text = delim === "`" ? valueMatch[1].replace(/\r\n?/g, "\n") : valueMatch[1];
          description = unescapeJsString(text).trim();
        }
      }
    }

    if (description !== null) {
      tools.push({
        name: id,
        description,
      });
    } else if (value !== null) {
      // A real tool whose description is not statically capturable. Warn on
      // stderr rather than dropping it silently (such a drop once hid
      // run-sequence from the scan); stdout must stay valid JSON.
      console.error(
        `extract-tools: WARNING: tool "${id}" in ${filePath} has a description that is not a single string/template literal (a concatenation, an interpolation, a method/operator suffix, or a non-literal value); skipping.`
      );
    } else {
      // Usually a nested object key, correctly not a tool. But a tool with no
      // `description` (optional in ToolDefinition), or one whose `description:`
      // is written BEFORE its `id:` (this scan is forward-only), lands here too
      // and would be dropped from the scan. The cases are not locally
      // distinguishable, so warn instead of dropping silently.
      console.error(
        `extract-tools: WARNING: id "${id}" in ${filePath} has no sibling description at its object scope; skipping (nested object key, a tool missing its description, or a description written before the id).`
      );
    }
  }

  return tools;
}

function extractFromFile(filePath) {
  return extractToolsFromSource(readFileSync(filePath, "utf8"), filePath);
}

/**
 * Deduplicate tools by name (first occurrence wins). A collision means a later
 * tool's description never reaches the security scan, so warn rather than let
 * it bypass silently. Exported so the warning path is unit-testable.
 *
 * @param {{name: string, description: string}[]} tools
 * @returns {{name: string, description: string}[]}
 */
export function dedupeToolsById(tools) {
  const seen = new Set();
  return tools.filter((t) => {
    if (seen.has(t.name)) {
      console.error(
        `extract-tools: WARNING: duplicate tool id "${t.name}"; keeping the first occurrence and skipping the rest.`
      );
      return false;
    }
    seen.add(t.name);
    return true;
  });
}

function extractAllTools() {
  const files = walk(toolsRoot);
  const tools = [];
  for (const f of files) {
    tools.push(...extractFromFile(f));
  }
  return dedupeToolsById(tools);
}

function main() {
  // MCP tools/list format
  console.log(JSON.stringify({ tools: extractAllTools() }, null, 2));
}

// Run only when invoked directly, not when imported by the test.
if (processArgv[1] && import.meta.url === pathToFileURL(processArgv[1]).href) {
  main();
}