oxc-project__oxc/napi/parser/src-js/raw-transfer/eager.js

import { createRequire } from "node:module";
import { TOKENS_OFFSET_POS_32, TOKENS_LEN_POS_32 } from "../generated/constants.js";
import { isJsAst, parseAsyncRawImpl, parseSyncRawImpl, returnBufferToCache } from "./common.js";

const require = createRequire(import.meta.url);

/**
 * Parse JS/TS source synchronously on current thread, using raw transfer to speed up deserialization.
 *
 * @param {string} filename - Filename
 * @param {string} sourceText - Source text of file
 * @param {Object} options - Parsing options
 * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`
 */
export function parseSyncRaw(filename, sourceText, options) {
  return parseSyncRawImpl(filename, sourceText, options, deserialize);
}

/**
 * Parse JS/TS source asynchronously, using raw transfer to speed up deserialization.
 *
 * Note that not all of the workload can happen on a separate thread.
 * Parsing on Rust side does happen in a separate thread, but deserialization of the AST to JS objects
 * has to happen on current thread. This synchronous deserialization work typically outweighs
 * the asynchronous parsing by a factor of around 3.
 *
 * i.e. the majority of the workload cannot be parallelized by using this method.
 *
 * Generally `parseSyncRaw` is preferable to use as it does not have the overhead of spawning a thread.
 * If you need to parallelize parsing multiple files, it is recommended to use worker threads.
 *
 * @param {string} filename - Filename
 * @param {string} sourceText - Source text of file
 * @param {Object} options - Parsing options
 * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`
 */
export function parse(filename, sourceText, options) {
  return parseAsyncRawImpl(filename, sourceText, options, deserialize);
}

// Deserializers are large files, so lazy-loaded.
// `deserialize` functions are stored in this array once loaded.
// Index into these arrays is `isJs * 1 + range * 2 + experimentalParent * 4`.
const deserializers = [null, null, null, null, null, null, null, null];
const deserializerNames = [
  "ts",
  "js",
  "ts_range",
  "js_range",
  "ts_parent",
  "js_parent",
  "ts_range_parent",
  "js_range_parent",
];

/**
 * Deserialize whole AST from buffer.
 *
 * @param {Uint8Array} buffer - Buffer containing AST in raw form
 * @param {string} sourceText - Source for the file
 * @param {number} sourceStartPos - Position of first byte of source text in buffer
 * @param {number} sourceByteLen - Length of source text in UTF-8 bytes
 * @param {Object} options - Parsing options
 * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`
 */
function deserialize(buffer, sourceText, sourceStartPos, sourceByteLen, options) {
  const isJs = isJsAst(buffer),
    range = !!options.range,
    parent = !!options.experimentalParent;

  // Lazy load deserializer, and deserialize buffer to JS objects
  const deserializerIndex = +isJs | (+range << 1) | (+parent << 2);
  let deserializeThis = deserializers[deserializerIndex];
  if (deserializeThis === null) {
    deserializeThis = deserializers[deserializerIndex] = require(
      `../generated/deserialize/${deserializerNames[deserializerIndex]}.js`,
    ).deserialize;
  }

  const data = deserializeThis(buffer, sourceText, sourceStartPos, sourceByteLen);

  // Add a line comment for hashbang if JS.
  // Do not add comment if TS, to match `@typescript-eslint/parser`.
  // See https://github.com/oxc-project/oxc/blob/ea784f5f082e4c53c98afde9bf983afd0b95e44e/napi/parser/src/lib.rs#L106-L130
  if (isJs) {
    const { hashbang } = data.program;
    if (hashbang !== null) {
      data.comments.unshift(
        range
          ? {
              type: "Line",
              value: hashbang.value,
              start: hashbang.start,
              end: hashbang.end,
              range: hashbang.range,
            }
          : { type: "Line", value: hashbang.value, start: hashbang.start, end: hashbang.end },
      );
    }
  }

  // Deserialize tokens
  const tokens = options.experimentalTokens ? deserializeTokens(buffer, sourceText, isJs) : null;

  // Return buffer to cache, to be reused
  returnBufferToCache(buffer);

  // We cannot lazily deserialize in the getters, because the buffer might be re-used to parse
  // another file before the getter is called
  if (tokens !== null) {
    return {
      get program() {
        return data.program;
      },
      get module() {
        return data.module;
      },
      get comments() {
        return data.comments;
      },
      get tokens() {
        return tokens;
      },
      get errors() {
        return data.errors;
      },
    };
  }

  return {
    get program() {
      return data.program;
    },
    get module() {
      return data.module;
    },
    get comments() {
      return data.comments;
    },
    get errors() {
      return data.errors;
    },
  };
}

// `ESTreeKind` discriminants (set by Rust side)
const PRIVATE_IDENTIFIER_KIND = 2;
const REGEXP_KIND = 8;

// Indexed by `ESTreeKind` discriminant (matches `ESTreeKind` enum in `estree_kind.rs`)
const TOKEN_TYPES = [
  "Identifier",
  "Keyword",
  "PrivateIdentifier",
  "Punctuator",
  "Numeric",
  "String",
  "Boolean",
  "Null",
  "RegularExpression",
  "Template",
  "JSXText",
  "JSXIdentifier",
];

// Mask for active bits in `ESTreeKind` discriminants
const TOKEN_KIND_MASK = 15;

// Details of Rust `Token` type
const TOKEN_SIZE = 16;

/**
 * Deserialize tokens from buffer.
 * @param {Uint8Array} buffer - Buffer containing AST in raw form
 * @param {string} sourceText - Source for the file
 * @param {boolean} isJs - `true` if parsing in JS mode
 * @returns {Object[]} - Array of token objects
 */
function deserializeTokens(buffer, sourceText, isJs) {
  const { int32 } = buffer;

  let pos = int32[TOKENS_OFFSET_POS_32];
  const len = int32[TOKENS_LEN_POS_32];
  const endPos = pos + len * TOKEN_SIZE;

  const tokens = [];
  while (pos < endPos) {
    tokens.push(deserializeToken(pos, int32, sourceText, isJs));
    pos += TOKEN_SIZE;
  }
  return tokens;
}

/**
 * Deserialize a token from buffer at position `pos`.
 * @param {number} pos - Position in buffer containing Rust `Token` type
 * @param {Int32Array} int32 - Buffer containing AST in raw form as an `Int32Array`
 * @param {string} sourceText - Source for the file
 * @param {boolean} isJs - `true` if parsing in JS mode
 * @returns {Object} - Token object
 */
function deserializeToken(pos, int32, sourceText, isJs) {
  const pos32 = pos >> 2,
    start = int32[pos32],
    end = int32[pos32 + 1],
    kindAndFlags = int32[pos32 + 2];

  let value = sourceText.slice(start, end);

  // `Kind` is byte at index 8 in `Token`.
  // `Kind` has 12 variants numbered from 0 to 11.
  // We have to mask the bottom byte (`& 0xFF`), so may as well mask off bits which can't be set in `Kind` at same time.
  const kind = kindAndFlags & TOKEN_KIND_MASK;

  if (kind === REGEXP_KIND) {
    const patternEnd = value.lastIndexOf("/");
    return {
      type: "RegularExpression",
      value,
      regex: {
        pattern: value.slice(1, patternEnd),
        flags: value.slice(patternEnd + 1),
      },
      start,
      end,
    };
  }

  // Strip leading `#` from private identifiers
  if (kind === PRIVATE_IDENTIFIER_KIND) value = value.slice(1);

  // Unescape identifiers, keywords, and private identifiers in JS mode.
  // `is_escaped` flag is in byte 10 of `Token`, and is a `bool`.
  if (isJs && kind <= PRIVATE_IDENTIFIER_KIND && (kindAndFlags & 0x10000) !== 0) {
    value = unescapeIdentifier(value);
  }

  return { type: TOKEN_TYPES[kind], value, start, end };
}

/**
 * Unescape an identifier.
 *
 * We do this on JS side, because escaped identifiers are so extremely rare that this function
 * is never called in practice anyway.
 *
 * @param {string} name - Identifier name to unescape
 * @returns {string} - Unescaped identifier name
 */
function unescapeIdentifier(name) {
  return name.replace(/\\u(?:\{([0-9a-fA-F]+)\}|([0-9a-fA-F]{4}))/g, (_, hex1, hex2) =>
    String.fromCodePoint(parseInt(hex1 ?? hex2, 16)),
  );
}